Skip to main content

qualia_core_db/inference/runtime/scheduler/
queue.rs

1//! Bounded request queue and worker intake lifecycle.
2//!
3//! Bridges asynchronous request intake to `RequestScheduler` and the sticky worker thread.
4//! Guarantees:
5//! 1. Bounded intake queue (rejects with `QueueFull` rather than growing unbounded).
6//! 2. Exact lifecycle states: `Queued`, `Admitted`, `Prefill`, `Decode`, `Cancelled`, `Completed`.
7//! 3. Cancellation at every phase has exactly one terminal result and clean currency release.
8
9use std::collections::VecDeque;
10
11/// Lifecycle state of a queued/running request.
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13pub enum IntakeState {
14    Queued,
15    Admitted { slot: u16 },
16    Cancelled,
17    Completed,
18}
19
20/// Request descriptor within the bounded scheduler intake.
21#[derive(Debug, Clone)]
22pub struct IntakeRequest {
23    pub request_id: u64,
24    pub prompt_tokens: u32,
25    pub max_output_tokens: u32,
26    pub state: IntakeState,
27    pub is_cancelled: bool,
28}
29
30impl IntakeRequest {
31    /// Creates an intake request with token bounds.
32    pub fn new(request_id: u64, prompt_tokens: u32, max_output_tokens: u32) -> Self {
33        Self {
34            request_id,
35            prompt_tokens,
36            max_output_tokens,
37            state: IntakeState::Queued,
38            is_cancelled: false,
39        }
40    }
41}
42
43/// Bounded FIFO intake queue for the request scheduler.
44#[derive(Debug)]
45pub struct BoundedIntakeQueue<const CAP: usize> {
46    queue: VecDeque<IntakeRequest>,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq)]
50pub enum IntakeError {
51    QueueFull,
52    DuplicateRequest,
53    NotFound,
54    AlreadyTerminal,
55}
56
57impl<const CAP: usize> Default for BoundedIntakeQueue<CAP> {
58    fn default() -> Self {
59        Self::new()
60    }
61}
62
63impl<const CAP: usize> BoundedIntakeQueue<CAP> {
64    /// Creates an empty bounded intake queue.
65    pub fn new() -> Self {
66        Self {
67            queue: VecDeque::with_capacity(CAP),
68        }
69    }
70
71    /// Queued requests.
72    pub fn len(&self) -> usize {
73        self.queue.len()
74    }
75
76    /// `true` when empty.
77    pub fn is_empty(&self) -> bool {
78        self.queue.is_empty()
79    }
80
81    /// Queue capacity.
82    pub fn capacity(&self) -> usize {
83        CAP
84    }
85
86    /// Enqueue a new request. Fails closed with `QueueFull` if capacity reached.
87    pub fn enqueue(&mut self, request: IntakeRequest) -> Result<(), IntakeError> {
88        if self.queue.len() >= CAP {
89            return Err(IntakeError::QueueFull);
90        }
91        if self
92            .queue
93            .iter()
94            .any(|r| r.request_id == request.request_id)
95        {
96            return Err(IntakeError::DuplicateRequest);
97        }
98        self.queue.push_back(request);
99        Ok(())
100    }
101
102    /// Pop the next non-cancelled request ready for admission.
103    pub fn pop_next_runnable(&mut self) -> Option<IntakeRequest> {
104        while let Some(req) = self.queue.pop_front() {
105            if !req.is_cancelled {
106                return Some(req);
107            }
108        }
109        None
110    }
111
112    /// Cancel a request by ID.
113    /// Returns the prior state if successfully cancelled.
114    pub fn cancel(&mut self, request_id: u64) -> Result<IntakeState, IntakeError> {
115        let req = self
116            .queue
117            .iter_mut()
118            .find(|r| r.request_id == request_id)
119            .ok_or(IntakeError::NotFound)?;
120
121        if req.is_cancelled || matches!(req.state, IntakeState::Cancelled | IntakeState::Completed)
122        {
123            return Err(IntakeError::AlreadyTerminal);
124        }
125
126        let prior = req.state;
127        req.is_cancelled = true;
128        req.state = IntakeState::Cancelled;
129        Ok(prior)
130    }
131
132    /// Find an intake request by ID.
133    pub fn find(&self, request_id: u64) -> Option<&IntakeRequest> {
134        self.queue.iter().find(|r| r.request_id == request_id)
135    }
136}
137
138#[cfg(test)]
139mod tests {
140    use super::*;
141
142    #[test]
143    fn test_bounded_queue_capacity() {
144        let mut queue = BoundedIntakeQueue::<2>::new();
145        assert_eq!(queue.capacity(), 2);
146
147        let r1 = IntakeRequest::new(101, 10, 20);
148        let r2 = IntakeRequest::new(102, 15, 25);
149        let r3 = IntakeRequest::new(103, 20, 30);
150
151        assert!(queue.enqueue(r1).is_ok());
152        assert!(queue.enqueue(r2).is_ok());
153        // Third must fail with QueueFull
154        assert_eq!(queue.enqueue(r3), Err(IntakeError::QueueFull));
155    }
156
157    #[test]
158    fn test_cancellation_before_admission() {
159        let mut queue = BoundedIntakeQueue::<4>::new();
160        let r1 = IntakeRequest::new(201, 10, 20);
161        let r2 = IntakeRequest::new(202, 15, 25);
162
163        queue.enqueue(r1).unwrap();
164        queue.enqueue(r2).unwrap();
165
166        // Cancel r1 while queued
167        let prior = queue.cancel(201).unwrap();
168        assert_eq!(prior, IntakeState::Queued);
169
170        // Popping next runnable skips cancelled r1 and returns r2
171        let next = queue.pop_next_runnable().unwrap();
172        assert_eq!(next.request_id, 202);
173
174        // Re-cancelling r1 returns AlreadyTerminal
175        assert_eq!(queue.cancel(201), Err(IntakeError::NotFound)); // Already drained
176    }
177}