qualia_core_db/inference/runtime/scheduler/
queue.rs1use std::collections::VecDeque;
10
11#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13pub enum IntakeState {
14 Queued,
15 Admitted { slot: u16 },
16 Cancelled,
17 Completed,
18}
19
20#[derive(Debug, Clone)]
22pub struct IntakeRequest {
23 pub request_id: u64,
24 pub prompt_tokens: u32,
25 pub max_output_tokens: u32,
26 pub state: IntakeState,
27 pub is_cancelled: bool,
28}
29
30impl IntakeRequest {
31 pub fn new(request_id: u64, prompt_tokens: u32, max_output_tokens: u32) -> Self {
33 Self {
34 request_id,
35 prompt_tokens,
36 max_output_tokens,
37 state: IntakeState::Queued,
38 is_cancelled: false,
39 }
40 }
41}
42
43#[derive(Debug)]
45pub struct BoundedIntakeQueue<const CAP: usize> {
46 queue: VecDeque<IntakeRequest>,
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq)]
50pub enum IntakeError {
51 QueueFull,
52 DuplicateRequest,
53 NotFound,
54 AlreadyTerminal,
55}
56
57impl<const CAP: usize> Default for BoundedIntakeQueue<CAP> {
58 fn default() -> Self {
59 Self::new()
60 }
61}
62
63impl<const CAP: usize> BoundedIntakeQueue<CAP> {
64 pub fn new() -> Self {
66 Self {
67 queue: VecDeque::with_capacity(CAP),
68 }
69 }
70
71 pub fn len(&self) -> usize {
73 self.queue.len()
74 }
75
76 pub fn is_empty(&self) -> bool {
78 self.queue.is_empty()
79 }
80
81 pub fn capacity(&self) -> usize {
83 CAP
84 }
85
86 pub fn enqueue(&mut self, request: IntakeRequest) -> Result<(), IntakeError> {
88 if self.queue.len() >= CAP {
89 return Err(IntakeError::QueueFull);
90 }
91 if self
92 .queue
93 .iter()
94 .any(|r| r.request_id == request.request_id)
95 {
96 return Err(IntakeError::DuplicateRequest);
97 }
98 self.queue.push_back(request);
99 Ok(())
100 }
101
102 pub fn pop_next_runnable(&mut self) -> Option<IntakeRequest> {
104 while let Some(req) = self.queue.pop_front() {
105 if !req.is_cancelled {
106 return Some(req);
107 }
108 }
109 None
110 }
111
112 pub fn cancel(&mut self, request_id: u64) -> Result<IntakeState, IntakeError> {
115 let req = self
116 .queue
117 .iter_mut()
118 .find(|r| r.request_id == request_id)
119 .ok_or(IntakeError::NotFound)?;
120
121 if req.is_cancelled || matches!(req.state, IntakeState::Cancelled | IntakeState::Completed)
122 {
123 return Err(IntakeError::AlreadyTerminal);
124 }
125
126 let prior = req.state;
127 req.is_cancelled = true;
128 req.state = IntakeState::Cancelled;
129 Ok(prior)
130 }
131
132 pub fn find(&self, request_id: u64) -> Option<&IntakeRequest> {
134 self.queue.iter().find(|r| r.request_id == request_id)
135 }
136}
137
138#[cfg(test)]
139mod tests {
140 use super::*;
141
142 #[test]
143 fn test_bounded_queue_capacity() {
144 let mut queue = BoundedIntakeQueue::<2>::new();
145 assert_eq!(queue.capacity(), 2);
146
147 let r1 = IntakeRequest::new(101, 10, 20);
148 let r2 = IntakeRequest::new(102, 15, 25);
149 let r3 = IntakeRequest::new(103, 20, 30);
150
151 assert!(queue.enqueue(r1).is_ok());
152 assert!(queue.enqueue(r2).is_ok());
153 assert_eq!(queue.enqueue(r3), Err(IntakeError::QueueFull));
155 }
156
157 #[test]
158 fn test_cancellation_before_admission() {
159 let mut queue = BoundedIntakeQueue::<4>::new();
160 let r1 = IntakeRequest::new(201, 10, 20);
161 let r2 = IntakeRequest::new(202, 15, 25);
162
163 queue.enqueue(r1).unwrap();
164 queue.enqueue(r2).unwrap();
165
166 let prior = queue.cancel(201).unwrap();
168 assert_eq!(prior, IntakeState::Queued);
169
170 let next = queue.pop_next_runnable().unwrap();
172 assert_eq!(next.request_id, 202);
173
174 assert_eq!(queue.cancel(201), Err(IntakeError::NotFound)); }
177}