qualia_core_db/inference/runtime/scheduler/
request_table.rs1use crate::inference::runtime::graph_assist::PrefixIdentity;
2use crate::inference::runtime::kv::paged::{
3 BlockPool, CopyOnWrite, SequenceBlockTable, TableError,
4};
5use crate::inference::runtime::kv::prefix::{PrefixKvError, PrefixKvStore};
6use crate::inference::runtime::memory_guard::{
7 HostMemorySample, MemoryPressureAction, MemoryPressureGuard,
8};
9
10use super::batch::{
11 RaggedBackendError, RaggedBatchItem, RaggedBatchOutput, RaggedBatchReceipt, RaggedDecodeBackend,
12};
13use super::drain::{DrainController, DrainError, DrainReason, DrainState};
14
15#[derive(Clone, Copy, Debug, PartialEq, Eq)]
16pub enum RequestState {
17 Empty,
18 Prefill,
19 Decode,
20}
21
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub struct Admission {
24 pub slot: u16,
25 pub prefix_hit: bool,
26 pub prefix_tokens: u32,
27}
28
29#[repr(C)]
30#[derive(Clone, Copy, Debug, PartialEq, Eq)]
31pub struct RequestView {
32 pub request_id: u64,
33 pub slot: u16,
34 pub state: RequestState,
35 pub token_count: u32,
36 pub prefix_tokens: u32,
37}
38
39#[derive(Clone, Copy, Debug, PartialEq, Eq)]
40pub enum SchedulerError {
41 DuplicateRequest,
42 Full,
43 UnknownRequest,
44 OutputTooSmall,
45 Draining,
46 MemoryPressure,
47 Drain(DrainError),
48 Prefix(PrefixKvError),
49 Table(TableError),
50}
51
52impl From<DrainError> for SchedulerError {
53 fn from(value: DrainError) -> Self {
54 Self::Drain(value)
55 }
56}
57
58#[derive(Clone, Copy, Debug, PartialEq, Eq)]
59pub enum DecodeRoundError {
60 Scheduler(SchedulerError),
61 ItemBufferTooSmall,
62 BlockTableBufferTooSmall,
63 OutputBufferTooSmall,
64 MissingInputToken,
65 Backend(RaggedBackendError),
66 BackendLaunchCount,
67 OutputIdentityMismatch,
68}
69
70impl From<SchedulerError> for DecodeRoundError {
71 fn from(value: SchedulerError) -> Self {
72 Self::Scheduler(value)
73 }
74}
75
76impl From<RaggedBackendError> for DecodeRoundError {
77 fn from(value: RaggedBackendError) -> Self {
78 Self::Backend(value)
79 }
80}
81
82impl From<PrefixKvError> for SchedulerError {
83 fn from(value: PrefixKvError) -> Self {
84 Self::Prefix(value)
85 }
86}
87
88impl From<TableError> for SchedulerError {
89 fn from(value: TableError) -> Self {
90 Self::Table(value)
91 }
92}
93
94struct RequestSlot {
95 request_id: u64,
96 state: RequestState,
97 token_count: u32,
98 prefix_tokens: u32,
99 next_token_id: u32,
100 token_ready: bool,
101 blocks: SequenceBlockTable,
102}
103
104impl RequestSlot {
105 fn new(logical_pages: u32) -> Self {
106 Self {
107 request_id: 0,
108 state: RequestState::Empty,
109 token_count: 0,
110 prefix_tokens: 0,
111 next_token_id: 0,
112 token_ready: false,
113 blocks: SequenceBlockTable::new(logical_pages),
114 }
115 }
116
117 fn clear(&mut self) {
118 self.request_id = 0;
119 self.state = RequestState::Empty;
120 self.token_count = 0;
121 self.prefix_tokens = 0;
122 self.next_token_id = 0;
123 self.token_ready = false;
124 }
125}
126
127pub struct RequestScheduler<const REQUESTS: usize> {
129 slots: [RequestSlot; REQUESTS],
130 drain: DrainController,
131 memory_admission_paused: bool,
132}
133
134impl<const REQUESTS: usize> RequestScheduler<REQUESTS> {
135 pub fn new(logical_pages: u32) -> Self {
137 Self {
138 slots: std::array::from_fn(|_| RequestSlot::new(logical_pages)),
139 drain: DrainController::new(0),
140 memory_admission_paused: false,
141 }
142 }
143
144 pub fn with_generation(logical_pages: u32, initial_generation: u64) -> Self {
146 Self {
147 slots: std::array::from_fn(|_| RequestSlot::new(logical_pages)),
148 drain: DrainController::new(initial_generation),
149 memory_admission_paused: false,
150 }
151 }
152
153 pub fn drain(&self) -> &DrainController {
155 &self.drain
156 }
157
158 pub fn drain_mut(&mut self) -> &mut DrainController {
160 &mut self.drain
161 }
162
163 pub fn request_drain(
165 &mut self,
166 reason: DrainReason,
167 current_epoch: u64,
168 ) -> Result<DrainState, DrainError> {
169 self.drain
170 .request_drain(reason, self.active_count(), current_epoch)
171 }
172
173 pub fn apply_memory_pressure(
180 &mut self,
181 guard: &MemoryPressureGuard,
182 sample: HostMemorySample,
183 current_epoch: u64,
184 ) -> MemoryPressureAction {
185 let action = guard.evaluate(sample);
186 match action {
187 MemoryPressureAction::Continue => {
188 if self.drain.is_admission_allowed() {
189 self.memory_admission_paused = false;
190 }
191 }
192 MemoryPressureAction::StopAdmissionAndShedCaches => {
193 self.memory_admission_paused = true;
194 }
195 MemoryPressureAction::DrainAndCheckpoint => {
196 self.memory_admission_paused = true;
197 if self.drain.is_admission_allowed() {
198 let _ = self.request_drain(DrainReason::MemoryPressure, current_epoch);
199 }
200 }
201 }
202 action
203 }
204
205 pub fn active_count(&self) -> usize {
207 self.slots
208 .iter()
209 .filter(|slot| slot.state != RequestState::Empty)
210 .count()
211 }
212
213 pub fn admit_with_prefix<const ENTRIES: usize, const PAGES: usize>(
215 &mut self,
216 request_id: u64,
217 identity: Option<PrefixIdentity>,
218 prefixes: &PrefixKvStore<ENTRIES, PAGES>,
219 pool: &mut BlockPool,
220 ) -> Result<Admission, SchedulerError> {
221 if !self.drain.is_admission_allowed() {
222 return Err(SchedulerError::Draining);
223 }
224 if self.memory_admission_paused {
225 return Err(SchedulerError::MemoryPressure);
226 }
227 if self
228 .slots
229 .iter()
230 .any(|slot| slot.state != RequestState::Empty && slot.request_id == request_id)
231 {
232 return Err(SchedulerError::DuplicateRequest);
233 }
234 let index = self
235 .slots
236 .iter()
237 .position(|slot| slot.state == RequestState::Empty)
238 .ok_or(SchedulerError::Full)?;
239 let prefix_tokens = match identity {
240 Some(identity) => prefixes
241 .attach(identity, &mut self.slots[index].blocks, pool)?
242 .unwrap_or(0),
243 None => 0,
244 };
245 let slot = &mut self.slots[index];
246 slot.request_id = request_id;
247 slot.state = if prefix_tokens == 0 {
248 RequestState::Prefill
249 } else {
250 RequestState::Decode
251 };
252 slot.token_count = prefix_tokens;
253 slot.prefix_tokens = prefix_tokens;
254 Ok(Admission {
255 slot: index as u16,
256 prefix_hit: prefix_tokens != 0,
257 prefix_tokens,
258 })
259 }
260
261 pub fn ensure_writable(
264 &mut self,
265 request_id: u64,
266 logical_page: u32,
267 pool: &mut BlockPool,
268 ) -> Result<CopyOnWrite, SchedulerError> {
269 let slot = self.find_mut(request_id)?;
270 slot.blocks
271 .ensure_writable(logical_page, pool)
272 .map_err(Into::into)
273 }
274
275 pub fn mark_prefill_complete(&mut self, request_id: u64) -> Result<(), SchedulerError> {
277 self.find_mut(request_id)?.state = RequestState::Decode;
278 Ok(())
279 }
280
281 pub fn seed_decode_token(
283 &mut self,
284 request_id: u64,
285 token_id: u32,
286 ) -> Result<(), SchedulerError> {
287 let slot = self.find_mut(request_id)?;
288 slot.next_token_id = token_id;
289 slot.token_ready = true;
290 Ok(())
291 }
292
293 pub fn record_token(&mut self, request_id: u64) -> Result<u32, SchedulerError> {
295 let slot = self.find_mut(request_id)?;
296 slot.token_count = slot.token_count.saturating_add(1);
297 Ok(slot.token_count)
298 }
299
300 pub fn runnable_into(&self, out: &mut [RequestView]) -> Result<usize, SchedulerError> {
302 let required = self.active_count();
303 if out.len() < required {
304 return Err(SchedulerError::OutputTooSmall);
305 }
306 let mut written = 0usize;
307 for (index, slot) in self.slots.iter().enumerate() {
308 if slot.state == RequestState::Empty {
309 continue;
310 }
311 out[written] = RequestView {
312 request_id: slot.request_id,
313 slot: index as u16,
314 state: slot.state,
315 token_count: slot.token_count,
316 prefix_tokens: slot.prefix_tokens,
317 };
318 written += 1;
319 }
320 Ok(written)
321 }
322
323 pub fn execute_decode_round<B: RaggedDecodeBackend>(
329 &mut self,
330 backend: &mut B,
331 item_scratch: &mut [RaggedBatchItem],
332 block_table_scratch: &mut [u32],
333 output_scratch: &mut [RaggedBatchOutput],
334 ) -> Result<RaggedBatchReceipt, DecodeRoundError> {
335 let decode_count = self
336 .slots
337 .iter()
338 .filter(|slot| slot.state == RequestState::Decode)
339 .count();
340 if item_scratch.len() < decode_count {
341 return Err(DecodeRoundError::ItemBufferTooSmall);
342 }
343 if output_scratch.len() < decode_count {
344 return Err(DecodeRoundError::OutputBufferTooSmall);
345 }
346 let required_table_entries = self
347 .slots
348 .iter()
349 .filter(|slot| slot.state == RequestState::Decode)
350 .map(|slot| slot.blocks.entries().len())
351 .sum::<usize>();
352 if block_table_scratch.len() < required_table_entries {
353 return Err(DecodeRoundError::BlockTableBufferTooSmall);
354 }
355
356 let mut item_count = 0usize;
357 let mut table_offset = 0usize;
358 for (index, slot) in self.slots.iter().enumerate() {
359 if slot.state != RequestState::Decode {
360 continue;
361 }
362 if !slot.token_ready {
363 return Err(DecodeRoundError::MissingInputToken);
364 }
365 let table = slot.blocks.entries();
366 block_table_scratch[table_offset..table_offset + table.len()].copy_from_slice(table);
367 item_scratch[item_count] = RaggedBatchItem {
368 request_id: slot.request_id,
369 slot: index as u32,
370 token_id: slot.next_token_id,
371 position: slot.token_count,
372 block_table_offset: table_offset as u32,
373 logical_pages: table.len() as u32,
374 _reserved: 0,
375 };
376 item_count += 1;
377 table_offset += table.len();
378 }
379 if item_count == 0 {
380 return Ok(RaggedBatchReceipt::default());
381 }
382
383 let receipt = backend.execute_ragged(
384 &item_scratch[..item_count],
385 &block_table_scratch[..table_offset],
386 &mut output_scratch[..item_count],
387 )?;
388 if receipt.batch_size != item_count as u32 {
389 return Err(DecodeRoundError::OutputIdentityMismatch);
390 }
391 if receipt.backend_launches != 1 {
392 return Err(DecodeRoundError::BackendLaunchCount);
393 }
394 for (item, output) in item_scratch[..item_count]
395 .iter()
396 .zip(&output_scratch[..item_count])
397 {
398 if output.request_id != item.request_id || output.slot != item.slot {
399 return Err(DecodeRoundError::OutputIdentityMismatch);
400 }
401 }
402 for output in &output_scratch[..item_count] {
403 let slot = &mut self.slots[output.slot as usize];
404 slot.next_token_id = output.next_token_id;
405 slot.token_count = slot.token_count.saturating_add(1);
406 slot.token_ready = true;
407 }
408 Ok(receipt)
409 }
410
411 pub fn finish(&mut self, request_id: u64, pool: &mut BlockPool) -> Result<(), SchedulerError> {
413 let slot = self.find_mut(request_id)?;
414 slot.blocks.release_all(pool)?;
415 slot.clear();
416 self.drain.poll_drain_progress(self.active_count());
417 Ok(())
418 }
419
420 fn find_mut(&mut self, request_id: u64) -> Result<&mut RequestSlot, SchedulerError> {
421 self.slots
422 .iter_mut()
423 .find(|slot| slot.state != RequestState::Empty && slot.request_id == request_id)
424 .ok_or(SchedulerError::UnknownRequest)
425 }
426}