Skip to main content

qualia_core_db/inference/runtime/scheduler/
request_table.rs

1use crate::inference::runtime::graph_assist::PrefixIdentity;
2use crate::inference::runtime::kv::paged::{
3    BlockPool, CopyOnWrite, SequenceBlockTable, TableError,
4};
5use crate::inference::runtime::kv::prefix::{PrefixKvError, PrefixKvStore};
6use crate::inference::runtime::memory_guard::{
7    HostMemorySample, MemoryPressureAction, MemoryPressureGuard,
8};
9
10use super::batch::{
11    RaggedBackendError, RaggedBatchItem, RaggedBatchOutput, RaggedBatchReceipt, RaggedDecodeBackend,
12};
13use super::drain::{DrainController, DrainError, DrainReason, DrainState};
14
15#[derive(Clone, Copy, Debug, PartialEq, Eq)]
16pub enum RequestState {
17    Empty,
18    Prefill,
19    Decode,
20}
21
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub struct Admission {
24    pub slot: u16,
25    pub prefix_hit: bool,
26    pub prefix_tokens: u32,
27}
28
29#[repr(C)]
30#[derive(Clone, Copy, Debug, PartialEq, Eq)]
31pub struct RequestView {
32    pub request_id: u64,
33    pub slot: u16,
34    pub state: RequestState,
35    pub token_count: u32,
36    pub prefix_tokens: u32,
37}
38
39#[derive(Clone, Copy, Debug, PartialEq, Eq)]
40pub enum SchedulerError {
41    DuplicateRequest,
42    Full,
43    UnknownRequest,
44    OutputTooSmall,
45    Draining,
46    MemoryPressure,
47    Drain(DrainError),
48    Prefix(PrefixKvError),
49    Table(TableError),
50}
51
52impl From<DrainError> for SchedulerError {
53    fn from(value: DrainError) -> Self {
54        Self::Drain(value)
55    }
56}
57
58#[derive(Clone, Copy, Debug, PartialEq, Eq)]
59pub enum DecodeRoundError {
60    Scheduler(SchedulerError),
61    ItemBufferTooSmall,
62    BlockTableBufferTooSmall,
63    OutputBufferTooSmall,
64    MissingInputToken,
65    Backend(RaggedBackendError),
66    BackendLaunchCount,
67    OutputIdentityMismatch,
68}
69
70impl From<SchedulerError> for DecodeRoundError {
71    fn from(value: SchedulerError) -> Self {
72        Self::Scheduler(value)
73    }
74}
75
76impl From<RaggedBackendError> for DecodeRoundError {
77    fn from(value: RaggedBackendError) -> Self {
78        Self::Backend(value)
79    }
80}
81
82impl From<PrefixKvError> for SchedulerError {
83    fn from(value: PrefixKvError) -> Self {
84        Self::Prefix(value)
85    }
86}
87
88impl From<TableError> for SchedulerError {
89    fn from(value: TableError) -> Self {
90        Self::Table(value)
91    }
92}
93
94struct RequestSlot {
95    request_id: u64,
96    state: RequestState,
97    token_count: u32,
98    prefix_tokens: u32,
99    next_token_id: u32,
100    token_ready: bool,
101    blocks: SequenceBlockTable,
102}
103
104impl RequestSlot {
105    fn new(logical_pages: u32) -> Self {
106        Self {
107            request_id: 0,
108            state: RequestState::Empty,
109            token_count: 0,
110            prefix_tokens: 0,
111            next_token_id: 0,
112            token_ready: false,
113            blocks: SequenceBlockTable::new(logical_pages),
114        }
115    }
116
117    fn clear(&mut self) {
118        self.request_id = 0;
119        self.state = RequestState::Empty;
120        self.token_count = 0;
121        self.prefix_tokens = 0;
122        self.next_token_id = 0;
123        self.token_ready = false;
124    }
125}
126
127/// Fixed-request-capacity scheduler. `REQUESTS` is the hard concurrent request bound.
128pub struct RequestScheduler<const REQUESTS: usize> {
129    slots: [RequestSlot; REQUESTS],
130    drain: DrainController,
131    memory_admission_paused: bool,
132}
133
134impl<const REQUESTS: usize> RequestScheduler<REQUESTS> {
135    /// Cold construction; allocates each fixed-capacity logical page table exactly once.
136    pub fn new(logical_pages: u32) -> Self {
137        Self {
138            slots: std::array::from_fn(|_| RequestSlot::new(logical_pages)),
139            drain: DrainController::new(0),
140            memory_admission_paused: false,
141        }
142    }
143
144    /// Creates a scheduler over `logical_pages` starting at `initial_generation`.
145    pub fn with_generation(logical_pages: u32, initial_generation: u64) -> Self {
146        Self {
147            slots: std::array::from_fn(|_| RequestSlot::new(logical_pages)),
148            drain: DrainController::new(initial_generation),
149            memory_admission_paused: false,
150        }
151    }
152
153    /// Drain controller for graceful shutdown.
154    pub fn drain(&self) -> &DrainController {
155        &self.drain
156    }
157
158    /// Mutable drain controller.
159    pub fn drain_mut(&mut self) -> &mut DrainController {
160        &mut self.drain
161    }
162
163    /// Initiates/updates drain state.
164    pub fn request_drain(
165        &mut self,
166        reason: DrainReason,
167        current_epoch: u64,
168    ) -> Result<DrainState, DrainError> {
169        self.drain
170            .request_drain(reason, self.active_count(), current_epoch)
171    }
172
173    /// Apply a host-memory policy sample before a new scheduling round.
174    ///
175    /// At critical pressure this transitions the scheduler into its existing
176    /// drain state, so active requests finish only through normal token-boundary
177    /// lifecycle handling. The caller performs cache shedding/checkpointing for
178    /// the returned action; this hot path remains allocation-free.
179    pub fn apply_memory_pressure(
180        &mut self,
181        guard: &MemoryPressureGuard,
182        sample: HostMemorySample,
183        current_epoch: u64,
184    ) -> MemoryPressureAction {
185        let action = guard.evaluate(sample);
186        match action {
187            MemoryPressureAction::Continue => {
188                if self.drain.is_admission_allowed() {
189                    self.memory_admission_paused = false;
190                }
191            }
192            MemoryPressureAction::StopAdmissionAndShedCaches => {
193                self.memory_admission_paused = true;
194            }
195            MemoryPressureAction::DrainAndCheckpoint => {
196                self.memory_admission_paused = true;
197                if self.drain.is_admission_allowed() {
198                    let _ = self.request_drain(DrainReason::MemoryPressure, current_epoch);
199                }
200            }
201        }
202        action
203    }
204
205    /// Currently active requests.
206    pub fn active_count(&self) -> usize {
207        self.slots
208            .iter()
209            .filter(|slot| slot.state != RequestState::Empty)
210            .count()
211    }
212
213    /// Admit a request and atomically attach graph-derived prefix pages when available.
214    pub fn admit_with_prefix<const ENTRIES: usize, const PAGES: usize>(
215        &mut self,
216        request_id: u64,
217        identity: Option<PrefixIdentity>,
218        prefixes: &PrefixKvStore<ENTRIES, PAGES>,
219        pool: &mut BlockPool,
220    ) -> Result<Admission, SchedulerError> {
221        if !self.drain.is_admission_allowed() {
222            return Err(SchedulerError::Draining);
223        }
224        if self.memory_admission_paused {
225            return Err(SchedulerError::MemoryPressure);
226        }
227        if self
228            .slots
229            .iter()
230            .any(|slot| slot.state != RequestState::Empty && slot.request_id == request_id)
231        {
232            return Err(SchedulerError::DuplicateRequest);
233        }
234        let index = self
235            .slots
236            .iter()
237            .position(|slot| slot.state == RequestState::Empty)
238            .ok_or(SchedulerError::Full)?;
239        let prefix_tokens = match identity {
240            Some(identity) => prefixes
241                .attach(identity, &mut self.slots[index].blocks, pool)?
242                .unwrap_or(0),
243            None => 0,
244        };
245        let slot = &mut self.slots[index];
246        slot.request_id = request_id;
247        slot.state = if prefix_tokens == 0 {
248            RequestState::Prefill
249        } else {
250            RequestState::Decode
251        };
252        slot.token_count = prefix_tokens;
253        slot.prefix_tokens = prefix_tokens;
254        Ok(Admission {
255            slot: index as u16,
256            prefix_hit: prefix_tokens != 0,
257            prefix_tokens,
258        })
259    }
260
261    /// Make a logical page writable. A `Copy` result instructs the backend to copy the physical
262    /// KV page before writing the next token.
263    pub fn ensure_writable(
264        &mut self,
265        request_id: u64,
266        logical_page: u32,
267        pool: &mut BlockPool,
268    ) -> Result<CopyOnWrite, SchedulerError> {
269        let slot = self.find_mut(request_id)?;
270        slot.blocks
271            .ensure_writable(logical_page, pool)
272            .map_err(Into::into)
273    }
274
275    /// Transitions `request_id` out of prefill; error when unknown/out-of-state.
276    pub fn mark_prefill_complete(&mut self, request_id: u64) -> Result<(), SchedulerError> {
277        self.find_mut(request_id)?.state = RequestState::Decode;
278        Ok(())
279    }
280
281    /// Publish the token consumed by the request's next decode step.
282    pub fn seed_decode_token(
283        &mut self,
284        request_id: u64,
285        token_id: u32,
286    ) -> Result<(), SchedulerError> {
287        let slot = self.find_mut(request_id)?;
288        slot.next_token_id = token_id;
289        slot.token_ready = true;
290        Ok(())
291    }
292
293    /// Records one decoded token for `request_id`; returns its sequence position.
294    pub fn record_token(&mut self, request_id: u64) -> Result<u32, SchedulerError> {
295        let slot = self.find_mut(request_id)?;
296        slot.token_count = slot.token_count.saturating_add(1);
297        Ok(slot.token_count)
298    }
299
300    /// Write all runnable requests into caller storage in stable slot order.
301    pub fn runnable_into(&self, out: &mut [RequestView]) -> Result<usize, SchedulerError> {
302        let required = self.active_count();
303        if out.len() < required {
304            return Err(SchedulerError::OutputTooSmall);
305        }
306        let mut written = 0usize;
307        for (index, slot) in self.slots.iter().enumerate() {
308            if slot.state == RequestState::Empty {
309                continue;
310            }
311            out[written] = RequestView {
312                request_id: slot.request_id,
313                slot: index as u16,
314                state: slot.state,
315                token_count: slot.token_count,
316                prefix_tokens: slot.prefix_tokens,
317            };
318            written += 1;
319        }
320        Ok(written)
321    }
322
323    /// Execute one ragged decode round with exactly one backend call.
324    ///
325    /// Active decode requests are lowered in stable scheduler-slot order. Their logical page
326    /// tables are concatenated into `block_table_scratch`; each item carries its table offset.
327    /// Outputs are identity-checked as a complete batch before scheduler state is mutated.
328    pub fn execute_decode_round<B: RaggedDecodeBackend>(
329        &mut self,
330        backend: &mut B,
331        item_scratch: &mut [RaggedBatchItem],
332        block_table_scratch: &mut [u32],
333        output_scratch: &mut [RaggedBatchOutput],
334    ) -> Result<RaggedBatchReceipt, DecodeRoundError> {
335        let decode_count = self
336            .slots
337            .iter()
338            .filter(|slot| slot.state == RequestState::Decode)
339            .count();
340        if item_scratch.len() < decode_count {
341            return Err(DecodeRoundError::ItemBufferTooSmall);
342        }
343        if output_scratch.len() < decode_count {
344            return Err(DecodeRoundError::OutputBufferTooSmall);
345        }
346        let required_table_entries = self
347            .slots
348            .iter()
349            .filter(|slot| slot.state == RequestState::Decode)
350            .map(|slot| slot.blocks.entries().len())
351            .sum::<usize>();
352        if block_table_scratch.len() < required_table_entries {
353            return Err(DecodeRoundError::BlockTableBufferTooSmall);
354        }
355
356        let mut item_count = 0usize;
357        let mut table_offset = 0usize;
358        for (index, slot) in self.slots.iter().enumerate() {
359            if slot.state != RequestState::Decode {
360                continue;
361            }
362            if !slot.token_ready {
363                return Err(DecodeRoundError::MissingInputToken);
364            }
365            let table = slot.blocks.entries();
366            block_table_scratch[table_offset..table_offset + table.len()].copy_from_slice(table);
367            item_scratch[item_count] = RaggedBatchItem {
368                request_id: slot.request_id,
369                slot: index as u32,
370                token_id: slot.next_token_id,
371                position: slot.token_count,
372                block_table_offset: table_offset as u32,
373                logical_pages: table.len() as u32,
374                _reserved: 0,
375            };
376            item_count += 1;
377            table_offset += table.len();
378        }
379        if item_count == 0 {
380            return Ok(RaggedBatchReceipt::default());
381        }
382
383        let receipt = backend.execute_ragged(
384            &item_scratch[..item_count],
385            &block_table_scratch[..table_offset],
386            &mut output_scratch[..item_count],
387        )?;
388        if receipt.batch_size != item_count as u32 {
389            return Err(DecodeRoundError::OutputIdentityMismatch);
390        }
391        if receipt.backend_launches != 1 {
392            return Err(DecodeRoundError::BackendLaunchCount);
393        }
394        for (item, output) in item_scratch[..item_count]
395            .iter()
396            .zip(&output_scratch[..item_count])
397        {
398            if output.request_id != item.request_id || output.slot != item.slot {
399                return Err(DecodeRoundError::OutputIdentityMismatch);
400            }
401        }
402        for output in &output_scratch[..item_count] {
403            let slot = &mut self.slots[output.slot as usize];
404            slot.next_token_id = output.next_token_id;
405            slot.token_count = slot.token_count.saturating_add(1);
406            slot.token_ready = true;
407        }
408        Ok(receipt)
409    }
410
411    /// Cancellation and normal completion have identical deterministic resource release.
412    pub fn finish(&mut self, request_id: u64, pool: &mut BlockPool) -> Result<(), SchedulerError> {
413        let slot = self.find_mut(request_id)?;
414        slot.blocks.release_all(pool)?;
415        slot.clear();
416        self.drain.poll_drain_progress(self.active_count());
417        Ok(())
418    }
419
420    fn find_mut(&mut self, request_id: u64) -> Result<&mut RequestSlot, SchedulerError> {
421        self.slots
422            .iter_mut()
423            .find(|slot| slot.state != RequestState::Empty && slot.request_id == request_id)
424            .ok_or(SchedulerError::UnknownRequest)
425    }
426}