Skip to main content

qualia_core_db/inference/runtime/scheduler/
policy.rs

1//! Deterministic prefill and decode scheduling policies.
2//!
3//! Provides deterministic fairness rules, bounded prefill chunk sizing, and prefill-vs-decode
4//! interleaving so long prefill operations cannot starve active decode requests.
5
6/// Policy governing chunk limits and prefill-vs-decode interleaving.
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub struct SchedulingPolicy {
9    /// Maximum tokens processed in a single prefill chunk.
10    pub max_prefill_chunk_tokens: u32,
11    /// Maximum consecutive prefill chunks permitted while decode requests are waiting.
12    pub max_consecutive_prefill_chunks: u32,
13}
14
15impl Default for SchedulingPolicy {
16    fn default() -> Self {
17        Self::DEFAULT
18    }
19}
20
21impl SchedulingPolicy {
22    pub const DEFAULT: Self = Self {
23        max_prefill_chunk_tokens: 512,
24        max_consecutive_prefill_chunks: 2,
25    };
26
27    /// Creates a policy with prefill-chunk size and consecutive-chunk cap.
28    pub fn new(max_prefill_chunk_tokens: u32, max_consecutive_prefill_chunks: u32) -> Self {
29        Self {
30            max_prefill_chunk_tokens: max_prefill_chunk_tokens.max(1),
31            max_consecutive_prefill_chunks: max_consecutive_prefill_chunks.max(1),
32        }
33    }
34
35    /// Select chunk length given remaining prompt tokens.
36    pub fn select_chunk_size(&self, remaining_tokens: u32) -> u32 {
37        remaining_tokens.min(self.max_prefill_chunk_tokens)
38    }
39
40    /// Decide whether prefill must yield to waiting decode requests.
41    pub fn should_yield_prefill_to_decode(
42        &self,
43        consecutive_prefill_count: u32,
44        has_waiting_decodes: bool,
45    ) -> bool {
46        has_waiting_decodes && consecutive_prefill_count >= self.max_consecutive_prefill_chunks
47    }
48}
49
50/// Tracking structure for request prefill progress.
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub struct PrefillProgress {
53    pub total_input_tokens: u32,
54    pub processed_tokens: u32,
55}
56
57impl PrefillProgress {
58    /// Tracks prefill for `total_input_tokens`.
59    pub fn new(total_input_tokens: u32) -> Self {
60        Self {
61            total_input_tokens,
62            processed_tokens: 0,
63        }
64    }
65
66    /// Prefill tokens not yet consumed.
67    pub fn remaining_tokens(&self) -> u32 {
68        self.total_input_tokens
69            .saturating_sub(self.processed_tokens)
70    }
71
72    /// `true` when prefill finished.
73    pub fn is_complete(&self) -> bool {
74        self.processed_tokens >= self.total_input_tokens
75    }
76
77    /// Consumes `chunk_tokens`; returns tokens consumed.
78    pub fn advance(&mut self, chunk_tokens: u32) -> u32 {
79        let actual = chunk_tokens.min(self.remaining_tokens());
80        self.processed_tokens += actual;
81        actual
82    }
83}
84
85#[cfg(test)]
86mod tests {
87    use super::*;
88
89    #[test]
90    fn test_chunk_size_selection() {
91        let policy = SchedulingPolicy::new(128, 2);
92        assert_eq!(policy.select_chunk_size(500), 128);
93        assert_eq!(policy.select_chunk_size(64), 64);
94        assert_eq!(policy.select_chunk_size(0), 0);
95    }
96
97    #[test]
98    fn test_prefill_yield_fairness() {
99        let policy = SchedulingPolicy::new(128, 2);
100        // Under threshold: should not yield even if decodes wait
101        assert!(!policy.should_yield_prefill_to_decode(1, true));
102        // At threshold with decodes waiting: must yield
103        assert!(policy.should_yield_prefill_to_decode(2, true));
104        // At threshold but NO decodes waiting: can continue
105        assert!(!policy.should_yield_prefill_to_decode(2, false));
106    }
107
108    #[test]
109    fn test_prefill_progress_advancement() {
110        let mut progress = PrefillProgress::new(100);
111        assert_eq!(progress.remaining_tokens(), 100);
112        assert!(!progress.is_complete());
113
114        let advanced = progress.advance(40);
115        assert_eq!(advanced, 40);
116        assert_eq!(progress.remaining_tokens(), 60);
117
118        let advanced = progress.advance(70);
119        assert_eq!(advanced, 60);
120        assert_eq!(progress.remaining_tokens(), 0);
121        assert!(progress.is_complete());
122    }
123}