qualia_core_db/inference/runtime/scheduler/
policy.rs1#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub struct SchedulingPolicy {
9 pub max_prefill_chunk_tokens: u32,
11 pub max_consecutive_prefill_chunks: u32,
13}
14
15impl Default for SchedulingPolicy {
16 fn default() -> Self {
17 Self::DEFAULT
18 }
19}
20
21impl SchedulingPolicy {
22 pub const DEFAULT: Self = Self {
23 max_prefill_chunk_tokens: 512,
24 max_consecutive_prefill_chunks: 2,
25 };
26
27 pub fn new(max_prefill_chunk_tokens: u32, max_consecutive_prefill_chunks: u32) -> Self {
29 Self {
30 max_prefill_chunk_tokens: max_prefill_chunk_tokens.max(1),
31 max_consecutive_prefill_chunks: max_consecutive_prefill_chunks.max(1),
32 }
33 }
34
35 pub fn select_chunk_size(&self, remaining_tokens: u32) -> u32 {
37 remaining_tokens.min(self.max_prefill_chunk_tokens)
38 }
39
40 pub fn should_yield_prefill_to_decode(
42 &self,
43 consecutive_prefill_count: u32,
44 has_waiting_decodes: bool,
45 ) -> bool {
46 has_waiting_decodes && consecutive_prefill_count >= self.max_consecutive_prefill_chunks
47 }
48}
49
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub struct PrefillProgress {
53 pub total_input_tokens: u32,
54 pub processed_tokens: u32,
55}
56
57impl PrefillProgress {
58 pub fn new(total_input_tokens: u32) -> Self {
60 Self {
61 total_input_tokens,
62 processed_tokens: 0,
63 }
64 }
65
66 pub fn remaining_tokens(&self) -> u32 {
68 self.total_input_tokens
69 .saturating_sub(self.processed_tokens)
70 }
71
72 pub fn is_complete(&self) -> bool {
74 self.processed_tokens >= self.total_input_tokens
75 }
76
77 pub fn advance(&mut self, chunk_tokens: u32) -> u32 {
79 let actual = chunk_tokens.min(self.remaining_tokens());
80 self.processed_tokens += actual;
81 actual
82 }
83}
84
85#[cfg(test)]
86mod tests {
87 use super::*;
88
89 #[test]
90 fn test_chunk_size_selection() {
91 let policy = SchedulingPolicy::new(128, 2);
92 assert_eq!(policy.select_chunk_size(500), 128);
93 assert_eq!(policy.select_chunk_size(64), 64);
94 assert_eq!(policy.select_chunk_size(0), 0);
95 }
96
97 #[test]
98 fn test_prefill_yield_fairness() {
99 let policy = SchedulingPolicy::new(128, 2);
100 assert!(!policy.should_yield_prefill_to_decode(1, true));
102 assert!(policy.should_yield_prefill_to_decode(2, true));
104 assert!(!policy.should_yield_prefill_to_decode(2, false));
106 }
107
108 #[test]
109 fn test_prefill_progress_advancement() {
110 let mut progress = PrefillProgress::new(100);
111 assert_eq!(progress.remaining_tokens(), 100);
112 assert!(!progress.is_complete());
113
114 let advanced = progress.advance(40);
115 assert_eq!(advanced, 40);
116 assert_eq!(progress.remaining_tokens(), 60);
117
118 let advanced = progress.advance(70);
119 assert_eq!(advanced, 60);
120 assert_eq!(progress.remaining_tokens(), 0);
121 assert!(progress.is_complete());
122 }
123}