qualia_core_db/inference/runtime/scheduler/
admission.rs1use crate::inference::runtime::budget::{BudgetError, MemoryPoolBudget, RequestReservation};
8use crate::inference::runtime::kv::paged::{BlockPool, PoolError};
9
10#[derive(Debug, Clone, PartialEq, Eq)]
12pub enum AdmissionError {
13 Budget(BudgetError),
14 Pool(PoolError),
15 InsufficientBlocks { requested: u32, available: usize },
16}
17
18impl From<BudgetError> for AdmissionError {
19 fn from(err: BudgetError) -> Self {
20 Self::Budget(err)
21 }
22}
23
24impl From<PoolError> for AdmissionError {
25 fn from(err: PoolError) -> Self {
26 Self::Pool(err)
27 }
28}
29
30#[derive(Debug)]
32pub struct AdmittedCurrencies {
33 pub reservation: RequestReservation,
34 pub allocated_blocks: Vec<u32>,
35}
36
37impl AdmittedCurrencies {
38 pub fn reserve(
47 budget: &MemoryPoolBudget,
48 pool: &mut BlockPool,
49 input_tokens: u32,
50 reserved_output_tokens: u32,
51 cow_blocks: u32,
52 active_cow_blocks: u32,
53 active_reserved_bytes: u64,
54 ) -> Result<Self, AdmissionError> {
55 let reservation = budget.reserve_with_active_cow(
57 input_tokens,
58 reserved_output_tokens,
59 cow_blocks,
60 active_cow_blocks,
61 active_reserved_bytes,
62 )?;
63
64 let total_blocks_needed = reservation
66 .required_blocks
67 .checked_add(reservation.cow_transient_blocks)
68 .ok_or(AdmissionError::Budget(BudgetError::IntegerOverflow))?;
69
70 if (total_blocks_needed as usize) > pool.free_count() {
71 return Err(AdmissionError::InsufficientBlocks {
72 requested: total_blocks_needed,
73 available: pool.free_count(),
74 });
75 }
76
77 let mut allocated_blocks = Vec::with_capacity(total_blocks_needed as usize);
79 for _ in 0..total_blocks_needed {
80 match pool.allocate() {
81 Ok(block) => allocated_blocks.push(block),
82 Err(err) => {
83 for b in allocated_blocks {
85 let _ = pool.release(b);
86 }
87 return Err(AdmissionError::Pool(err));
88 }
89 }
90 }
91
92 Ok(Self {
93 reservation,
94 allocated_blocks,
95 })
96 }
97
98 pub fn release(&mut self, pool: &mut BlockPool) {
100 for block in self.allocated_blocks.drain(..) {
101 let _ = pool.release(block);
102 }
103 }
104
105 pub fn allocated_blocks(&self) -> &[u32] {
107 &self.allocated_blocks
108 }
109}
110
111#[cfg(test)]
112mod tests {
113 use super::*;
114 use crate::inference::runtime::budget::{BlockGeometry, ModelMemoryProfile};
115
116 fn fixture_budget() -> MemoryPoolBudget {
117 let profile = ModelMemoryProfile::new(100, 100, 100, 100).unwrap();
118 let geometry = BlockGeometry::new(16, 64).unwrap();
119 MemoryPoolBudget::new(2000, profile, geometry, 20, 4).unwrap()
120 }
121
122 #[test]
123 fn test_currency_reservation_success_and_release() {
124 let budget = fixture_budget();
125 let mut pool = BlockPool::new(20);
126
127 let currencies = AdmittedCurrencies::reserve(&budget, &mut pool, 16, 16, 1, 0, 0);
129 assert!(currencies.is_ok());
130 let mut c = currencies.unwrap();
131 assert_eq!(c.reservation.required_blocks, 2);
132 assert_eq!(c.reservation.cow_transient_blocks, 1);
133 assert_eq!(c.allocated_blocks.len(), 3);
134 assert_eq!(pool.free_count(), 17);
135
136 c.release(&mut pool);
138 assert_eq!(pool.free_count(), 20);
139 assert!(c.allocated_blocks.is_empty());
140 }
141
142 #[test]
143 fn test_currency_reservation_insufficient_physical_blocks() {
144 let budget = fixture_budget();
145 let mut pool = BlockPool::new(2); let err = AdmittedCurrencies::reserve(&budget, &mut pool, 16, 16, 1, 0, 0);
149 assert!(matches!(
150 err,
151 Err(AdmissionError::InsufficientBlocks {
152 requested: 3,
153 available: 2
154 })
155 ));
156 assert_eq!(pool.free_count(), 2);
158 }
159
160 #[test]
161 fn test_currency_reservation_cumulative_cow_headroom_exceeded() {
162 let budget = fixture_budget();
163 let mut pool = BlockPool::new(20);
164
165 let err = AdmittedCurrencies::reserve(&budget, &mut pool, 16, 16, 2, 3, 0);
167 assert!(matches!(
168 err,
169 Err(AdmissionError::Budget(
170 BudgetError::InsufficientCowHeadroom {
171 requested_blocks: 5,
172 max_cow_blocks: 4
173 }
174 ))
175 ));
176 assert_eq!(pool.free_count(), 20);
177 }
178}