qualia_core_db/inference/runtime/scheduler/
drain.rs1#[derive(Clone, Copy, Debug, PartialEq, Eq)]
8pub enum DrainState {
9 Normal,
11 Draining,
13 Quiesced,
15 Maintenance,
17}
18
19#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21pub enum DrainReason {
22 ModelEviction,
24 PoolReconfiguration,
26 GraphRebuild,
28 ThermalThrottle,
30 MemoryPressure,
32 Administrative,
34}
35
36#[derive(Clone, Copy, Debug, PartialEq, Eq)]
38pub enum DrainError {
39 InvalidState(DrainState),
41 NotYetQuiesced(usize),
43}
44
45#[derive(Clone, Copy, Debug)]
47pub struct DrainController {
48 state: DrainState,
49 reason: Option<DrainReason>,
50 active_generation: u64,
51 drain_start_epoch: u64,
52}
53
54impl DrainController {
55 pub const fn new(initial_generation: u64) -> Self {
57 Self {
58 state: DrainState::Normal,
59 reason: None,
60 active_generation: initial_generation,
61 drain_start_epoch: 0,
62 }
63 }
64
65 #[inline]
67 pub fn state(&self) -> DrainState {
68 self.state
69 }
70
71 #[inline]
73 pub fn active_generation(&self) -> u64 {
74 self.active_generation
75 }
76
77 #[inline]
79 pub fn reason(&self) -> Option<DrainReason> {
80 self.reason
81 }
82
83 #[inline]
85 pub fn is_admission_allowed(&self) -> bool {
86 self.state == DrainState::Normal
87 }
88
89 pub fn request_drain(
93 &mut self,
94 reason: DrainReason,
95 active_requests: usize,
96 current_epoch: u64,
97 ) -> Result<DrainState, DrainError> {
98 match self.state {
99 DrainState::Normal => {
100 self.reason = Some(reason);
101 self.drain_start_epoch = current_epoch;
102 if active_requests == 0 {
103 self.state = DrainState::Quiesced;
104 } else {
105 self.state = DrainState::Draining;
106 }
107 Ok(self.state)
108 }
109 DrainState::Draining => {
110 self.reason = Some(reason);
112 if active_requests == 0 {
113 self.state = DrainState::Quiesced;
114 }
115 Ok(self.state)
116 }
117 DrainState::Quiesced | DrainState::Maintenance => {
118 Err(DrainError::InvalidState(self.state))
119 }
120 }
121 }
122
123 pub fn poll_drain_progress(&mut self, active_requests: usize) -> DrainState {
127 if self.state == DrainState::Draining && active_requests == 0 {
128 self.state = DrainState::Quiesced;
129 }
130 self.state
131 }
132
133 pub fn abort_drain(&mut self) -> Result<(), DrainError> {
135 match self.state {
136 DrainState::Draining | DrainState::Quiesced => {
137 self.state = DrainState::Normal;
138 self.reason = None;
139 self.drain_start_epoch = 0;
140 Ok(())
141 }
142 DrainState::Normal | DrainState::Maintenance => {
143 Err(DrainError::InvalidState(self.state))
144 }
145 }
146 }
147
148 pub fn begin_maintenance(&mut self) -> Result<(), DrainError> {
150 if self.state != DrainState::Quiesced {
151 return Err(DrainError::InvalidState(self.state));
152 }
153 self.state = DrainState::Maintenance;
154 Ok(())
155 }
156
157 pub fn complete_maintenance(&mut self, new_generation: u64) -> Result<(), DrainError> {
159 if self.state != DrainState::Maintenance {
160 return Err(DrainError::InvalidState(self.state));
161 }
162 self.active_generation = new_generation;
163 self.state = DrainState::Normal;
164 self.reason = None;
165 self.drain_start_epoch = 0;
166 Ok(())
167 }
168}
169
170#[cfg(test)]
171mod tests {
172 use super::*;
173
174 #[test]
175 fn test_drain_lifecycle_with_active_requests() {
176 let mut controller = DrainController::new(1);
177 assert_eq!(controller.state(), DrainState::Normal);
178 assert!(controller.is_admission_allowed());
179
180 let state = controller
182 .request_drain(DrainReason::ModelEviction, 2, 100)
183 .unwrap();
184 assert_eq!(state, DrainState::Draining);
185 assert!(!controller.is_admission_allowed());
186 assert_eq!(controller.reason(), Some(DrainReason::ModelEviction));
187
188 assert_eq!(controller.poll_drain_progress(1), DrainState::Draining);
190
191 assert_eq!(controller.poll_drain_progress(0), DrainState::Quiesced);
193 assert_eq!(controller.state(), DrainState::Quiesced);
194
195 assert!(controller.begin_maintenance().is_ok());
197 assert_eq!(controller.state(), DrainState::Maintenance);
198
199 assert!(controller.complete_maintenance(2).is_ok());
201 assert_eq!(controller.state(), DrainState::Normal);
202 assert_eq!(controller.active_generation(), 2);
203 assert!(controller.is_admission_allowed());
204 assert_eq!(controller.reason(), None);
205 }
206
207 #[test]
208 fn test_immediate_quiesce_when_idle() {
209 let mut controller = DrainController::new(10);
210 let state = controller
211 .request_drain(DrainReason::GraphRebuild, 0, 50)
212 .unwrap();
213 assert_eq!(state, DrainState::Quiesced);
215
216 assert!(controller.begin_maintenance().is_ok());
217 assert!(controller.complete_maintenance(11).is_ok());
218 assert_eq!(controller.active_generation(), 11);
219 }
220
221 #[test]
222 fn test_abort_drain() {
223 let mut controller = DrainController::new(5);
224 controller
225 .request_drain(DrainReason::ThermalThrottle, 3, 20)
226 .unwrap();
227 assert_eq!(controller.state(), DrainState::Draining);
228
229 assert!(controller.abort_drain().is_ok());
230 assert_eq!(controller.state(), DrainState::Normal);
231 assert!(controller.is_admission_allowed());
232 assert_eq!(controller.active_generation(), 5);
233 }
234
235 #[test]
236 fn test_invalid_state_transitions() {
237 let mut controller = DrainController::new(1);
238 assert_eq!(
240 controller.begin_maintenance(),
241 Err(DrainError::InvalidState(DrainState::Normal))
242 );
243 assert_eq!(
245 controller.complete_maintenance(2),
246 Err(DrainError::InvalidState(DrainState::Normal))
247 );
248 }
249}