Skip to main content

qualia_core_db/inference/runtime/scheduler/
drain.rs

1//! Safe maintenance, idle-drain state machine, and generation invalidation (Work Package F7).
2//!
3//! Provides deterministic coordination for quiescing active continuous-batching executions
4//! before model eviction, memory defragmentation, CUDA graph rebuilds, or block-pool re-sizing.
5
6/// Lifecycle states of the scheduler drain controller.
7#[derive(Clone, Copy, Debug, PartialEq, Eq)]
8pub enum DrainState {
9    /// Normal operation: admissions and scheduling proceed without restriction.
10    Normal,
11    /// Draining: new admissions are rejected/paused; in-flight requests proceed to completion.
12    Draining,
13    /// Quiesced: all in-flight requests have retired; safe to modify device context or pools.
14    Quiesced,
15    /// Maintenance: an exclusive maintenance operation is currently running.
16    Maintenance,
17}
18
19/// The trigger or reason for requesting a scheduler drain.
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21pub enum DrainReason {
22    /// Model residency swap or unmount.
23    ModelEviction,
24    /// Block pool expansion, compaction, or re-sizing.
25    PoolReconfiguration,
26    /// CUDA / hardware graph re-capture or cache invalidation.
27    GraphRebuild,
28    /// Thermal governor throttle or emergency cooling.
29    ThermalThrottle,
30    /// Host commit/available-memory policy requires a safe inference drain.
31    MemoryPressure,
32    /// Manual operator or administrative command.
33    Administrative,
34}
35
36/// Errors returned by drain controller state transitions.
37#[derive(Clone, Copy, Debug, PartialEq, Eq)]
38pub enum DrainError {
39    /// An operation is invalid in the current drain state.
40    InvalidState(DrainState),
41    /// Active requests still exist when attempting to enter maintenance directly.
42    NotYetQuiesced(usize),
43}
44
45/// Controller coordinating safe quiescence and maintenance transitions.
46#[derive(Clone, Copy, Debug)]
47pub struct DrainController {
48    state: DrainState,
49    reason: Option<DrainReason>,
50    active_generation: u64,
51    drain_start_epoch: u64,
52}
53
54impl DrainController {
55    /// Construct a new controller starting in `Normal` state at generation `initial_generation`.
56    pub const fn new(initial_generation: u64) -> Self {
57        Self {
58            state: DrainState::Normal,
59            reason: None,
60            active_generation: initial_generation,
61            drain_start_epoch: 0,
62        }
63    }
64
65    /// Current lifecycle state.
66    #[inline]
67    pub fn state(&self) -> DrainState {
68        self.state
69    }
70
71    /// Active model residency generation.
72    #[inline]
73    pub fn active_generation(&self) -> u64 {
74        self.active_generation
75    }
76
77    /// Current drain reason, if draining or in maintenance.
78    #[inline]
79    pub fn reason(&self) -> Option<DrainReason> {
80        self.reason
81    }
82
83    /// Whether new admissions are allowed into the scheduler.
84    #[inline]
85    pub fn is_admission_allowed(&self) -> bool {
86        self.state == DrainState::Normal
87    }
88
89    /// Request a drain for maintenance.
90    ///
91    /// If there are already 0 active requests, immediately transitions to `Quiesced`.
92    pub fn request_drain(
93        &mut self,
94        reason: DrainReason,
95        active_requests: usize,
96        current_epoch: u64,
97    ) -> Result<DrainState, DrainError> {
98        match self.state {
99            DrainState::Normal => {
100                self.reason = Some(reason);
101                self.drain_start_epoch = current_epoch;
102                if active_requests == 0 {
103                    self.state = DrainState::Quiesced;
104                } else {
105                    self.state = DrainState::Draining;
106                }
107                Ok(self.state)
108            }
109            DrainState::Draining => {
110                // Update reason if already draining
111                self.reason = Some(reason);
112                if active_requests == 0 {
113                    self.state = DrainState::Quiesced;
114                }
115                Ok(self.state)
116            }
117            DrainState::Quiesced | DrainState::Maintenance => {
118                Err(DrainError::InvalidState(self.state))
119            }
120        }
121    }
122
123    /// Notify the controller of the current count of active requests.
124    ///
125    /// If in `Draining` state and `active_requests == 0`, transitions to `Quiesced`.
126    pub fn poll_drain_progress(&mut self, active_requests: usize) -> DrainState {
127        if self.state == DrainState::Draining && active_requests == 0 {
128            self.state = DrainState::Quiesced;
129        }
130        self.state
131    }
132
133    /// Abort an in-progress drain and return to `Normal` operation without bumping generation.
134    pub fn abort_drain(&mut self) -> Result<(), DrainError> {
135        match self.state {
136            DrainState::Draining | DrainState::Quiesced => {
137                self.state = DrainState::Normal;
138                self.reason = None;
139                self.drain_start_epoch = 0;
140                Ok(())
141            }
142            DrainState::Normal | DrainState::Maintenance => {
143                Err(DrainError::InvalidState(self.state))
144            }
145        }
146    }
147
148    /// Begin exclusive maintenance. Requires the controller to be in `Quiesced` state.
149    pub fn begin_maintenance(&mut self) -> Result<(), DrainError> {
150        if self.state != DrainState::Quiesced {
151            return Err(DrainError::InvalidState(self.state));
152        }
153        self.state = DrainState::Maintenance;
154        Ok(())
155    }
156
157    /// Complete maintenance, bumping to `new_generation` and restoring `Normal` operation.
158    pub fn complete_maintenance(&mut self, new_generation: u64) -> Result<(), DrainError> {
159        if self.state != DrainState::Maintenance {
160            return Err(DrainError::InvalidState(self.state));
161        }
162        self.active_generation = new_generation;
163        self.state = DrainState::Normal;
164        self.reason = None;
165        self.drain_start_epoch = 0;
166        Ok(())
167    }
168}
169
170#[cfg(test)]
171mod tests {
172    use super::*;
173
174    #[test]
175    fn test_drain_lifecycle_with_active_requests() {
176        let mut controller = DrainController::new(1);
177        assert_eq!(controller.state(), DrainState::Normal);
178        assert!(controller.is_admission_allowed());
179
180        // 1. Request drain while 2 requests are active
181        let state = controller
182            .request_drain(DrainReason::ModelEviction, 2, 100)
183            .unwrap();
184        assert_eq!(state, DrainState::Draining);
185        assert!(!controller.is_admission_allowed());
186        assert_eq!(controller.reason(), Some(DrainReason::ModelEviction));
187
188        // 2. Poll while 1 request remains -> still Draining
189        assert_eq!(controller.poll_drain_progress(1), DrainState::Draining);
190
191        // 3. Poll with 0 requests -> transitions to Quiesced
192        assert_eq!(controller.poll_drain_progress(0), DrainState::Quiesced);
193        assert_eq!(controller.state(), DrainState::Quiesced);
194
195        // 4. Begin maintenance
196        assert!(controller.begin_maintenance().is_ok());
197        assert_eq!(controller.state(), DrainState::Maintenance);
198
199        // 5. Complete maintenance with bumped generation
200        assert!(controller.complete_maintenance(2).is_ok());
201        assert_eq!(controller.state(), DrainState::Normal);
202        assert_eq!(controller.active_generation(), 2);
203        assert!(controller.is_admission_allowed());
204        assert_eq!(controller.reason(), None);
205    }
206
207    #[test]
208    fn test_immediate_quiesce_when_idle() {
209        let mut controller = DrainController::new(10);
210        let state = controller
211            .request_drain(DrainReason::GraphRebuild, 0, 50)
212            .unwrap();
213        // Zero active requests -> directly Quiesced
214        assert_eq!(state, DrainState::Quiesced);
215
216        assert!(controller.begin_maintenance().is_ok());
217        assert!(controller.complete_maintenance(11).is_ok());
218        assert_eq!(controller.active_generation(), 11);
219    }
220
221    #[test]
222    fn test_abort_drain() {
223        let mut controller = DrainController::new(5);
224        controller
225            .request_drain(DrainReason::ThermalThrottle, 3, 20)
226            .unwrap();
227        assert_eq!(controller.state(), DrainState::Draining);
228
229        assert!(controller.abort_drain().is_ok());
230        assert_eq!(controller.state(), DrainState::Normal);
231        assert!(controller.is_admission_allowed());
232        assert_eq!(controller.active_generation(), 5);
233    }
234
235    #[test]
236    fn test_invalid_state_transitions() {
237        let mut controller = DrainController::new(1);
238        // Cannot begin maintenance when in Normal state
239        assert_eq!(
240            controller.begin_maintenance(),
241            Err(DrainError::InvalidState(DrainState::Normal))
242        );
243        // Cannot complete maintenance when in Normal state
244        assert_eq!(
245            controller.complete_maintenance(2),
246            Err(DrainError::InvalidState(DrainState::Normal))
247        );
248    }
249}