Skip to main content

qualia_core_db/inference/runtime/
memory_guard.rs

1//! Fail-closed host-memory pressure policy for native inference.
2//!
3//! Platform probes provide committed and available-memory samples; this module
4//! makes the decision deterministic. It deliberately never tries to force a
5//! giant working set resident: that is not a Windows residency guarantee and
6//! can make system-wide pressure worse.
7
8/// A point-in-time sample supplied by a platform monitor.
9#[repr(C)]
10#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
11pub struct HostMemorySample {
12    /// Immediately available host memory, not virtual address-space size.
13    pub available_host_bytes: u64,
14    /// Commit attributed to the supervised inference process/job.
15    pub inference_commit_bytes: u64,
16    /// Number of active inference requests at this sample point.
17    pub active_requests: u32,
18}
19
20/// Cold-configured host-memory thresholds for one native runtime.
21#[repr(C)]
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub struct MemoryPressurePolicy {
24    /// Below this, no new prefill/decode requests may be admitted.
25    pub stop_admission_available_bytes: u64,
26    /// Below this, active work must drain at a token boundary.
27    pub drain_available_bytes: u64,
28    /// Commit ceiling for the supervised inference job/process.
29    pub max_inference_commit_bytes: u64,
30}
31
32#[derive(Clone, Copy, Debug, PartialEq, Eq)]
33pub enum MemoryPolicyError {
34    ZeroThreshold,
35    InvalidThresholdOrder,
36}
37
38impl MemoryPressurePolicy {
39    /// Creates a default `MemoryPressurePolicy`.
40    pub const fn new(
41        stop_admission_available_bytes: u64,
42        drain_available_bytes: u64,
43        max_inference_commit_bytes: u64,
44    ) -> Result<Self, MemoryPolicyError> {
45        if stop_admission_available_bytes == 0
46            || drain_available_bytes == 0
47            || max_inference_commit_bytes == 0
48        {
49            return Err(MemoryPolicyError::ZeroThreshold);
50        }
51        if drain_available_bytes >= stop_admission_available_bytes {
52            return Err(MemoryPolicyError::InvalidThresholdOrder);
53        }
54        Ok(Self {
55            stop_admission_available_bytes,
56            drain_available_bytes,
57            max_inference_commit_bytes,
58        })
59    }
60}
61
62/// Required reaction before an allocation failure or OS-level instability.
63#[derive(Clone, Copy, Debug, PartialEq, Eq)]
64pub enum MemoryPressureAction {
65    Continue,
66    /// Reject new work and release only reconstructible caches outside active
67    /// request state (for example, cold prefix/PLE neighbourhood entries).
68    StopAdmissionAndShedCaches,
69    /// Stop admission, drain at token boundaries, checkpoint compatible state,
70    /// then release the model/runtime allocation through normal lifecycle code.
71    DrainAndCheckpoint,
72}
73
74impl MemoryPressureAction {
75    /// `MemoryPressureAction::blocks_admission`.
76    pub const fn blocks_admission(self) -> bool {
77        !matches!(self, Self::Continue)
78    }
79
80    /// `MemoryPressureAction::requires_drain`.
81    pub const fn requires_drain(self) -> bool {
82        matches!(self, Self::DrainAndCheckpoint)
83    }
84}
85
86/// Pure, zero-allocation pressure evaluator.
87#[derive(Clone, Copy, Debug)]
88pub struct MemoryPressureGuard {
89    policy: MemoryPressurePolicy,
90}
91
92impl MemoryPressureGuard {
93    /// Creates a default `MemoryPressureGuard`.
94    pub const fn new(policy: MemoryPressurePolicy) -> Self {
95        Self { policy }
96    }
97
98    /// `MemoryPressureGuard::policy`.
99    pub const fn policy(&self) -> MemoryPressurePolicy {
100        self.policy
101    }
102
103    /// Evaluates and returns the result.
104    pub const fn evaluate(&self, sample: HostMemorySample) -> MemoryPressureAction {
105        if sample.available_host_bytes <= self.policy.drain_available_bytes
106            || sample.inference_commit_bytes >= self.policy.max_inference_commit_bytes
107        {
108            MemoryPressureAction::DrainAndCheckpoint
109        } else if sample.available_host_bytes <= self.policy.stop_admission_available_bytes {
110            MemoryPressureAction::StopAdmissionAndShedCaches
111        } else {
112            MemoryPressureAction::Continue
113        }
114    }
115}
116
117#[cfg(test)]
118mod tests {
119    use super::*;
120
121    const GIB: u64 = 1024 * 1024 * 1024;
122
123    #[test]
124    fn transitions_before_host_memory_is_exhausted() {
125        let policy = MemoryPressurePolicy::new(8 * GIB, 4 * GIB, 52 * GIB).unwrap();
126        let guard = MemoryPressureGuard::new(policy);
127
128        assert_eq!(
129            guard.evaluate(HostMemorySample {
130                available_host_bytes: 12 * GIB,
131                inference_commit_bytes: 30 * GIB,
132                active_requests: 1,
133            }),
134            MemoryPressureAction::Continue
135        );
136        assert_eq!(
137            guard.evaluate(HostMemorySample {
138                available_host_bytes: 7 * GIB,
139                inference_commit_bytes: 30 * GIB,
140                active_requests: 1,
141            }),
142            MemoryPressureAction::StopAdmissionAndShedCaches
143        );
144        assert_eq!(
145            guard.evaluate(HostMemorySample {
146                available_host_bytes: 3 * GIB,
147                inference_commit_bytes: 30 * GIB,
148                active_requests: 1,
149            }),
150            MemoryPressureAction::DrainAndCheckpoint
151        );
152    }
153
154    #[test]
155    fn commit_ceiling_drains_even_when_available_memory_looks_healthy() {
156        let policy = MemoryPressurePolicy::new(8, 4, 100).unwrap();
157        let guard = MemoryPressureGuard::new(policy);
158        assert!(guard
159            .evaluate(HostMemorySample {
160                available_host_bytes: 1000,
161                inference_commit_bytes: 100,
162                active_requests: 1,
163            })
164            .requires_drain());
165    }
166}