qualia_core_db/inference/runtime/
memory_guard.rs1#[repr(C)]
10#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
11pub struct HostMemorySample {
12 pub available_host_bytes: u64,
14 pub inference_commit_bytes: u64,
16 pub active_requests: u32,
18}
19
20#[repr(C)]
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub struct MemoryPressurePolicy {
24 pub stop_admission_available_bytes: u64,
26 pub drain_available_bytes: u64,
28 pub max_inference_commit_bytes: u64,
30}
31
32#[derive(Clone, Copy, Debug, PartialEq, Eq)]
33pub enum MemoryPolicyError {
34 ZeroThreshold,
35 InvalidThresholdOrder,
36}
37
38impl MemoryPressurePolicy {
39 pub const fn new(
41 stop_admission_available_bytes: u64,
42 drain_available_bytes: u64,
43 max_inference_commit_bytes: u64,
44 ) -> Result<Self, MemoryPolicyError> {
45 if stop_admission_available_bytes == 0
46 || drain_available_bytes == 0
47 || max_inference_commit_bytes == 0
48 {
49 return Err(MemoryPolicyError::ZeroThreshold);
50 }
51 if drain_available_bytes >= stop_admission_available_bytes {
52 return Err(MemoryPolicyError::InvalidThresholdOrder);
53 }
54 Ok(Self {
55 stop_admission_available_bytes,
56 drain_available_bytes,
57 max_inference_commit_bytes,
58 })
59 }
60}
61
62#[derive(Clone, Copy, Debug, PartialEq, Eq)]
64pub enum MemoryPressureAction {
65 Continue,
66 StopAdmissionAndShedCaches,
69 DrainAndCheckpoint,
72}
73
74impl MemoryPressureAction {
75 pub const fn blocks_admission(self) -> bool {
77 !matches!(self, Self::Continue)
78 }
79
80 pub const fn requires_drain(self) -> bool {
82 matches!(self, Self::DrainAndCheckpoint)
83 }
84}
85
86#[derive(Clone, Copy, Debug)]
88pub struct MemoryPressureGuard {
89 policy: MemoryPressurePolicy,
90}
91
92impl MemoryPressureGuard {
93 pub const fn new(policy: MemoryPressurePolicy) -> Self {
95 Self { policy }
96 }
97
98 pub const fn policy(&self) -> MemoryPressurePolicy {
100 self.policy
101 }
102
103 pub const fn evaluate(&self, sample: HostMemorySample) -> MemoryPressureAction {
105 if sample.available_host_bytes <= self.policy.drain_available_bytes
106 || sample.inference_commit_bytes >= self.policy.max_inference_commit_bytes
107 {
108 MemoryPressureAction::DrainAndCheckpoint
109 } else if sample.available_host_bytes <= self.policy.stop_admission_available_bytes {
110 MemoryPressureAction::StopAdmissionAndShedCaches
111 } else {
112 MemoryPressureAction::Continue
113 }
114 }
115}
116
117#[cfg(test)]
118mod tests {
119 use super::*;
120
121 const GIB: u64 = 1024 * 1024 * 1024;
122
123 #[test]
124 fn transitions_before_host_memory_is_exhausted() {
125 let policy = MemoryPressurePolicy::new(8 * GIB, 4 * GIB, 52 * GIB).unwrap();
126 let guard = MemoryPressureGuard::new(policy);
127
128 assert_eq!(
129 guard.evaluate(HostMemorySample {
130 available_host_bytes: 12 * GIB,
131 inference_commit_bytes: 30 * GIB,
132 active_requests: 1,
133 }),
134 MemoryPressureAction::Continue
135 );
136 assert_eq!(
137 guard.evaluate(HostMemorySample {
138 available_host_bytes: 7 * GIB,
139 inference_commit_bytes: 30 * GIB,
140 active_requests: 1,
141 }),
142 MemoryPressureAction::StopAdmissionAndShedCaches
143 );
144 assert_eq!(
145 guard.evaluate(HostMemorySample {
146 available_host_bytes: 3 * GIB,
147 inference_commit_bytes: 30 * GIB,
148 active_requests: 1,
149 }),
150 MemoryPressureAction::DrainAndCheckpoint
151 );
152 }
153
154 #[test]
155 fn commit_ceiling_drains_even_when_available_memory_looks_healthy() {
156 let policy = MemoryPressurePolicy::new(8, 4, 100).unwrap();
157 let guard = MemoryPressureGuard::new(policy);
158 assert!(guard
159 .evaluate(HostMemorySample {
160 available_host_bytes: 1000,
161 inference_commit_bytes: 100,
162 active_requests: 1,
163 })
164 .requires_drain());
165 }
166}