1#[cfg(test)]
3use super::placement::ExpertPlacementPolicy;
4use etalon_inference::moe::offload::MoeOffloadManager as CoreManager;
5pub use etalon_inference::moe::offload::{
6 compute_principled_slot_capacity, query_dynamic_hardware_tier, DynamicVramStatus,
7 ExpertCacheTelemetry, ExpertResidencyProfile, PersonalHardwareTier, SlotAccessOutcome,
8 DEFAULT_GPU_EXPERT_SLOTS,
9};
10#[derive(Debug)]
11pub struct MoeOffloadManager(CoreManager);
12impl core::ops::Deref for MoeOffloadManager {
13 type Target = CoreManager;
14 fn deref(&self) -> &CoreManager {
15 &self.0
16 }
17}
18impl core::ops::DerefMut for MoeOffloadManager {
19 fn deref_mut(&mut self) -> &mut CoreManager {
20 &mut self.0
21 }
22}
23impl MoeOffloadManager {
24 pub fn new(slots: usize, bytes_per_expert: usize) -> Self {
25 Self(CoreManager::new(slots, bytes_per_expert))
26 }
27 pub fn with_tier(slots: usize, bytes_per_expert: usize, tier: PersonalHardwareTier) -> Self {
28 Self(CoreManager::with_tier(slots, bytes_per_expert, tier))
29 }
30 pub fn from_residency_profile(profile: ExpertResidencyProfile) -> (Self, PersonalHardwareTier) {
31 let (manager, tier) = CoreManager::from_residency_profile(profile);
32 (Self(manager), tier)
33 }
34 pub fn from_dynamic_query(
36 backbone_bytes: u64,
37 kv_cache_bytes: u64,
38 bytes_per_expert: usize,
39 total_experts: usize,
40 ) -> Self {
41 let (available_bytes, is_unified) = query_dynamic_accelerator_budget();
42 let profile = ExpertResidencyProfile {
43 available_accelerator_bytes: available_bytes,
44 backbone_bytes,
45 kv_cache_bytes,
46 bytes_per_expert,
47 total_experts,
48 is_unified_memory: is_unified,
49 };
50 let (manager, _) = Self::from_residency_profile(profile);
51 manager
52 }
53}
54
55pub fn query_dynamic_accelerator_budget() -> (u64, bool) {
60 let is_unified = std::env::var("QUALIA_FORCE_UNIFIED_MEMORY")
61 .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
62 .unwrap_or_else(|_| cfg!(target_os = "macos"));
63
64 if let Ok(bytes_str) = std::env::var("QUALIA_VRAM_OVERRIDE_BYTES") {
65 if let Ok(bytes) = bytes_str.trim().parse::<u64>() {
66 return (bytes, is_unified);
67 }
68 }
69
70 if let Ok(mb_str) = std::env::var("QUALIA_VRAM_OVERRIDE_MB") {
71 if let Ok(mb) = mb_str.trim().parse::<u64>() {
72 return (mb.saturating_mul(1024 * 1024), is_unified);
73 }
74 }
75
76 if is_unified {
77 (16 * 1024 * 1024 * 1024, true)
79 } else {
80 (11_500 * 1024 * 1024, false)
82 }
83}
84
85#[cfg(test)]
86mod tests {
87 use super::super::placement::ExpertPlacementCalibration;
88 use super::*;
89
90 #[test]
91 fn test_moe_offload_manager_lru_and_hits() {
92 let mut manager = MoeOffloadManager::new(2, 1024);
93
94 let res1 = manager.resolve_expert(10, false, 0);
96 assert_eq!(
97 res1,
98 SlotAccessOutcome::MissFetch {
99 evict_slot_id: 0,
100 evicted_expert_id: None,
101 }
102 );
103
104 let res2 = manager.resolve_expert(20, false, 0);
106 assert_eq!(
107 res2,
108 SlotAccessOutcome::MissFetch {
109 evict_slot_id: 1,
110 evicted_expert_id: None,
111 }
112 );
113
114 let res3 = manager.resolve_expert(10, false, 0);
116 assert_eq!(res3, SlotAccessOutcome::Hit { slot_id: 0 });
117
118 let res4 = manager.resolve_expert(30, false, 0);
120 assert_eq!(
121 res4,
122 SlotAccessOutcome::MissFetch {
123 evict_slot_id: 1,
124 evicted_expert_id: Some(20),
125 }
126 );
127
128 assert_eq!(manager.telemetry.hits, 1);
129 assert_eq!(manager.telemetry.misses, 3);
130 assert_eq!(manager.telemetry.gpu_fetches, 3);
131 assert_eq!(manager.telemetry.bytes_transferred, 3072);
132 }
133
134 #[test]
135 fn test_cpu_hybrid_dispatch_on_contention() {
136 let mut manager = MoeOffloadManager::new(2, 1024);
137 let res = manager.resolve_expert(99, true, 4);
138 assert_eq!(res, SlotAccessOutcome::MissComputeCpu);
139 assert_eq!(manager.telemetry.cpu_evaluations, 1);
140 }
141
142 #[test]
143 fn measured_placement_overrides_legacy_queue_ratio() {
144 let mut manager = MoeOffloadManager::new(8, 240 * 1024 * 1024);
145 manager.set_placement_policy(ExpertPlacementPolicy::from_calibration(
146 ExpertPlacementCalibration {
147 transfer_bytes_per_second: 1_000_000_000,
148 cpu_experts_per_second: 20.0,
149 gpu_experts_per_second: 1_000.0,
150 gpu_launch_overhead_ns: 100_000,
151 max_inflight_transfers: 4,
152 },
153 1,
154 ));
155 assert_eq!(
156 manager.resolve_expert(7, true, 0),
157 SlotAccessOutcome::MissComputeCpu
158 );
159 }
160
161 #[test]
162 fn test_principled_hardware_tier_computation() {
163 let bytes_per_exp = 240 * 1024 * 1024; let backbone = 2800 * 1024 * 1024; let kv = 1024 * 1024 * 1024; let (slots_12g, tier_12g) = compute_principled_slot_capacity(
169 11_500 * 1024 * 1024,
170 backbone,
171 kv,
172 bytes_per_exp,
173 256,
174 false,
175 );
176 assert_eq!(tier_12g, PersonalHardwareTier::PersonalGpuStaging);
177 assert_eq!(slots_12g, 31);
178
179 let (slots_mac, tier_mac) = compute_principled_slot_capacity(
181 64 * 1024 * 1024 * 1024,
182 backbone,
183 kv,
184 bytes_per_exp,
185 256,
186 true,
187 );
188 assert_eq!(tier_mac, PersonalHardwareTier::AppleUnifiedMemory);
189 assert_eq!(slots_mac, 256);
190
191 let (slots_workstation, tier_workstation) = compute_principled_slot_capacity(
193 80 * 1024 * 1024 * 1024,
194 backbone,
195 kv,
196 bytes_per_exp,
197 256,
198 false,
199 );
200 assert_eq!(tier_workstation, PersonalHardwareTier::WorkstationResident);
201 assert_eq!(slots_workstation, 256);
202
203 let (slots_edge, tier_edge) = compute_principled_slot_capacity(
205 5 * 1024 * 1024 * 1024,
206 backbone,
207 kv,
208 bytes_per_exp,
209 256,
210 false,
211 );
212 assert_eq!(tier_edge, PersonalHardwareTier::ConstrainedEdge);
213 assert_eq!(slots_edge, 5);
214 }
215
216 #[test]
217 fn profile_constructs_manager_at_measured_slot_capacity() {
218 let profile = ExpertResidencyProfile {
219 available_accelerator_bytes: 11_500 * 1024 * 1024,
220 backbone_bytes: 2_800 * 1024 * 1024,
221 kv_cache_bytes: 1_024 * 1024 * 1024,
222 bytes_per_expert: 240 * 1024 * 1024,
223 total_experts: 256,
224 is_unified_memory: false,
225 };
226 let (manager, tier) = MoeOffloadManager::from_residency_profile(profile);
227 assert_eq!(tier, PersonalHardwareTier::PersonalGpuStaging);
228 assert_eq!(manager.capacity(), 31);
229 }
230
231 #[test]
232 fn dynamic_query_and_vram_status_testing() {
233 let (budget, is_unified) = query_dynamic_accelerator_budget();
234 assert!(budget > 0);
235
236 let (slots, tier) = query_dynamic_hardware_tier(
237 budget,
238 2_800 * 1024 * 1024,
239 1_024 * 1024 * 1024,
240 240 * 1024 * 1024,
241 256,
242 is_unified,
243 );
244 assert!(slots >= 4);
245 assert_ne!(tier, PersonalHardwareTier::WorkstationResident);
246
247 let mut manager = MoeOffloadManager::from_dynamic_query(
248 2_800 * 1024 * 1024,
249 1_024 * 1024 * 1024,
250 240 * 1024 * 1024,
251 256,
252 );
253 assert_eq!(manager.capacity(), slots);
254
255 let outcome1 = manager.resolve_expert(10, false, 0);
257 assert!(matches!(outcome1, SlotAccessOutcome::MissFetch { .. }));
258
259 let outcome2 = manager.resolve_expert(10, false, 0);
260 assert!(matches!(outcome2, SlotAccessOutcome::Hit { slot_id: 0 }));
261
262 let status = manager.query_vram_status(budget);
263 assert_eq!(status.total_slots, slots);
264 assert_eq!(status.hits, 1);
265 assert_eq!(status.misses, 1);
266 assert!((status.hit_rate - 0.5).abs() < 1e-6);
267 assert_eq!(status.hardware_tier, manager.hardware_tier());
268 }
269
270 #[test]
271 fn exercise_staging_fallback_and_release_under_routed_workload() {
272 let mut manager = MoeOffloadManager::new(4, 240 * 1024 * 1024);
274
275 manager.pin_expert(0, 999);
277
278 let out_1a = manager.resolve_expert(10, false, 0);
280 assert!(matches!(
281 out_1a,
282 SlotAccessOutcome::MissFetch {
283 evict_slot_id: 1,
284 ..
285 }
286 ));
287 let out_1b = manager.resolve_expert(11, false, 0);
288 assert!(matches!(
289 out_1b,
290 SlotAccessOutcome::MissFetch {
291 evict_slot_id: 2,
292 ..
293 }
294 ));
295
296 let out_2a = manager.resolve_expert(10, false, 0);
298 assert_eq!(out_2a, SlotAccessOutcome::Hit { slot_id: 1 });
299 let out_2b = manager.resolve_expert(12, false, 0);
300 assert!(matches!(
301 out_2b,
302 SlotAccessOutcome::MissFetch {
303 evict_slot_id: 3,
304 ..
305 }
306 ));
307
308 let out_3a = manager.resolve_expert(11, false, 0);
310 assert_eq!(out_3a, SlotAccessOutcome::Hit { slot_id: 2 });
311 let out_3b = manager.resolve_expert(13, false, 0);
312 if let SlotAccessOutcome::MissFetch { evict_slot_id, .. } = out_3b {
313 assert_ne!(
314 evict_slot_id, 0,
315 "Pinned shared expert in slot 0 must NEVER be evicted"
316 );
317 } else {
318 panic!("Expected MissFetch");
319 }
320
321 let out_4a = manager.resolve_expert(999, false, 0);
323 assert_eq!(out_4a, SlotAccessOutcome::Hit { slot_id: 0 });
324
325 let out_5a = manager.resolve_expert(14, true, 8);
327 assert_eq!(out_5a, SlotAccessOutcome::MissComputeCpu);
328
329 let out_6a = manager.resolve_expert(14, false, 0);
331 assert!(matches!(out_6a, SlotAccessOutcome::MissFetch { .. }));
332
333 let telem = manager.telemetry;
335 assert!(telem.hits >= 3);
336 assert!(telem.misses >= 5);
337 assert!(telem.gpu_fetches >= 4);
338 assert_eq!(telem.cpu_evaluations, 1);
339 assert!(telem.bytes_transferred >= 4 * 240 * 1024 * 1024);
340 }
341}