Skip to main content

qualia_core_db/inference/moe/
expert_cache.rs

1//! Host memory observations over canonical expert-cache mechanics.
2#[cfg(test)]
3use super::placement::ExpertPlacementPolicy;
4use etalon_inference::moe::offload::MoeOffloadManager as CoreManager;
5pub use etalon_inference::moe::offload::{
6    compute_principled_slot_capacity, query_dynamic_hardware_tier, DynamicVramStatus,
7    ExpertCacheTelemetry, ExpertResidencyProfile, PersonalHardwareTier, SlotAccessOutcome,
8    DEFAULT_GPU_EXPERT_SLOTS,
9};
10#[derive(Debug)]
11pub struct MoeOffloadManager(CoreManager);
12impl core::ops::Deref for MoeOffloadManager {
13    type Target = CoreManager;
14    fn deref(&self) -> &CoreManager {
15        &self.0
16    }
17}
18impl core::ops::DerefMut for MoeOffloadManager {
19    fn deref_mut(&mut self) -> &mut CoreManager {
20        &mut self.0
21    }
22}
23impl MoeOffloadManager {
24    pub fn new(slots: usize, bytes_per_expert: usize) -> Self {
25        Self(CoreManager::new(slots, bytes_per_expert))
26    }
27    pub fn with_tier(slots: usize, bytes_per_expert: usize, tier: PersonalHardwareTier) -> Self {
28        Self(CoreManager::with_tier(slots, bytes_per_expert, tier))
29    }
30    pub fn from_residency_profile(profile: ExpertResidencyProfile) -> (Self, PersonalHardwareTier) {
31        let (manager, tier) = CoreManager::from_residency_profile(profile);
32        (Self(manager), tier)
33    }
34    /// Create an offload manager by dynamically querying the host accelerator budget.
35    pub fn from_dynamic_query(
36        backbone_bytes: u64,
37        kv_cache_bytes: u64,
38        bytes_per_expert: usize,
39        total_experts: usize,
40    ) -> Self {
41        let (available_bytes, is_unified) = query_dynamic_accelerator_budget();
42        let profile = ExpertResidencyProfile {
43            available_accelerator_bytes: available_bytes,
44            backbone_bytes,
45            kv_cache_bytes,
46            bytes_per_expert,
47            total_experts,
48            is_unified_memory: is_unified,
49        };
50        let (manager, _) = Self::from_residency_profile(profile);
51        manager
52    }
53}
54
55/// Query dynamic accelerator budget and memory architecture (discrete vs unified).
56///
57/// Reads environment overrides (`QUALIA_VRAM_OVERRIDE_BYTES`, `QUALIA_VRAM_OVERRIDE_MB`, `QUALIA_FORCE_UNIFIED_MEMORY`),
58/// falling back to platform defaults (e.g. Apple Silicon unified memory on macOS or RTX A2000 discrete budget on Windows/Linux).
59pub fn query_dynamic_accelerator_budget() -> (u64, bool) {
60    let is_unified = std::env::var("QUALIA_FORCE_UNIFIED_MEMORY")
61        .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
62        .unwrap_or_else(|_| cfg!(target_os = "macos"));
63
64    if let Ok(bytes_str) = std::env::var("QUALIA_VRAM_OVERRIDE_BYTES") {
65        if let Ok(bytes) = bytes_str.trim().parse::<u64>() {
66            return (bytes, is_unified);
67        }
68    }
69
70    if let Ok(mb_str) = std::env::var("QUALIA_VRAM_OVERRIDE_MB") {
71        if let Ok(mb) = mb_str.trim().parse::<u64>() {
72            return (mb.saturating_mul(1024 * 1024), is_unified);
73        }
74    }
75
76    if is_unified {
77        // macOS Apple Silicon default: 16 GiB unified pool available for compute
78        (16 * 1024 * 1024 * 1024, true)
79    } else {
80        // Personal Discrete GPU default (e.g. RTX A2000 12GB with ~11.5 GiB usable VRAM headroom)
81        (11_500 * 1024 * 1024, false)
82    }
83}
84
85#[cfg(test)]
86mod tests {
87    use super::super::placement::ExpertPlacementCalibration;
88    use super::*;
89
90    #[test]
91    fn test_moe_offload_manager_lru_and_hits() {
92        let mut manager = MoeOffloadManager::new(2, 1024);
93
94        // Access expert 10 -> MissFetch into slot 0
95        let res1 = manager.resolve_expert(10, false, 0);
96        assert_eq!(
97            res1,
98            SlotAccessOutcome::MissFetch {
99                evict_slot_id: 0,
100                evicted_expert_id: None,
101            }
102        );
103
104        // Access expert 20 -> MissFetch into slot 1
105        let res2 = manager.resolve_expert(20, false, 0);
106        assert_eq!(
107            res2,
108            SlotAccessOutcome::MissFetch {
109                evict_slot_id: 1,
110                evicted_expert_id: None,
111            }
112        );
113
114        // Access expert 10 -> Hit in slot 0
115        let res3 = manager.resolve_expert(10, false, 0);
116        assert_eq!(res3, SlotAccessOutcome::Hit { slot_id: 0 });
117
118        // Access expert 30 -> Slot 1 is LRU (expert 20 was accessed before expert 10)
119        let res4 = manager.resolve_expert(30, false, 0);
120        assert_eq!(
121            res4,
122            SlotAccessOutcome::MissFetch {
123                evict_slot_id: 1,
124                evicted_expert_id: Some(20),
125            }
126        );
127
128        assert_eq!(manager.telemetry.hits, 1);
129        assert_eq!(manager.telemetry.misses, 3);
130        assert_eq!(manager.telemetry.gpu_fetches, 3);
131        assert_eq!(manager.telemetry.bytes_transferred, 3072);
132    }
133
134    #[test]
135    fn test_cpu_hybrid_dispatch_on_contention() {
136        let mut manager = MoeOffloadManager::new(2, 1024);
137        let res = manager.resolve_expert(99, true, 4);
138        assert_eq!(res, SlotAccessOutcome::MissComputeCpu);
139        assert_eq!(manager.telemetry.cpu_evaluations, 1);
140    }
141
142    #[test]
143    fn measured_placement_overrides_legacy_queue_ratio() {
144        let mut manager = MoeOffloadManager::new(8, 240 * 1024 * 1024);
145        manager.set_placement_policy(ExpertPlacementPolicy::from_calibration(
146            ExpertPlacementCalibration {
147                transfer_bytes_per_second: 1_000_000_000,
148                cpu_experts_per_second: 20.0,
149                gpu_experts_per_second: 1_000.0,
150                gpu_launch_overhead_ns: 100_000,
151                max_inflight_transfers: 4,
152            },
153            1,
154        ));
155        assert_eq!(
156            manager.resolve_expert(7, true, 0),
157            SlotAccessOutcome::MissComputeCpu
158        );
159    }
160
161    #[test]
162    fn test_principled_hardware_tier_computation() {
163        let bytes_per_exp = 240 * 1024 * 1024; // 240 MiB per expert
164        let backbone = 2800 * 1024 * 1024; // 2.8 GiB
165        let kv = 1024 * 1024 * 1024; // 1 GiB
166
167        // 1. Personal GPU 12GB (e.g. RTX A2000, 3060)
168        let (slots_12g, tier_12g) = compute_principled_slot_capacity(
169            11_500 * 1024 * 1024,
170            backbone,
171            kv,
172            bytes_per_exp,
173            256,
174            false,
175        );
176        assert_eq!(tier_12g, PersonalHardwareTier::PersonalGpuStaging);
177        assert_eq!(slots_12g, 31);
178
179        // 2. Apple Silicon 64GB Unified Memory (Mac Studio)
180        let (slots_mac, tier_mac) = compute_principled_slot_capacity(
181            64 * 1024 * 1024 * 1024,
182            backbone,
183            kv,
184            bytes_per_exp,
185            256,
186            true,
187        );
188        assert_eq!(tier_mac, PersonalHardwareTier::AppleUnifiedMemory);
189        assert_eq!(slots_mac, 256);
190
191        // 3. Workstation 80GB VRAM (A100/H100)
192        let (slots_workstation, tier_workstation) = compute_principled_slot_capacity(
193            80 * 1024 * 1024 * 1024,
194            backbone,
195            kv,
196            bytes_per_exp,
197            256,
198            false,
199        );
200        assert_eq!(tier_workstation, PersonalHardwareTier::WorkstationResident);
201        assert_eq!(slots_workstation, 256);
202
203        // 4. Constrained Edge 5GB VRAM
204        let (slots_edge, tier_edge) = compute_principled_slot_capacity(
205            5 * 1024 * 1024 * 1024,
206            backbone,
207            kv,
208            bytes_per_exp,
209            256,
210            false,
211        );
212        assert_eq!(tier_edge, PersonalHardwareTier::ConstrainedEdge);
213        assert_eq!(slots_edge, 5);
214    }
215
216    #[test]
217    fn profile_constructs_manager_at_measured_slot_capacity() {
218        let profile = ExpertResidencyProfile {
219            available_accelerator_bytes: 11_500 * 1024 * 1024,
220            backbone_bytes: 2_800 * 1024 * 1024,
221            kv_cache_bytes: 1_024 * 1024 * 1024,
222            bytes_per_expert: 240 * 1024 * 1024,
223            total_experts: 256,
224            is_unified_memory: false,
225        };
226        let (manager, tier) = MoeOffloadManager::from_residency_profile(profile);
227        assert_eq!(tier, PersonalHardwareTier::PersonalGpuStaging);
228        assert_eq!(manager.capacity(), 31);
229    }
230
231    #[test]
232    fn dynamic_query_and_vram_status_testing() {
233        let (budget, is_unified) = query_dynamic_accelerator_budget();
234        assert!(budget > 0);
235
236        let (slots, tier) = query_dynamic_hardware_tier(
237            budget,
238            2_800 * 1024 * 1024,
239            1_024 * 1024 * 1024,
240            240 * 1024 * 1024,
241            256,
242            is_unified,
243        );
244        assert!(slots >= 4);
245        assert_ne!(tier, PersonalHardwareTier::WorkstationResident);
246
247        let mut manager = MoeOffloadManager::from_dynamic_query(
248            2_800 * 1024 * 1024,
249            1_024 * 1024 * 1024,
250            240 * 1024 * 1024,
251            256,
252        );
253        assert_eq!(manager.capacity(), slots);
254
255        // Record some hits/misses to check telemetry in DynamicVramStatus
256        let outcome1 = manager.resolve_expert(10, false, 0);
257        assert!(matches!(outcome1, SlotAccessOutcome::MissFetch { .. }));
258
259        let outcome2 = manager.resolve_expert(10, false, 0);
260        assert!(matches!(outcome2, SlotAccessOutcome::Hit { slot_id: 0 }));
261
262        let status = manager.query_vram_status(budget);
263        assert_eq!(status.total_slots, slots);
264        assert_eq!(status.hits, 1);
265        assert_eq!(status.misses, 1);
266        assert!((status.hit_rate - 0.5).abs() < 1e-6);
267        assert_eq!(status.hardware_tier, manager.hardware_tier());
268    }
269
270    #[test]
271    fn exercise_staging_fallback_and_release_under_routed_workload() {
272        // Staging capacity = 4 slots (RTX A2000 constrained edge setup)
273        let mut manager = MoeOffloadManager::new(4, 240 * 1024 * 1024);
274
275        // Pin shared expert (ID 999) in slot 0: must never be evicted
276        manager.pin_expert(0, 999);
277
278        // Step 1: Experts 10 and 11 miss -> fetch into slots 1 and 2
279        let out_1a = manager.resolve_expert(10, false, 0);
280        assert!(matches!(
281            out_1a,
282            SlotAccessOutcome::MissFetch {
283                evict_slot_id: 1,
284                ..
285            }
286        ));
287        let out_1b = manager.resolve_expert(11, false, 0);
288        assert!(matches!(
289            out_1b,
290            SlotAccessOutcome::MissFetch {
291                evict_slot_id: 2,
292                ..
293            }
294        ));
295
296        // Step 2: Expert 10 hits in slot 1; Expert 12 misses -> fetches into slot 3
297        let out_2a = manager.resolve_expert(10, false, 0);
298        assert_eq!(out_2a, SlotAccessOutcome::Hit { slot_id: 1 });
299        let out_2b = manager.resolve_expert(12, false, 0);
300        assert!(matches!(
301            out_2b,
302            SlotAccessOutcome::MissFetch {
303                evict_slot_id: 3,
304                ..
305            }
306        ));
307
308        // Step 3: Expert 11 hits in slot 2; Expert 13 misses -> evicts LRU (slot 1 or 3, NEVER slot 0)
309        let out_3a = manager.resolve_expert(11, false, 0);
310        assert_eq!(out_3a, SlotAccessOutcome::Hit { slot_id: 2 });
311        let out_3b = manager.resolve_expert(13, false, 0);
312        if let SlotAccessOutcome::MissFetch { evict_slot_id, .. } = out_3b {
313            assert_ne!(
314                evict_slot_id, 0,
315                "Pinned shared expert in slot 0 must NEVER be evicted"
316            );
317        } else {
318            panic!("Expected MissFetch");
319        }
320
321        // Step 4: Access pinned shared expert (always a hit in slot 0)
322        let out_4a = manager.resolve_expert(999, false, 0);
323        assert_eq!(out_4a, SlotAccessOutcome::Hit { slot_id: 0 });
324
325        // Step 5: High PCIe contention triggers hybrid CPU execution fallback
326        let out_5a = manager.resolve_expert(14, true, 8);
327        assert_eq!(out_5a, SlotAccessOutcome::MissComputeCpu);
328
329        // Step 6: Normal query without CPU hybrid falls back to fetching into GPU
330        let out_6a = manager.resolve_expert(14, false, 0);
331        assert!(matches!(out_6a, SlotAccessOutcome::MissFetch { .. }));
332
333        // Verify telemetry integrity
334        let telem = manager.telemetry;
335        assert!(telem.hits >= 3);
336        assert!(telem.misses >= 5);
337        assert!(telem.gpu_fetches >= 4);
338        assert_eq!(telem.cpu_evaluations, 1);
339        assert!(telem.bytes_transferred >= 4 * 240 * 1024 * 1024);
340    }
341}