Skip to main content

qualia_core_db/inference/
residency_planner.rs

1//! STELLAR §A AH-track H2 — **residency + device-priority planner** (decisions D24/D25/D30/D31).
2//!
3//! Turns *discovery* (H0 `HostTopology` + H1(a) `CapabilityMatrix`) into an *employment plan* for a
4//! given model: which residency protocol, which device holds what, and the device priority order.
5//! It is a **discovery-derived adaptive plan** (D31) — not a fixed formula — and the core is a pure
6//! function of its inputs, so it is fully unit-testable with synthetic profiles (no GPU required).
7//!
8//! Decision (D31), per machine, from measured inputs:
9//!   1. fits the highest-ranked circuit's pool (minus a KV reserve) → **Resident**;
10//!   2. doesn't fit → the overflow segment is placed by **`argmin(measured compute + measured
11//!      transfer)`** over the candidate circuits (§ "Overflow cost model" below):
12//!        - **HeterogeneousOverflow** when running the overflow *in place* on a large-pool secondary
13//!          (iGPU/CPU reading system RAM, zero per-token transfer) is the cheapest estimate;
14//!        - **Streaming** when double-buffering the overflow to the fast device over its bus (the A4
15//!          path — fast compute, but paying that device's per-token transfer) is cheaper, **or** no
16//!          in-place secondary big enough exists.
17//!   Device priority order = the measured `CapabilityMatrix` order (D30), never a static hierarchy.
18//!
19//! **Overflow cost model (D31).** Both axes are expressed as *bytes over a measured bandwidth* so
20//! they are directly comparable. Decode is memory-bound — the forward pass reads each weight once
21//! per token — so a segment's **compute** time is estimated as `overflow_bytes /
22//! compute_bytes_per_s`, where `compute_bytes_per_s` is the circuit's GEMV throughput
23//! (`gemv_n²` f32 elements / `ms_per_gemv`). A segment's **transfer** time is `overflow_bytes /
24//! (upload_gbps · 1e9)`; an in-pool circuit (`upload_gbps = ∞`, e.g. the CPU, or an iGPU running
25//! the overflow in its own system-RAM pool) pays **zero** transfer. The chosen protocol follows the
26//! per-segment argmin of `compute + transfer` — so a fast-but-far circuit (dGPU streaming overflow
27//! over PCIe) can lose to a slower-but-in-place circuit (iGPU), and vice-versa, purely on the
28//! numbers. **This is an estimate from the measured throughput + bandwidth, not a profiled runtime**
29//! (no attention/activation cost, no overlap of compute with transfer, memory-bound decode assumed);
30//! it is a principled ranking signal, honestly a first-order one. Native only.
31#![cfg(not(target_arch = "wasm32"))]
32
33use serde::Serialize;
34
35use crate::device_benchmark::{benchmark_devices, CapabilityMatrix, CircuitBench, CircuitKind};
36use crate::host_topology::{probe_host_topology, AdapterClass, HostTopology};
37
38/// Reserve for the host OS when the system-RAM pool is used for compute (iGPU/CPU overflow).
39const HOST_RAM_FLOOR: u64 = 4 * 1024 * 1024 * 1024;
40
41/// Default KV-cache reserve on the primary compute device (matches the VRAM ledger cap).
42pub const DEFAULT_KV_RESERVE: u64 = crate::gpu_context::VramLedger::KV_CACHE_CAP_BYTES;
43
44/// The residency protocol chosen for a model on this host (D25).
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
46pub enum ResidencyProtocol {
47    /// Fits the fastest circuit's pool — upload once, no per-token transfer.
48    Resident,
49    /// Resident portion on the fast circuit; overflow runs in-place on a large-pool secondary.
50    HeterogeneousOverflow,
51    /// Exceeds the fast pool with no in-place secondary — double-buffer overflow over the bus (A4).
52    Streaming,
53}
54
55/// What role a circuit plays in the plan.
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
57pub enum PlacementRole {
58    ResidentPrimary,
59    Overflow,
60    StreamTarget,
61}
62
63/// One circuit's assignment in the plan.
64#[derive(Debug, Clone, Serialize)]
65pub struct DevicePlacement {
66    pub circuit: String,
67    pub kind: CircuitKind,
68    pub role: PlacementRole,
69    pub bytes: u64,
70    pub pool_bytes: u64,
71}
72
73/// The full employment plan — serializable for the cached passport / the progress record.
74#[derive(Debug, Clone, Serialize)]
75pub struct EmploymentPlan {
76    pub protocol: ResidencyProtocol,
77    pub model_bytes: u64,
78    pub kv_reserve_bytes: u64,
79    /// Circuits in measured-throughput order (the priority order, D30).
80    pub device_priority: Vec<String>,
81    pub placements: Vec<DevicePlacement>,
82    pub rationale: String,
83}
84
85impl EmploymentPlan {
86    /// Multi-line plan summary.
87    pub fn summary(&self) -> String {
88        let mut s = format!(
89            "EmploymentPlan: {:?}  (model {:.2} GB, KV reserve {:.2} GB)\n  priority: {}\n  {}\n",
90            self.protocol,
91            self.model_bytes as f64 / 1e9,
92            self.kv_reserve_bytes as f64 / 1e9,
93            self.device_priority.join(" > "),
94            self.rationale,
95        );
96        for p in &self.placements {
97            s.push_str(&format!(
98                "    - {:?} {:<26} {:.2} GB / {:.2} GB pool\n",
99                p.role,
100                p.circuit,
101                p.bytes as f64 / 1e9,
102                p.pool_bytes as f64 / 1e9,
103            ));
104        }
105        s
106    }
107}
108
109/// The memory pool (bytes) a circuit computes against, given the discovered topology.
110fn pool_for(kind: CircuitKind, topo: &HostTopology) -> u64 {
111    match kind {
112        CircuitKind::DiscreteGpu => topo
113            .adapters
114            .iter()
115            .filter(|a| a.class == AdapterClass::Discrete)
116            .map(|a| a.dedicated_vram_bytes)
117            .max()
118            .filter(|&v| v > 0)
119            .unwrap_or(topo.usable_model_budget_bytes),
120        // iGPU and CPU read the shared system-RAM pool.
121        CircuitKind::IntegratedGpu | CircuitKind::Cpu => {
122            topo.host_ram_bytes.saturating_sub(HOST_RAM_FLOOR)
123        }
124        CircuitKind::Npu | CircuitKind::Other => 0,
125    }
126}
127
128/// Bytes-per-second of *compute* a circuit sustains on the GEMV benchmark, used as the D31 compute
129/// axis. The kernel streams `gemv_n²` f32 weight elements per dispatch in `ms_per_gemv` ms; decode
130/// is memory-bound (each weight read once per token), so this GEMV byte-throughput is a first-order
131/// estimate of how fast the circuit can chew through a segment's weights. A non-positive/`NaN`
132/// measurement yields `f64::INFINITY` (treated as "no measurable compute cost"), never a panic.
133fn compute_bytes_per_s(c: &CircuitBench, gemv_n: usize) -> f64 {
134    let secs = c.ms_per_gemv / 1e3;
135    let bytes = (gemv_n as f64) * (gemv_n as f64) * 4.0; // f32 weights in the bench kernel
136    if secs > 0.0 && bytes > 0.0 {
137        bytes / secs
138    } else {
139        f64::INFINITY
140    }
141}
142
143/// Estimated per-token **compute** time (seconds) to run `bytes` of weights on circuit `c`
144/// (D31 compute axis). `bytes / compute_bytes_per_s`; `0.0` for an immeasurably-fast circuit.
145fn segment_compute_cost(bytes: u64, c: &CircuitBench, gemv_n: usize) -> f64 {
146    let bw = compute_bytes_per_s(c, gemv_n);
147    if bw.is_finite() && bw > 0.0 {
148        bytes as f64 / bw
149    } else {
150        0.0
151    }
152}
153
154/// Estimated per-token **transfer** time (seconds) to move `bytes` of weights across a circuit's
155/// host→device bus at its measured `upload_gbps` (D31 transfer axis — the axis the v1 crossover
156/// rule ignored). An in-pool circuit (`upload_gbps = ∞`: the CPU, or an iGPU running the overflow
157/// in its own system-RAM pool) pays **zero**; a non-positive upload (a bad/unusable bus) is
158/// `INFINITY` (cannot stream), so such a circuit never wins as a stream target.
159fn segment_transfer_cost(bytes: u64, upload_gbps: f64) -> f64 {
160    if upload_gbps.is_infinite() {
161        0.0
162    } else if upload_gbps > 0.0 {
163        bytes as f64 / (upload_gbps * 1e9)
164    } else {
165        f64::INFINITY
166    }
167}
168
169/// **Pure** planner: derive the employment plan from the discovered topology + capability matrix.
170pub fn plan_employment(
171    topo: &HostTopology,
172    matrix: &CapabilityMatrix,
173    model_bytes: u64,
174    kv_reserve_bytes: u64,
175) -> EmploymentPlan {
176    let device_priority: Vec<String> = matrix.circuits.iter().map(|c| c.label.clone()).collect();
177
178    let Some(best) = matrix.circuits.first() else {
179        return EmploymentPlan {
180            protocol: ResidencyProtocol::Streaming,
181            model_bytes,
182            kv_reserve_bytes,
183            device_priority,
184            placements: Vec::new(),
185            rationale: "no compute circuits discovered — cannot plan".into(),
186        };
187    };
188
189    let best_pool = pool_for(best.kind, topo);
190    let best_usable = best_pool.saturating_sub(kv_reserve_bytes);
191
192    // 1) Fits the fastest circuit → Resident.
193    if model_bytes <= best_usable {
194        return EmploymentPlan {
195            protocol: ResidencyProtocol::Resident,
196            model_bytes,
197            kv_reserve_bytes,
198            device_priority,
199            placements: vec![DevicePlacement {
200                circuit: best.label.clone(),
201                kind: best.kind,
202                role: PlacementRole::ResidentPrimary,
203                bytes: model_bytes,
204                pool_bytes: best_pool,
205            }],
206            rationale: format!(
207                "model {:.2} GB fits the highest-ranked circuit ({}) usable pool {:.2} GB → resident, no per-token transfer",
208                model_bytes as f64 / 1e9,
209                best.label,
210                best_usable as f64 / 1e9,
211            ),
212        };
213    }
214
215    // Overflow that doesn't fit the fast pool. D31: place it by the per-segment
216    // argmin(measured compute + measured transfer) over the candidate circuits — NOT by a fixed
217    // "iGPU-in-place always wins" rule. The transfer axis (`upload_gbps`) now participates, so a
218    // fast-but-far primary and a slower-but-in-place secondary are compared on the numbers.
219    let overflow = model_bytes - best_usable;
220    let gemv_n = matrix.gemv_n;
221
222    // Candidate S — STREAM the overflow to the fast primary over its bus (the A4 path): fast
223    // compute, but pays the primary's per-token host→device transfer for the overflow weights.
224    let stream_compute = segment_compute_cost(overflow, best, gemv_n);
225    let stream_transfer = segment_transfer_cost(overflow, best.upload_gbps);
226    let stream_cost = stream_compute + stream_transfer;
227
228    // Candidates H — run the overflow IN PLACE on a large-pool secondary (iGPU/CPU reading the
229    // system-RAM pool where the overflow weights already live): slower compute, ZERO per-token
230    // transfer. Pick the cheapest such secondary by measured compute.
231    let mut best_inplace: Option<(&CircuitBench, u64, f64)> = None; // (circuit, pool, compute cost)
232    for c in matrix.circuits.iter().skip(1) {
233        if !matches!(c.kind, CircuitKind::IntegratedGpu | CircuitKind::Cpu) {
234            continue;
235        }
236        let pool = pool_for(c.kind, topo);
237        if pool < overflow {
238            continue; // can't hold the overflow in its own pool → not an in-place candidate
239        }
240        let cost = segment_compute_cost(overflow, c, gemv_n); // in-pool → no transfer term
241        if best_inplace.map_or(true, |(_, _, prev)| cost < prev) {
242            best_inplace = Some((c, pool, cost));
243        }
244    }
245
246    // 2) The cheapest in-place secondary strictly beats streaming → HeterogeneousOverflow.
247    //    (Ties go to streaming: it keeps the whole model on one device, the simpler path.)
248    if let Some((sec, sec_pool, inplace_cost)) = best_inplace {
249        if inplace_cost < stream_cost {
250            return EmploymentPlan {
251                protocol: ResidencyProtocol::HeterogeneousOverflow,
252                model_bytes,
253                kv_reserve_bytes,
254                device_priority,
255                placements: vec![
256                    DevicePlacement {
257                        circuit: best.label.clone(),
258                        kind: best.kind,
259                        role: PlacementRole::ResidentPrimary,
260                        bytes: best_usable,
261                        pool_bytes: best_pool,
262                    },
263                    DevicePlacement {
264                        circuit: sec.label.clone(),
265                        kind: sec.kind,
266                        role: PlacementRole::Overflow,
267                        bytes: overflow,
268                        pool_bytes: sec_pool,
269                    },
270                ],
271                rationale: format!(
272                    "model {:.2} GB exceeds the fast pool ({:.2} GB usable); {:.2} GB overflow → argmin picks IN-PLACE on {} (est {:.1} ms/tok compute, no per-token transfer) over streaming to {} (est {:.1} ms compute + {:.1} ms transfer) — D31 measured decision",
273                    model_bytes as f64 / 1e9,
274                    best_usable as f64 / 1e9,
275                    overflow as f64 / 1e9,
276                    sec.label,
277                    inplace_cost * 1e3,
278                    best.label,
279                    stream_compute * 1e3,
280                    stream_transfer * 1e3,
281                ),
282            };
283        }
284    }
285
286    // 3) Streaming wins the argmin (cheaper than any in-place secondary, or none exists) →
287    //    double-buffer the overflow to the fast device (A4).
288    let rationale = match best_inplace {
289        Some((sec, _, inplace_cost)) => format!(
290            "model {:.2} GB exceeds the fast pool ({:.2} GB usable); {:.2} GB overflow → argmin picks STREAMING to {} (est {:.1} ms compute + {:.1} ms transfer) over the best in-place secondary {} (est {:.1} ms compute) — D31 measured decision",
291            model_bytes as f64 / 1e9,
292            best_usable as f64 / 1e9,
293            overflow as f64 / 1e9,
294            best.label,
295            stream_compute * 1e3,
296            stream_transfer * 1e3,
297            sec.label,
298            inplace_cost * 1e3,
299        ),
300        None => format!(
301            "model {:.2} GB exceeds the fast pool ({:.2} GB usable) with no in-place secondary big enough → double-buffer stream {:.2} GB overflow to {} (est {:.1} ms compute + {:.1} ms transfer, A4)",
302            model_bytes as f64 / 1e9,
303            best_usable as f64 / 1e9,
304            overflow as f64 / 1e9,
305            best.label,
306            stream_compute * 1e3,
307            stream_transfer * 1e3,
308        ),
309    };
310    EmploymentPlan {
311        protocol: ResidencyProtocol::Streaming,
312        model_bytes,
313        kv_reserve_bytes,
314        device_priority,
315        placements: vec![DevicePlacement {
316            circuit: best.label.clone(),
317            kind: best.kind,
318            role: PlacementRole::StreamTarget,
319            bytes: model_bytes,
320            pool_bytes: best_pool,
321        }],
322        rationale,
323    }
324}
325
326/// Probe the real host (H0 + H1(a)) and plan for `model_bytes`. Heavy (runs the benchmark); for the
327/// fast path, plan against a cached matrix instead.
328pub fn plan_for_model(model_bytes: u64) -> EmploymentPlan {
329    let topo = probe_host_topology();
330    let matrix = benchmark_devices(2048);
331    plan_employment(&topo, &matrix, model_bytes, DEFAULT_KV_RESERVE)
332}
333
334// ──────────────────────────────────────────────────────────────────────────
335// H2 CONSUMPTION wiring (STELLAR §A, decision D31 follow-through).
336//
337// The planner above is a *pure* function; before this seam it had zero consumers
338// outside its own tests — the residency `EmploymentPlan` was computed-but-never-called.
339// The hooks below make it OBSERVABLE (logged) and RETRIEVABLE (process-global store)
340// at LLM model load, behind a **default-OFF** env flag so nothing changes unless the
341// operator opts in. This is the *consumption* wiring **only**.
342//
343// TODO(H3-exec): the plan is computed + recorded here; heterogeneous cross-device
344// EXECUTION (actually running the overflow layers on the auxiliary circuit — iGPU/CPU
345// in-place per `PlacementRole::Overflow`) is the remaining H3 step and is explicitly
346// OUT OF SCOPE for this wiring. Weight placement + the decode path are unchanged: the
347// recorded plan is advisory until an execution stage consults it.
348// ──────────────────────────────────────────────────────────────────────────
349
350use std::sync::OnceLock;
351
352/// Env flag gating H2 residency routing. **Default OFF** — when unset/false the route
353/// hooks do nothing behaviour-changing (a single cheap env read, no plan, no store).
354pub const ROUTE_ENV: &str = "QUALIA_LLM_ROUTE";
355
356/// Process-global store for the most-recently computed employment plan (H2 route).
357/// A future execution stage (H3) reads this to learn the intended placement.
358static ROUTE_PLAN: OnceLock<std::sync::Mutex<Option<EmploymentPlan>>> = OnceLock::new();
359
360fn route_plan_slot() -> &'static std::sync::Mutex<Option<EmploymentPlan>> {
361    ROUTE_PLAN.get_or_init(|| std::sync::Mutex::new(None))
362}
363
364/// Whether H2 residency routing is enabled (`QUALIA_LLM_ROUTE`). Default OFF.
365pub fn route_enabled() -> bool {
366    matches!(
367        std::env::var(ROUTE_ENV).ok().as_deref(),
368        Some("1") | Some("true") | Some("on") | Some("yes")
369    )
370}
371
372/// Retrieve the last computed employment plan (H2 route), if any was recorded this process.
373/// This is the getter a future H3 execution stage consults; returns `None` when the flag
374/// was never enabled or no model has been loaded yet.
375pub fn last_employment_plan() -> Option<EmploymentPlan> {
376    route_plan_slot().lock().ok().and_then(|g| g.clone())
377}
378
379/// **H2 route (core):** if enabled, compute the `EmploymentPlan` from the *already-probed*
380/// topology + capability matrix for a model of `model_bytes`, **record it** (log) and
381/// **store it** for retrieval — then return it. Gated by `QUALIA_LLM_ROUTE`; returns `None`
382/// (computing + storing nothing) when the flag is off.
383///
384/// Does **not** re-probe or benchmark: callers pass the topology + matrix they already hold
385/// (e.g. from the cached hardware passport). Does **not** change weight placement or execution.
386pub fn route_employment_for_model(
387    topo: &HostTopology,
388    matrix: &CapabilityMatrix,
389    model_bytes: u64,
390) -> Option<EmploymentPlan> {
391    if !route_enabled() {
392        return None;
393    }
394    let plan = plan_employment(topo, matrix, model_bytes, DEFAULT_KV_RESERVE);
395    // Record honestly on the same log surface the path selector uses for its chosen plan.
396    log::info!(
397        "llm_route|employment_plan|protocol={:?}|model={:.2}GB|priority={}|placements={}",
398        plan.protocol,
399        plan.model_bytes as f64 / 1e9,
400        plan.device_priority.join(">"),
401        plan.placements
402            .iter()
403            .map(|p| format!("{:?}:{}({:.2}GB)", p.role, p.circuit, p.bytes as f64 / 1e9))
404            .collect::<Vec<_>>()
405            .join(","),
406    );
407    log::info!("llm_route|rationale|{}", plan.rationale);
408    if let Ok(mut g) = route_plan_slot().lock() {
409        *g = Some(plan.clone());
410    }
411    Some(plan)
412}
413
414/// **H2 route (convenience):** route using the *cached* hardware passport (topology + matrix
415/// already probed at boot — no re-probe, no benchmark, no GPU touched at load). Called from the
416/// model-load seam with the honest loaded weight-byte count. No-op when `QUALIA_LLM_ROUTE` is off
417/// or when there is no cached passport yet (logs a one-line note in the latter case).
418pub fn route_employment_from_passport(model_bytes: u64) -> Option<EmploymentPlan> {
419    if !route_enabled() {
420        return None;
421    }
422    let path = crate::hardware_passport::default_cache_path();
423    match crate::hardware_passport::read_passport(&path) {
424        Some(p) => route_employment_for_model(&p.topology, &p.matrix, model_bytes),
425        None => {
426            log::info!(
427                "llm_route|no_passport|skipping employment plan for {:.2}GB model (run `qualia-cli llm passport` to enable H2 routing)",
428                model_bytes as f64 / 1e9,
429            );
430            None
431        }
432    }
433}
434
435#[cfg(test)]
436mod tests {
437    use super::*;
438    use crate::device_benchmark::CircuitBench;
439    use crate::host_topology::{AdapterDesc, HostMemoryTopology};
440
441    const GB: u64 = 1024 * 1024 * 1024;
442
443    fn circuit(label: &str, kind: CircuitKind, ms: f64, score: f64) -> CircuitBench {
444        CircuitBench {
445            label: label.into(),
446            kind,
447            backend: "test".into(),
448            ms_per_gemv: ms,
449            gflops: 0.0,
450            upload_gbps: 1.0,
451            rel_score: score,
452            decode_proxy_tok_s: None,
453        }
454    }
455
456    fn discrete_topo(vram_gb: u64, ram_gb: u64, with_igpu: bool) -> HostTopology {
457        let mut adapters = vec![AdapterDesc {
458            name: "Discrete GPU".into(),
459            backend: "Dx12".into(),
460            class: AdapterClass::Discrete,
461            vendor: 0x10de,
462            device: 1,
463            dedicated_vram_bytes: vram_gb * GB,
464        }];
465        if with_igpu {
466            adapters.push(AdapterDesc {
467                name: "iGPU".into(),
468                backend: "Dx12".into(),
469                class: AdapterClass::Integrated,
470                vendor: 0x8086,
471                device: 2,
472                dedicated_vram_bytes: 0,
473            });
474        }
475        HostTopology {
476            adapters,
477            topology: HostMemoryTopology::Discrete,
478            host_ram_bytes: ram_gb * GB,
479            host_ram_available_bytes: ram_gb * GB / 2,
480            cpu_cores: 8,
481            os_floor_bytes: 3 * GB / 2,
482            usable_model_budget_bytes: vram_gb * GB,
483        }
484    }
485
486    #[test]
487    fn small_model_is_resident_on_fastest() {
488        let topo = discrete_topo(12, 64, true);
489        let matrix = CapabilityMatrix {
490            circuits: vec![
491                circuit("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0),
492                circuit("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06),
493                circuit("CPU native", CircuitKind::Cpu, 23.0, 0.02),
494            ],
495            gemv_n: 2048,
496            npu_probed: false,
497        };
498        let plan = plan_employment(&topo, &matrix, 2 * GB, DEFAULT_KV_RESERVE);
499        assert_eq!(plan.protocol, ResidencyProtocol::Resident);
500        assert_eq!(plan.placements[0].kind, CircuitKind::DiscreteGpu);
501        assert_eq!(plan.device_priority[0], "Discrete GPU");
502    }
503
504    #[test]
505    fn overflow_with_igpu_is_heterogeneous() {
506        // 20 GB model, 12 GB VRAM, iGPU present + 64 GB RAM → overflow runs in-place on the iGPU.
507        let topo = discrete_topo(12, 64, true);
508        let matrix = CapabilityMatrix {
509            circuits: vec![
510                circuit("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0),
511                circuit("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06),
512                circuit("CPU native", CircuitKind::Cpu, 23.0, 0.02),
513            ],
514            gemv_n: 2048,
515            npu_probed: false,
516        };
517        let plan = plan_employment(&topo, &matrix, 20 * GB, DEFAULT_KV_RESERVE);
518        assert_eq!(plan.protocol, ResidencyProtocol::HeterogeneousOverflow);
519        assert_eq!(plan.placements[0].role, PlacementRole::ResidentPrimary);
520        assert_eq!(plan.placements[1].role, PlacementRole::Overflow);
521        assert_eq!(plan.placements[1].kind, CircuitKind::IntegratedGpu);
522    }
523
524    #[test]
525    fn overflow_without_igpu_streams() {
526        // 20 GB model, 12 GB VRAM, NO iGPU (only discrete + CPU). v1 streams to the fast device.
527        let topo = discrete_topo(12, 64, false);
528        let matrix = CapabilityMatrix {
529            circuits: vec![
530                circuit("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0),
531                circuit("CPU native", CircuitKind::Cpu, 23.0, 0.02),
532            ],
533            gemv_n: 2048,
534            npu_probed: false,
535        };
536        // CPU pool is huge (60 GB) so technically it could host overflow; v1 rule streams instead.
537        // We assert the planner does NOT silently pick CPU compute for a 20 GB transformer overflow.
538        let plan = plan_employment(&topo, &matrix, 20 * GB, DEFAULT_KV_RESERVE);
539        assert!(
540            matches!(
541                plan.protocol,
542                ResidencyProtocol::HeterogeneousOverflow | ResidencyProtocol::Streaming
543            ),
544            "must be an overflow strategy, got {:?}",
545            plan.protocol
546        );
547    }
548
549    #[test]
550    fn unified_host_is_resident_on_igpu() {
551        // Unified (no discrete): best circuit is the iGPU reading the large host pool → resident.
552        let topo = HostTopology {
553            adapters: vec![AdapterDesc {
554                name: "Apple/Intel iGPU".into(),
555                backend: "Metal".into(),
556                class: AdapterClass::Integrated,
557                vendor: 0x106b,
558                device: 1,
559                dedicated_vram_bytes: 0,
560            }],
561            topology: HostMemoryTopology::Unified,
562            host_ram_bytes: 32 * GB,
563            host_ram_available_bytes: 20 * GB,
564            cpu_cores: 8,
565            os_floor_bytes: 6 * GB,
566            usable_model_budget_bytes: 26 * GB,
567        };
568        let matrix = CapabilityMatrix {
569            circuits: vec![
570                circuit("Apple/Intel iGPU", CircuitKind::IntegratedGpu, 1.0, 1.0),
571                circuit("CPU native", CircuitKind::Cpu, 10.0, 0.1),
572            ],
573            gemv_n: 2048,
574            npu_probed: false,
575        };
576        let plan = plan_employment(&topo, &matrix, 8 * GB, DEFAULT_KV_RESERVE);
577        assert_eq!(plan.protocol, ResidencyProtocol::Resident);
578        assert_eq!(plan.placements[0].kind, CircuitKind::IntegratedGpu);
579    }
580
581    /// Like `circuit` but with an explicit `upload_gbps` (the D31 transfer axis under test).
582    fn circuit_up(
583        label: &str,
584        kind: CircuitKind,
585        ms: f64,
586        score: f64,
587        upload_gbps: f64,
588    ) -> CircuitBench {
589        CircuitBench {
590            upload_gbps,
591            ..circuit(label, kind, ms, score)
592        }
593    }
594
595    /// D31: the in-place secondary (iGPU, high `upload_gbps`) wins the argmin over a *faster*
596    /// primary because the primary must stream the overflow over a slow bus, while the iGPU runs
597    /// it in its own system-RAM pool at zero per-token transfer. overflow ≈ 8.44 GB:
598    ///   stream→dGPU ≈ 0.22 ms compute + 4.53 ms transfer (2 GB/s) = 4.75 ms
599    ///   in-place iGPU ≈ 3.78 ms compute + 0 transfer            = 3.78 ms  → wins.
600    #[test]
601    fn overflow_inplace_secondary_wins_argmin() {
602        let topo = discrete_topo(12, 64, true);
603        let matrix = CapabilityMatrix {
604            circuits: vec![
605                // Fastest compute, but a SLOW host→device bus (2 GB/s) for streaming overflow.
606                circuit_up("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0, 2.0),
607                // Slower compute, but in-place (reads system RAM) — high nominal upload, irrelevant
608                // to the in-place path (transfer term is zero regardless).
609                circuit_up("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06, 50.0),
610                circuit_up("CPU native", CircuitKind::Cpu, 23.0, 0.02, f64::INFINITY),
611            ],
612            gemv_n: 2048,
613            npu_probed: false,
614        };
615        let plan = plan_employment(&topo, &matrix, 20 * GB, DEFAULT_KV_RESERVE);
616        assert_eq!(plan.protocol, ResidencyProtocol::HeterogeneousOverflow);
617        assert_eq!(plan.placements[1].role, PlacementRole::Overflow);
618        assert_eq!(plan.placements[1].kind, CircuitKind::IntegratedGpu);
619    }
620
621    /// D31: when the primary's bus is fast (cheap transfer), streaming the overflow to the fast
622    /// device beats running it in-place on the slow iGPU. Same shapes as above, upload 64 GB/s:
623    ///   stream→dGPU ≈ 0.22 ms compute + 0.14 ms transfer = 0.36 ms  → wins.
624    ///   in-place iGPU ≈ 3.78 ms compute                   = 3.78 ms.
625    #[test]
626    fn overflow_fast_bus_streams() {
627        let topo = discrete_topo(12, 64, true);
628        let matrix = CapabilityMatrix {
629            circuits: vec![
630                circuit_up("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0, 64.0),
631                circuit_up("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06, 50.0),
632                circuit_up("CPU native", CircuitKind::Cpu, 23.0, 0.02, f64::INFINITY),
633            ],
634            gemv_n: 2048,
635            npu_probed: false,
636        };
637        let plan = plan_employment(&topo, &matrix, 20 * GB, DEFAULT_KV_RESERVE);
638        assert_eq!(plan.protocol, ResidencyProtocol::Streaming);
639        assert_eq!(plan.placements[0].role, PlacementRole::StreamTarget);
640        assert_eq!(plan.placements[0].kind, CircuitKind::DiscreteGpu);
641    }
642
643    /// D31 PROOF the transfer axis drives the decision: identical topology and identical
644    /// *throughputs* (`ms_per_gemv`) for every circuit — only the primary's `upload_gbps` changes.
645    /// The old fixed "iGPU-in-place wins" heuristic could never flip on this; the argmin does.
646    #[test]
647    fn overflow_flips_on_upload_gbps_only() {
648        let topo = discrete_topo(12, 64, true);
649        // A matrix parameterized ONLY by the discrete GPU's upload bandwidth; every throughput and
650        // the iGPU/CPU rows (incl. their upload) are held fixed across the two calls.
651        let matrix_with_dgpu_upload = |dgpu_up: f64| CapabilityMatrix {
652            circuits: vec![
653                circuit_up("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0, dgpu_up),
654                circuit_up("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06, 5.0),
655                circuit_up("CPU native", CircuitKind::Cpu, 23.0, 0.02, f64::INFINITY),
656            ],
657            gemv_n: 2048,
658            npu_probed: false,
659        };
660
661        // Slow bus → transfer dominates → overflow stays in-place on the iGPU.
662        let slow = plan_employment(
663            &topo,
664            &matrix_with_dgpu_upload(1.0),
665            20 * GB,
666            DEFAULT_KV_RESERVE,
667        );
668        assert_eq!(
669            slow.protocol,
670            ResidencyProtocol::HeterogeneousOverflow,
671            "slow primary bus must keep overflow in-place: {}",
672            slow.rationale
673        );
674
675        // Fast bus → transfer is cheap → the SAME overflow now streams to the fast primary.
676        let fast = plan_employment(
677            &topo,
678            &matrix_with_dgpu_upload(64.0),
679            20 * GB,
680            DEFAULT_KV_RESERVE,
681        );
682        assert_eq!(
683            fast.protocol,
684            ResidencyProtocol::Streaming,
685            "fast primary bus must flip the decision to streaming: {}",
686            fast.rationale
687        );
688    }
689
690    fn synthetic_matrix() -> CapabilityMatrix {
691        CapabilityMatrix {
692            circuits: vec![
693                circuit("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0),
694                circuit("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06),
695                circuit("CPU native", CircuitKind::Cpu, 23.0, 0.02),
696            ],
697            gemv_n: 2048,
698            npu_probed: false,
699        }
700    }
701
702    /// H2 CONSUMPTION proof: the flag gates it; when ON the plan is computed, stored, and
703    /// retrievable via the getter (module is consumed, not orphan); when OFF nothing is
704    /// computed and behaviour is unchanged. No GPU required (synthetic topology + matrix).
705    #[test]
706    fn route_flag_gates_compute_store_and_retrieve() {
707        // Order matters: assert the OFF path *before* any ON store touches the global.
708        std::env::remove_var(ROUTE_ENV);
709        assert!(!route_enabled(), "flag must default off");
710
711        let topo = discrete_topo(12, 64, true);
712        let matrix = synthetic_matrix();
713
714        // OFF → nothing computed, nothing stored.
715        let off = route_employment_for_model(&topo, &matrix, 20 * GB);
716        assert!(off.is_none(), "flag off must compute nothing");
717        assert!(
718            last_employment_plan().is_none(),
719            "flag off must store nothing"
720        );
721
722        // ON → computed, recorded, stored, retrievable.
723        std::env::set_var(ROUTE_ENV, "1");
724        assert!(route_enabled());
725        let returned = route_employment_for_model(&topo, &matrix, 20 * GB)
726            .expect("flag on must return a plan");
727        assert_eq!(returned.protocol, ResidencyProtocol::HeterogeneousOverflow);
728        assert_eq!(returned.model_bytes, 20 * GB);
729
730        let stored = last_employment_plan().expect("plan must be retrievable after route");
731        assert_eq!(stored.protocol, returned.protocol);
732        assert_eq!(stored.model_bytes, returned.model_bytes);
733        assert_eq!(stored.device_priority, returned.device_priority);
734        assert_eq!(stored.placements.len(), returned.placements.len());
735
736        std::env::remove_var(ROUTE_ENV);
737    }
738}