1#![cfg(not(target_arch = "wasm32"))]
32
33use serde::Serialize;
34
35use crate::device_benchmark::{benchmark_devices, CapabilityMatrix, CircuitBench, CircuitKind};
36use crate::host_topology::{probe_host_topology, AdapterClass, HostTopology};
37
38const HOST_RAM_FLOOR: u64 = 4 * 1024 * 1024 * 1024;
40
41pub const DEFAULT_KV_RESERVE: u64 = crate::gpu_context::VramLedger::KV_CACHE_CAP_BYTES;
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
46pub enum ResidencyProtocol {
47 Resident,
49 HeterogeneousOverflow,
51 Streaming,
53}
54
55#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
57pub enum PlacementRole {
58 ResidentPrimary,
59 Overflow,
60 StreamTarget,
61}
62
63#[derive(Debug, Clone, Serialize)]
65pub struct DevicePlacement {
66 pub circuit: String,
67 pub kind: CircuitKind,
68 pub role: PlacementRole,
69 pub bytes: u64,
70 pub pool_bytes: u64,
71}
72
73#[derive(Debug, Clone, Serialize)]
75pub struct EmploymentPlan {
76 pub protocol: ResidencyProtocol,
77 pub model_bytes: u64,
78 pub kv_reserve_bytes: u64,
79 pub device_priority: Vec<String>,
81 pub placements: Vec<DevicePlacement>,
82 pub rationale: String,
83}
84
85impl EmploymentPlan {
86 pub fn summary(&self) -> String {
88 let mut s = format!(
89 "EmploymentPlan: {:?} (model {:.2} GB, KV reserve {:.2} GB)\n priority: {}\n {}\n",
90 self.protocol,
91 self.model_bytes as f64 / 1e9,
92 self.kv_reserve_bytes as f64 / 1e9,
93 self.device_priority.join(" > "),
94 self.rationale,
95 );
96 for p in &self.placements {
97 s.push_str(&format!(
98 " - {:?} {:<26} {:.2} GB / {:.2} GB pool\n",
99 p.role,
100 p.circuit,
101 p.bytes as f64 / 1e9,
102 p.pool_bytes as f64 / 1e9,
103 ));
104 }
105 s
106 }
107}
108
109fn pool_for(kind: CircuitKind, topo: &HostTopology) -> u64 {
111 match kind {
112 CircuitKind::DiscreteGpu => topo
113 .adapters
114 .iter()
115 .filter(|a| a.class == AdapterClass::Discrete)
116 .map(|a| a.dedicated_vram_bytes)
117 .max()
118 .filter(|&v| v > 0)
119 .unwrap_or(topo.usable_model_budget_bytes),
120 CircuitKind::IntegratedGpu | CircuitKind::Cpu => {
122 topo.host_ram_bytes.saturating_sub(HOST_RAM_FLOOR)
123 }
124 CircuitKind::Npu | CircuitKind::Other => 0,
125 }
126}
127
128fn compute_bytes_per_s(c: &CircuitBench, gemv_n: usize) -> f64 {
134 let secs = c.ms_per_gemv / 1e3;
135 let bytes = (gemv_n as f64) * (gemv_n as f64) * 4.0; if secs > 0.0 && bytes > 0.0 {
137 bytes / secs
138 } else {
139 f64::INFINITY
140 }
141}
142
143fn segment_compute_cost(bytes: u64, c: &CircuitBench, gemv_n: usize) -> f64 {
146 let bw = compute_bytes_per_s(c, gemv_n);
147 if bw.is_finite() && bw > 0.0 {
148 bytes as f64 / bw
149 } else {
150 0.0
151 }
152}
153
154fn segment_transfer_cost(bytes: u64, upload_gbps: f64) -> f64 {
160 if upload_gbps.is_infinite() {
161 0.0
162 } else if upload_gbps > 0.0 {
163 bytes as f64 / (upload_gbps * 1e9)
164 } else {
165 f64::INFINITY
166 }
167}
168
169pub fn plan_employment(
171 topo: &HostTopology,
172 matrix: &CapabilityMatrix,
173 model_bytes: u64,
174 kv_reserve_bytes: u64,
175) -> EmploymentPlan {
176 let device_priority: Vec<String> = matrix.circuits.iter().map(|c| c.label.clone()).collect();
177
178 let Some(best) = matrix.circuits.first() else {
179 return EmploymentPlan {
180 protocol: ResidencyProtocol::Streaming,
181 model_bytes,
182 kv_reserve_bytes,
183 device_priority,
184 placements: Vec::new(),
185 rationale: "no compute circuits discovered — cannot plan".into(),
186 };
187 };
188
189 let best_pool = pool_for(best.kind, topo);
190 let best_usable = best_pool.saturating_sub(kv_reserve_bytes);
191
192 if model_bytes <= best_usable {
194 return EmploymentPlan {
195 protocol: ResidencyProtocol::Resident,
196 model_bytes,
197 kv_reserve_bytes,
198 device_priority,
199 placements: vec![DevicePlacement {
200 circuit: best.label.clone(),
201 kind: best.kind,
202 role: PlacementRole::ResidentPrimary,
203 bytes: model_bytes,
204 pool_bytes: best_pool,
205 }],
206 rationale: format!(
207 "model {:.2} GB fits the highest-ranked circuit ({}) usable pool {:.2} GB → resident, no per-token transfer",
208 model_bytes as f64 / 1e9,
209 best.label,
210 best_usable as f64 / 1e9,
211 ),
212 };
213 }
214
215 let overflow = model_bytes - best_usable;
220 let gemv_n = matrix.gemv_n;
221
222 let stream_compute = segment_compute_cost(overflow, best, gemv_n);
225 let stream_transfer = segment_transfer_cost(overflow, best.upload_gbps);
226 let stream_cost = stream_compute + stream_transfer;
227
228 let mut best_inplace: Option<(&CircuitBench, u64, f64)> = None; for c in matrix.circuits.iter().skip(1) {
233 if !matches!(c.kind, CircuitKind::IntegratedGpu | CircuitKind::Cpu) {
234 continue;
235 }
236 let pool = pool_for(c.kind, topo);
237 if pool < overflow {
238 continue; }
240 let cost = segment_compute_cost(overflow, c, gemv_n); if best_inplace.map_or(true, |(_, _, prev)| cost < prev) {
242 best_inplace = Some((c, pool, cost));
243 }
244 }
245
246 if let Some((sec, sec_pool, inplace_cost)) = best_inplace {
249 if inplace_cost < stream_cost {
250 return EmploymentPlan {
251 protocol: ResidencyProtocol::HeterogeneousOverflow,
252 model_bytes,
253 kv_reserve_bytes,
254 device_priority,
255 placements: vec![
256 DevicePlacement {
257 circuit: best.label.clone(),
258 kind: best.kind,
259 role: PlacementRole::ResidentPrimary,
260 bytes: best_usable,
261 pool_bytes: best_pool,
262 },
263 DevicePlacement {
264 circuit: sec.label.clone(),
265 kind: sec.kind,
266 role: PlacementRole::Overflow,
267 bytes: overflow,
268 pool_bytes: sec_pool,
269 },
270 ],
271 rationale: format!(
272 "model {:.2} GB exceeds the fast pool ({:.2} GB usable); {:.2} GB overflow → argmin picks IN-PLACE on {} (est {:.1} ms/tok compute, no per-token transfer) over streaming to {} (est {:.1} ms compute + {:.1} ms transfer) — D31 measured decision",
273 model_bytes as f64 / 1e9,
274 best_usable as f64 / 1e9,
275 overflow as f64 / 1e9,
276 sec.label,
277 inplace_cost * 1e3,
278 best.label,
279 stream_compute * 1e3,
280 stream_transfer * 1e3,
281 ),
282 };
283 }
284 }
285
286 let rationale = match best_inplace {
289 Some((sec, _, inplace_cost)) => format!(
290 "model {:.2} GB exceeds the fast pool ({:.2} GB usable); {:.2} GB overflow → argmin picks STREAMING to {} (est {:.1} ms compute + {:.1} ms transfer) over the best in-place secondary {} (est {:.1} ms compute) — D31 measured decision",
291 model_bytes as f64 / 1e9,
292 best_usable as f64 / 1e9,
293 overflow as f64 / 1e9,
294 best.label,
295 stream_compute * 1e3,
296 stream_transfer * 1e3,
297 sec.label,
298 inplace_cost * 1e3,
299 ),
300 None => format!(
301 "model {:.2} GB exceeds the fast pool ({:.2} GB usable) with no in-place secondary big enough → double-buffer stream {:.2} GB overflow to {} (est {:.1} ms compute + {:.1} ms transfer, A4)",
302 model_bytes as f64 / 1e9,
303 best_usable as f64 / 1e9,
304 overflow as f64 / 1e9,
305 best.label,
306 stream_compute * 1e3,
307 stream_transfer * 1e3,
308 ),
309 };
310 EmploymentPlan {
311 protocol: ResidencyProtocol::Streaming,
312 model_bytes,
313 kv_reserve_bytes,
314 device_priority,
315 placements: vec![DevicePlacement {
316 circuit: best.label.clone(),
317 kind: best.kind,
318 role: PlacementRole::StreamTarget,
319 bytes: model_bytes,
320 pool_bytes: best_pool,
321 }],
322 rationale,
323 }
324}
325
326pub fn plan_for_model(model_bytes: u64) -> EmploymentPlan {
329 let topo = probe_host_topology();
330 let matrix = benchmark_devices(2048);
331 plan_employment(&topo, &matrix, model_bytes, DEFAULT_KV_RESERVE)
332}
333
334use std::sync::OnceLock;
351
352pub const ROUTE_ENV: &str = "QUALIA_LLM_ROUTE";
355
356static ROUTE_PLAN: OnceLock<std::sync::Mutex<Option<EmploymentPlan>>> = OnceLock::new();
359
360fn route_plan_slot() -> &'static std::sync::Mutex<Option<EmploymentPlan>> {
361 ROUTE_PLAN.get_or_init(|| std::sync::Mutex::new(None))
362}
363
364pub fn route_enabled() -> bool {
366 matches!(
367 std::env::var(ROUTE_ENV).ok().as_deref(),
368 Some("1") | Some("true") | Some("on") | Some("yes")
369 )
370}
371
372pub fn last_employment_plan() -> Option<EmploymentPlan> {
376 route_plan_slot().lock().ok().and_then(|g| g.clone())
377}
378
379pub fn route_employment_for_model(
387 topo: &HostTopology,
388 matrix: &CapabilityMatrix,
389 model_bytes: u64,
390) -> Option<EmploymentPlan> {
391 if !route_enabled() {
392 return None;
393 }
394 let plan = plan_employment(topo, matrix, model_bytes, DEFAULT_KV_RESERVE);
395 log::info!(
397 "llm_route|employment_plan|protocol={:?}|model={:.2}GB|priority={}|placements={}",
398 plan.protocol,
399 plan.model_bytes as f64 / 1e9,
400 plan.device_priority.join(">"),
401 plan.placements
402 .iter()
403 .map(|p| format!("{:?}:{}({:.2}GB)", p.role, p.circuit, p.bytes as f64 / 1e9))
404 .collect::<Vec<_>>()
405 .join(","),
406 );
407 log::info!("llm_route|rationale|{}", plan.rationale);
408 if let Ok(mut g) = route_plan_slot().lock() {
409 *g = Some(plan.clone());
410 }
411 Some(plan)
412}
413
414pub fn route_employment_from_passport(model_bytes: u64) -> Option<EmploymentPlan> {
419 if !route_enabled() {
420 return None;
421 }
422 let path = crate::hardware_passport::default_cache_path();
423 match crate::hardware_passport::read_passport(&path) {
424 Some(p) => route_employment_for_model(&p.topology, &p.matrix, model_bytes),
425 None => {
426 log::info!(
427 "llm_route|no_passport|skipping employment plan for {:.2}GB model (run `qualia-cli llm passport` to enable H2 routing)",
428 model_bytes as f64 / 1e9,
429 );
430 None
431 }
432 }
433}
434
435#[cfg(test)]
436mod tests {
437 use super::*;
438 use crate::device_benchmark::CircuitBench;
439 use crate::host_topology::{AdapterDesc, HostMemoryTopology};
440
441 const GB: u64 = 1024 * 1024 * 1024;
442
443 fn circuit(label: &str, kind: CircuitKind, ms: f64, score: f64) -> CircuitBench {
444 CircuitBench {
445 label: label.into(),
446 kind,
447 backend: "test".into(),
448 ms_per_gemv: ms,
449 gflops: 0.0,
450 upload_gbps: 1.0,
451 rel_score: score,
452 decode_proxy_tok_s: None,
453 }
454 }
455
456 fn discrete_topo(vram_gb: u64, ram_gb: u64, with_igpu: bool) -> HostTopology {
457 let mut adapters = vec![AdapterDesc {
458 name: "Discrete GPU".into(),
459 backend: "Dx12".into(),
460 class: AdapterClass::Discrete,
461 vendor: 0x10de,
462 device: 1,
463 dedicated_vram_bytes: vram_gb * GB,
464 }];
465 if with_igpu {
466 adapters.push(AdapterDesc {
467 name: "iGPU".into(),
468 backend: "Dx12".into(),
469 class: AdapterClass::Integrated,
470 vendor: 0x8086,
471 device: 2,
472 dedicated_vram_bytes: 0,
473 });
474 }
475 HostTopology {
476 adapters,
477 topology: HostMemoryTopology::Discrete,
478 host_ram_bytes: ram_gb * GB,
479 host_ram_available_bytes: ram_gb * GB / 2,
480 cpu_cores: 8,
481 os_floor_bytes: 3 * GB / 2,
482 usable_model_budget_bytes: vram_gb * GB,
483 }
484 }
485
486 #[test]
487 fn small_model_is_resident_on_fastest() {
488 let topo = discrete_topo(12, 64, true);
489 let matrix = CapabilityMatrix {
490 circuits: vec![
491 circuit("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0),
492 circuit("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06),
493 circuit("CPU native", CircuitKind::Cpu, 23.0, 0.02),
494 ],
495 gemv_n: 2048,
496 npu_probed: false,
497 };
498 let plan = plan_employment(&topo, &matrix, 2 * GB, DEFAULT_KV_RESERVE);
499 assert_eq!(plan.protocol, ResidencyProtocol::Resident);
500 assert_eq!(plan.placements[0].kind, CircuitKind::DiscreteGpu);
501 assert_eq!(plan.device_priority[0], "Discrete GPU");
502 }
503
504 #[test]
505 fn overflow_with_igpu_is_heterogeneous() {
506 let topo = discrete_topo(12, 64, true);
508 let matrix = CapabilityMatrix {
509 circuits: vec![
510 circuit("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0),
511 circuit("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06),
512 circuit("CPU native", CircuitKind::Cpu, 23.0, 0.02),
513 ],
514 gemv_n: 2048,
515 npu_probed: false,
516 };
517 let plan = plan_employment(&topo, &matrix, 20 * GB, DEFAULT_KV_RESERVE);
518 assert_eq!(plan.protocol, ResidencyProtocol::HeterogeneousOverflow);
519 assert_eq!(plan.placements[0].role, PlacementRole::ResidentPrimary);
520 assert_eq!(plan.placements[1].role, PlacementRole::Overflow);
521 assert_eq!(plan.placements[1].kind, CircuitKind::IntegratedGpu);
522 }
523
524 #[test]
525 fn overflow_without_igpu_streams() {
526 let topo = discrete_topo(12, 64, false);
528 let matrix = CapabilityMatrix {
529 circuits: vec![
530 circuit("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0),
531 circuit("CPU native", CircuitKind::Cpu, 23.0, 0.02),
532 ],
533 gemv_n: 2048,
534 npu_probed: false,
535 };
536 let plan = plan_employment(&topo, &matrix, 20 * GB, DEFAULT_KV_RESERVE);
539 assert!(
540 matches!(
541 plan.protocol,
542 ResidencyProtocol::HeterogeneousOverflow | ResidencyProtocol::Streaming
543 ),
544 "must be an overflow strategy, got {:?}",
545 plan.protocol
546 );
547 }
548
549 #[test]
550 fn unified_host_is_resident_on_igpu() {
551 let topo = HostTopology {
553 adapters: vec![AdapterDesc {
554 name: "Apple/Intel iGPU".into(),
555 backend: "Metal".into(),
556 class: AdapterClass::Integrated,
557 vendor: 0x106b,
558 device: 1,
559 dedicated_vram_bytes: 0,
560 }],
561 topology: HostMemoryTopology::Unified,
562 host_ram_bytes: 32 * GB,
563 host_ram_available_bytes: 20 * GB,
564 cpu_cores: 8,
565 os_floor_bytes: 6 * GB,
566 usable_model_budget_bytes: 26 * GB,
567 };
568 let matrix = CapabilityMatrix {
569 circuits: vec![
570 circuit("Apple/Intel iGPU", CircuitKind::IntegratedGpu, 1.0, 1.0),
571 circuit("CPU native", CircuitKind::Cpu, 10.0, 0.1),
572 ],
573 gemv_n: 2048,
574 npu_probed: false,
575 };
576 let plan = plan_employment(&topo, &matrix, 8 * GB, DEFAULT_KV_RESERVE);
577 assert_eq!(plan.protocol, ResidencyProtocol::Resident);
578 assert_eq!(plan.placements[0].kind, CircuitKind::IntegratedGpu);
579 }
580
581 fn circuit_up(
583 label: &str,
584 kind: CircuitKind,
585 ms: f64,
586 score: f64,
587 upload_gbps: f64,
588 ) -> CircuitBench {
589 CircuitBench {
590 upload_gbps,
591 ..circuit(label, kind, ms, score)
592 }
593 }
594
595 #[test]
601 fn overflow_inplace_secondary_wins_argmin() {
602 let topo = discrete_topo(12, 64, true);
603 let matrix = CapabilityMatrix {
604 circuits: vec![
605 circuit_up("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0, 2.0),
607 circuit_up("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06, 50.0),
610 circuit_up("CPU native", CircuitKind::Cpu, 23.0, 0.02, f64::INFINITY),
611 ],
612 gemv_n: 2048,
613 npu_probed: false,
614 };
615 let plan = plan_employment(&topo, &matrix, 20 * GB, DEFAULT_KV_RESERVE);
616 assert_eq!(plan.protocol, ResidencyProtocol::HeterogeneousOverflow);
617 assert_eq!(plan.placements[1].role, PlacementRole::Overflow);
618 assert_eq!(plan.placements[1].kind, CircuitKind::IntegratedGpu);
619 }
620
621 #[test]
626 fn overflow_fast_bus_streams() {
627 let topo = discrete_topo(12, 64, true);
628 let matrix = CapabilityMatrix {
629 circuits: vec![
630 circuit_up("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0, 64.0),
631 circuit_up("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06, 50.0),
632 circuit_up("CPU native", CircuitKind::Cpu, 23.0, 0.02, f64::INFINITY),
633 ],
634 gemv_n: 2048,
635 npu_probed: false,
636 };
637 let plan = plan_employment(&topo, &matrix, 20 * GB, DEFAULT_KV_RESERVE);
638 assert_eq!(plan.protocol, ResidencyProtocol::Streaming);
639 assert_eq!(plan.placements[0].role, PlacementRole::StreamTarget);
640 assert_eq!(plan.placements[0].kind, CircuitKind::DiscreteGpu);
641 }
642
643 #[test]
647 fn overflow_flips_on_upload_gbps_only() {
648 let topo = discrete_topo(12, 64, true);
649 let matrix_with_dgpu_upload = |dgpu_up: f64| CapabilityMatrix {
652 circuits: vec![
653 circuit_up("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0, dgpu_up),
654 circuit_up("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06, 5.0),
655 circuit_up("CPU native", CircuitKind::Cpu, 23.0, 0.02, f64::INFINITY),
656 ],
657 gemv_n: 2048,
658 npu_probed: false,
659 };
660
661 let slow = plan_employment(
663 &topo,
664 &matrix_with_dgpu_upload(1.0),
665 20 * GB,
666 DEFAULT_KV_RESERVE,
667 );
668 assert_eq!(
669 slow.protocol,
670 ResidencyProtocol::HeterogeneousOverflow,
671 "slow primary bus must keep overflow in-place: {}",
672 slow.rationale
673 );
674
675 let fast = plan_employment(
677 &topo,
678 &matrix_with_dgpu_upload(64.0),
679 20 * GB,
680 DEFAULT_KV_RESERVE,
681 );
682 assert_eq!(
683 fast.protocol,
684 ResidencyProtocol::Streaming,
685 "fast primary bus must flip the decision to streaming: {}",
686 fast.rationale
687 );
688 }
689
690 fn synthetic_matrix() -> CapabilityMatrix {
691 CapabilityMatrix {
692 circuits: vec![
693 circuit("Discrete GPU", CircuitKind::DiscreteGpu, 0.4, 1.0),
694 circuit("iGPU", CircuitKind::IntegratedGpu, 7.0, 0.06),
695 circuit("CPU native", CircuitKind::Cpu, 23.0, 0.02),
696 ],
697 gemv_n: 2048,
698 npu_probed: false,
699 }
700 }
701
702 #[test]
706 fn route_flag_gates_compute_store_and_retrieve() {
707 std::env::remove_var(ROUTE_ENV);
709 assert!(!route_enabled(), "flag must default off");
710
711 let topo = discrete_topo(12, 64, true);
712 let matrix = synthetic_matrix();
713
714 let off = route_employment_for_model(&topo, &matrix, 20 * GB);
716 assert!(off.is_none(), "flag off must compute nothing");
717 assert!(
718 last_employment_plan().is_none(),
719 "flag off must store nothing"
720 );
721
722 std::env::set_var(ROUTE_ENV, "1");
724 assert!(route_enabled());
725 let returned = route_employment_for_model(&topo, &matrix, 20 * GB)
726 .expect("flag on must return a plan");
727 assert_eq!(returned.protocol, ResidencyProtocol::HeterogeneousOverflow);
728 assert_eq!(returned.model_bytes, 20 * GB);
729
730 let stored = last_employment_plan().expect("plan must be retrievable after route");
731 assert_eq!(stored.protocol, returned.protocol);
732 assert_eq!(stored.model_bytes, returned.model_bytes);
733 assert_eq!(stored.device_priority, returned.device_priority);
734 assert_eq!(stored.placements.len(), returned.placements.len());
735
736 std::env::remove_var(ROUTE_ENV);
737 }
738}