qualia_core_db/platform/hetero_dispatch.rs
1//! Vendor-neutral heterogeneous compute dispatch + storage / precision policy.
2//!
3//! This module **replaces the former CUDA/cuFile GPUDirect-Storage bridge**
4//! (`cuda_bridge.rs`, removed). That bridge was the engine's one vendor-locked
5//! appendage — NVIDIA-only, Linux-only, and unverifiable without specific hardware.
6//! Removing it dissolves the hardware boundary entirely. The four capabilities it
7//! was meant to provide are delivered here on the engine's GENERAL stack — portable
8//! `wgpu` ([`super::gpu`]) + memory-mapped I/O ([`super::host`]) — so they build,
9//! run, and verify on any backend (Vulkan / DX12 / Metal / WebGPU), no vendor SDK:
10//!
11//! 1. **Transfer policy** ([`ZeroCopyStrategy`]) — portable GPU buffers use staging.
12//! Shared physical RAM alone does not establish a backend host-memory import.
13//! 2. **Hardware-agnostic fallback dispatcher** ([`HeterogeneousDispatcher`]) —
14//! route a job to the GPU (`wgpu`) when it fits, else CPU; and tile a
15//! matmul across passes when VRAM is exhausted instead of hard-failing.
16//! 3. **Kernel stream fusion** ([`plan_fusion`]) — group consecutive same-shape
17//! element-wise tensor ops into a single `wgpu` compute pass (the portable
18//! analogue of CUDA stream fusion; the engine's fused shaders already do this at
19//! the shader level). Fewer passes ⇒ fewer dispatch / PCIe round-trips.
20//! 4. **Mixed-precision policy** ([`select_precision`]) — pick f32/f16/q8/q4 from
21//! the host's VRAM / power / thermal budget.
22//!
23//! ## The one capability deliberately NOT ported
24//! NVIDIA GPUDirect-Storage's *true* NVMe→VRAM DMA (bypassing system RAM) has **no
25//! portable `wgpu` equivalent**. The vendor-neutral substitute (and the engine's
26//! actual path) is `mmap` + OS page cache + a one-time staging upload — zero-heap,
27//! standard OS mechanics, identical across an A2000 / Apple M-series / generic Linux
28//! box. GDS-class throughput only matters when streaming a 70B model off an NVMe
29//! array into an 80 GB datacenter GPU — the deployment the affordability rail
30//! explicitly does not target, so nothing on the critical path is lost.
31//!
32//! ## Future, optional Vulkan zero-copy fast-path (documented, NOT built)
33//! If a specific deployment ever justifies skipping the staging copy, the
34//! vendor-neutral way is a `wgpu-hal` Vulkan fast-path that imports the mmap'd
35//! weights as device memory via `VK_EXT_external_memory_host` (broadly supported,
36//! cross-vendor) + Resizable-BAR — lit up ONLY on the Vulkan backend, behind a
37//! `vulkan_zero_copy_import` feature, additive over the portable path. It is
38//! deliberately unbuilt: it needs `unsafe` wgpu-hal and is backend-specific
39//! (DX12/Metal have their own external-memory mechanisms), which would re-introduce
40//! exactly the per-backend coupling that removing CUDA just eliminated. Build it
41//! only when a real deployment needs it; never as the core storage dependency.
42//!
43//! All routines here are pure-scalar policy / planning logic — **zero heap**, no
44//! recursion, run anywhere.
45#![cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
46
47// ── 1. Unified-memory zero-copy strategy ───────────────────────────────────────
48
49/// How weight data reaches the GPU for a given device.
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51pub enum ZeroCopyStrategy {
52 /// CPU in-pool data, or host memory explicitly imported by a backend.
53 MmapDirect,
54 /// Upload through the portable GPU buffer path, including integrated GPUs.
55 StagingUpload,
56}
57
58impl ZeroCopyStrategy {
59 /// Topology alone cannot establish an imported host mapping.
60 pub fn for_device(_is_unified_memory: bool) -> Self {
61 Self::StagingUpload
62 }
63
64 /// Portable GPU allocations require uploads regardless of adapter class.
65 pub fn for_wgpu_device_type(_device_type: etalon::DeviceType) -> Self {
66 Self::StagingUpload
67 }
68
69 /// Whether a host→device copy is required (false on unified memory).
70 pub fn requires_host_copy(self) -> bool {
71 matches!(self, Self::StagingUpload)
72 }
73}
74
75// ── 2. Hardware-agnostic fallback dispatcher ────────────────────────────────────
76
77/// The compute backend a job is routed to.
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub enum ComputeBackend {
80 /// Portable `wgpu` GPU path ([`super::gpu::WebGpuIntegrator`]).
81 Gpu,
82 /// Reserved for a production executor verified for the requested workload.
83 Npu,
84 /// CPU fallback — always available, never hard-fails.
85 Cpu,
86}
87
88/// What the host can offer the dispatcher.
89#[derive(Debug, Clone, Copy, PartialEq, Eq)]
90pub struct HostCapabilities {
91 pub gpu_available: bool,
92 /// Usable VRAM in bytes (e.g. `wgpu` `limits().max_buffer_size`).
93 pub vram_available: u64,
94 /// Discovery only; this flag does not establish a verified NPU executor.
95 pub npu_available: bool,
96 pub cpu_threads: u32,
97}
98
99/// Routes compute jobs across GPU / NPU / CPU and degrades gracefully under VRAM
100/// pressure instead of hard-failing (the behaviour the CUDA bridge lacked).
101#[derive(Debug, Clone, Copy)]
102pub struct HeterogeneousDispatcher {
103 caps: HostCapabilities,
104}
105
106impl HeterogeneousDispatcher {
107 /// Creates a default `HeterogeneousDispatcher`.
108 pub fn new(caps: HostCapabilities) -> Self {
109 Self { caps }
110 }
111
112 /// Choose a backend for a job needing `vram_required` bytes: GPU if present and
113 /// it fits → else CPU. NPU discovery alone cannot select an execution path. When the GPU is
114 /// present but the job is larger than VRAM, the caller should GPU-tile (see
115 /// [`Self::gpu_tiles`]) rather than fall straight to CPU.
116 pub fn select_backend(&self, vram_required: u64) -> ComputeBackend {
117 if self.caps.gpu_available {
118 if vram_required <= self.caps.vram_available || self.caps.vram_available > 0 {
119 // GPU present: it fits, or it can be tiled to fit (see gpu_tiles).
120 return ComputeBackend::Gpu;
121 }
122 }
123 ComputeBackend::Cpu
124 }
125
126 /// How many sequential tiles a `total_bytes` GPU job must be split into so each
127 /// tile fits in available VRAM — graceful degradation instead of an OOM hard
128 /// fail. Returns 1 when it already fits (or when there's no GPU/VRAM to tile
129 /// into, in which case the job runs on the CPU as a single pass).
130 pub fn gpu_tiles(&self, total_bytes: u64) -> u32 {
131 if !self.caps.gpu_available || self.caps.vram_available == 0 {
132 return 1;
133 }
134 // ceil(total / vram), clamped to ≥ 1.
135 let tiles = total_bytes.div_ceil(self.caps.vram_available);
136 tiles.max(1).min(u32::MAX as u64) as u32
137 }
138
139 /// Dispatch capabilities.
140 pub fn capabilities(&self) -> HostCapabilities {
141 self.caps
142 }
143}
144
145// ── 3. Kernel / stream fusion planning ──────────────────────────────────────────
146
147/// The fusability class of a tensor op in a dispatch sequence.
148#[derive(Debug, Clone, Copy, PartialEq, Eq)]
149pub enum TensorOpKind {
150 /// Element-wise (add, mul, activation…): fusable with adjacent same-shape
151 /// element-wise ops into one compute pass.
152 Elementwise,
153 /// A fusion barrier (reduction, matmul, reshape): forces a new pass.
154 Barrier,
155}
156
157/// One op in a planned dispatch sequence. `shape` is a shape token; element-wise ops
158/// only fuse with neighbours of the same shape.
159#[derive(Debug, Clone, Copy, PartialEq, Eq)]
160pub struct TensorOp {
161 pub kind: TensorOpKind,
162 pub shape: u64,
163}
164
165/// Plan stream fusion: the number of `wgpu` compute passes a sequence needs after
166/// fusing each maximal run of same-shape element-wise ops into a single pass.
167/// Barriers each take their own pass. Result is in `1..=ops.len()`; fewer passes ⇒
168/// fewer dispatch / PCIe round-trips. Zero-heap (single linear scan).
169pub fn plan_fusion(ops: &[TensorOp]) -> u32 {
170 let mut passes = 0u32;
171 let mut i = 0usize;
172 while i < ops.len() {
173 passes += 1;
174 if ops[i].kind == TensorOpKind::Barrier {
175 i += 1;
176 continue;
177 }
178 // Absorb the maximal run of same-shape element-wise ops into this pass.
179 let shape = ops[i].shape;
180 i += 1;
181 while i < ops.len() && ops[i].kind == TensorOpKind::Elementwise && ops[i].shape == shape {
182 i += 1;
183 }
184 }
185 passes
186}
187
188// ── 4. Mixed-precision policy ────────────────────────────────────────────────────
189
190/// Numeric precision for weights / activations.
191#[derive(Debug, Clone, Copy, PartialEq, Eq)]
192pub enum Precision {
193 F32,
194 F16,
195 Q8,
196 Q4,
197}
198
199impl Precision {
200 /// Bytes per weight element.
201 pub fn bytes_per_weight(self) -> f64 {
202 match self {
203 Self::F32 => 4.0,
204 Self::F16 => 2.0,
205 Self::Q8 => 1.0,
206 Self::Q4 => 0.5,
207 }
208 }
209}
210
211/// The host's power / thermal / memory envelope for the precision decision.
212#[derive(Debug, Clone, Copy, PartialEq)]
213pub struct PowerThermalBudget {
214 pub vram_budget_bytes: u64,
215 /// Available power headroom, milliwatts.
216 pub power_budget_mw: u32,
217 /// Degrees C below the thermal-throttle point.
218 pub thermal_headroom_c: f32,
219}
220
221/// Pick the precision tuned to the host's thermodynamic / power / memory constraints:
222/// choose the coarsest precision that still fits `param_count` weights in the VRAM
223/// budget, and — when power or thermal headroom is tight — never run the heavy
224/// precisions (f32/f16 draw more power and generate more heat). Returns `Q4` if even
225/// q4 overflows VRAM (the caller must then tile or offload).
226pub fn select_precision(param_count: u64, budget: &PowerThermalBudget) -> Precision {
227 let fits = |p: Precision| {
228 (param_count as f64 * p.bytes_per_weight()) as u64 <= budget.vram_budget_bytes
229 };
230 // Tight envelope ⇒ cap the *maximum* precision we'll consider (lower precision =
231 // fewer FLOPs = less heat/draw). Thresholds: < 5°C headroom or < 15 W budget.
232 let throttling = budget.thermal_headroom_c < 5.0 || budget.power_budget_mw < 15_000;
233 let candidates: &[Precision] = if throttling {
234 &[Precision::Q8, Precision::Q4]
235 } else {
236 &[Precision::F32, Precision::F16, Precision::Q8, Precision::Q4]
237 };
238 for &p in candidates {
239 if fits(p) {
240 return p;
241 }
242 }
243 Precision::Q4
244}
245
246#[cfg(test)]
247mod tests {
248 use super::*;
249
250 #[test]
251 fn gpu_topology_does_not_establish_host_memory_import() {
252 for unified in [false, true] {
253 let strategy = ZeroCopyStrategy::for_device(unified);
254 assert_eq!(strategy, ZeroCopyStrategy::StagingUpload);
255 assert!(strategy.requires_host_copy());
256 }
257 for device_type in [
258 etalon::DeviceType::IntegratedGpu,
259 etalon::DeviceType::DiscreteGpu,
260 etalon::DeviceType::Cpu,
261 etalon::DeviceType::VirtualGpu,
262 etalon::DeviceType::Other,
263 ] {
264 assert_eq!(
265 ZeroCopyStrategy::for_wgpu_device_type(device_type),
266 ZeroCopyStrategy::StagingUpload
267 );
268 }
269 assert!(!ZeroCopyStrategy::MmapDirect.requires_host_copy());
270 }
271
272 #[test]
273 fn dispatcher_routes_and_falls_back() {
274 // GPU present and job fits → GPU.
275 let d = HeterogeneousDispatcher::new(HostCapabilities {
276 gpu_available: true,
277 vram_available: 8 << 30,
278 npu_available: true,
279 cpu_threads: 16,
280 });
281 assert_eq!(d.select_backend(1 << 30), ComputeBackend::Gpu);
282
283 // Detection alone does not establish a production NPU executor.
284 let d2 = HeterogeneousDispatcher::new(HostCapabilities {
285 gpu_available: false,
286 vram_available: 0,
287 npu_available: true,
288 cpu_threads: 8,
289 });
290 assert_eq!(d2.select_backend(1 << 30), ComputeBackend::Cpu);
291
292 // No GPU, no NPU → CPU (always works, never hard-fails).
293 let d3 = HeterogeneousDispatcher::new(HostCapabilities {
294 gpu_available: false,
295 vram_available: 0,
296 npu_available: false,
297 cpu_threads: 4,
298 });
299 assert_eq!(d3.select_backend(1 << 30), ComputeBackend::Cpu);
300 }
301
302 #[test]
303 fn vram_exhaustion_tiles_instead_of_failing() {
304 let d = HeterogeneousDispatcher::new(HostCapabilities {
305 gpu_available: true,
306 vram_available: 2 << 30, // 2 GiB
307 npu_available: false,
308 cpu_threads: 8,
309 });
310 assert_eq!(d.gpu_tiles(1 << 30), 1, "fits in one tile");
311 assert_eq!(d.gpu_tiles(2 << 30), 1, "exactly fits");
312 assert_eq!(
313 d.gpu_tiles(5 << 30),
314 3,
315 "5 GiB / 2 GiB → 3 tiles, no OOM hard-fail"
316 );
317 // No GPU → single CPU pass.
318 let cpu = HeterogeneousDispatcher::new(HostCapabilities {
319 gpu_available: false,
320 vram_available: 0,
321 npu_available: false,
322 cpu_threads: 4,
323 });
324 assert_eq!(cpu.gpu_tiles(99 << 30), 1);
325 }
326
327 #[test]
328 fn fusion_collapses_elementwise_runs() {
329 let ew = |s| TensorOp {
330 kind: TensorOpKind::Elementwise,
331 shape: s,
332 };
333 let barrier = |s| TensorOp {
334 kind: TensorOpKind::Barrier,
335 shape: s,
336 };
337
338 // Three same-shape element-wise ops fuse into one pass.
339 assert_eq!(plan_fusion(&[ew(1), ew(1), ew(1)]), 1);
340 // A barrier (matmul/reduction) splits the run: EW | barrier | EW = 3 passes.
341 assert_eq!(plan_fusion(&[ew(1), barrier(1), ew(1)]), 3);
342 // Different shapes don't fuse.
343 assert_eq!(plan_fusion(&[ew(1), ew(2)]), 2);
344 // Mixed run: (EW EW) | barrier | (EW EW EW) = 3 passes.
345 assert_eq!(
346 plan_fusion(&[ew(1), ew(1), barrier(9), ew(2), ew(2), ew(2)]),
347 3
348 );
349 assert_eq!(plan_fusion(&[]), 0);
350 }
351
352 #[test]
353 fn precision_fits_budget_and_respects_thermals() {
354 let gib = 1u64 << 30;
355 // 1B params, roomy VRAM + headroom → f32.
356 let roomy = PowerThermalBudget {
357 vram_budget_bytes: 8 * gib,
358 power_budget_mw: 60_000,
359 thermal_headroom_c: 30.0,
360 };
361 assert_eq!(select_precision(1_000_000_000, &roomy), Precision::F32);
362
363 // 1B params, only ~1.5 GiB VRAM → must drop to q8 (1 GB) — f32/f16 overflow.
364 let tight_vram = PowerThermalBudget {
365 vram_budget_bytes: gib + gib / 2,
366 power_budget_mw: 60_000,
367 thermal_headroom_c: 30.0,
368 };
369 assert_eq!(select_precision(1_000_000_000, &tight_vram), Precision::Q8);
370
371 // Throttling (low thermal headroom) → never f32/f16 even with VRAM to spare.
372 let throttling = PowerThermalBudget {
373 vram_budget_bytes: 64 * gib,
374 power_budget_mw: 60_000,
375 thermal_headroom_c: 2.0,
376 };
377 assert_eq!(select_precision(1_000_000_000, &throttling), Precision::Q8);
378
379 // Model too big for any precision → Q4 (caller tiles/offloads).
380 let huge = PowerThermalBudget {
381 vram_budget_bytes: gib,
382 power_budget_mw: 60_000,
383 thermal_headroom_c: 30.0,
384 };
385 assert_eq!(select_precision(10_000_000_000, &huge), Precision::Q4);
386 }
387}