Skip to main content

qualia_core_db/platform/
hetero_dispatch.rs

1//! Vendor-neutral heterogeneous compute dispatch + storage / precision policy.
2//!
3//! This module **replaces the former CUDA/cuFile GPUDirect-Storage bridge**
4//! (`cuda_bridge.rs`, removed). That bridge was the engine's one vendor-locked
5//! appendage — NVIDIA-only, Linux-only, and unverifiable without specific hardware.
6//! Removing it dissolves the hardware boundary entirely. The four capabilities it
7//! was meant to provide are delivered here on the engine's GENERAL stack — portable
8//! `wgpu` ([`super::gpu`]) + memory-mapped I/O ([`super::host`]) — so they build,
9//! run, and verify on any backend (Vulkan / DX12 / Metal / WebGPU), no vendor SDK:
10//!
11//! 1. **Transfer policy** ([`ZeroCopyStrategy`]) — portable GPU buffers use staging.
12//!    Shared physical RAM alone does not establish a backend host-memory import.
13//! 2. **Hardware-agnostic fallback dispatcher** ([`HeterogeneousDispatcher`]) —
14//!    route a job to the GPU (`wgpu`) when it fits, else CPU; and tile a
15//!    matmul across passes when VRAM is exhausted instead of hard-failing.
16//! 3. **Kernel stream fusion** ([`plan_fusion`]) — group consecutive same-shape
17//!    element-wise tensor ops into a single `wgpu` compute pass (the portable
18//!    analogue of CUDA stream fusion; the engine's fused shaders already do this at
19//!    the shader level). Fewer passes ⇒ fewer dispatch / PCIe round-trips.
20//! 4. **Mixed-precision policy** ([`select_precision`]) — pick f32/f16/q8/q4 from
21//!    the host's VRAM / power / thermal budget.
22//!
23//! ## The one capability deliberately NOT ported
24//! NVIDIA GPUDirect-Storage's *true* NVMe→VRAM DMA (bypassing system RAM) has **no
25//! portable `wgpu` equivalent**. The vendor-neutral substitute (and the engine's
26//! actual path) is `mmap` + OS page cache + a one-time staging upload — zero-heap,
27//! standard OS mechanics, identical across an A2000 / Apple M-series / generic Linux
28//! box. GDS-class throughput only matters when streaming a 70B model off an NVMe
29//! array into an 80 GB datacenter GPU — the deployment the affordability rail
30//! explicitly does not target, so nothing on the critical path is lost.
31//!
32//! ## Future, optional Vulkan zero-copy fast-path (documented, NOT built)
33//! If a specific deployment ever justifies skipping the staging copy, the
34//! vendor-neutral way is a `wgpu-hal` Vulkan fast-path that imports the mmap'd
35//! weights as device memory via `VK_EXT_external_memory_host` (broadly supported,
36//! cross-vendor) + Resizable-BAR — lit up ONLY on the Vulkan backend, behind a
37//! `vulkan_zero_copy_import` feature, additive over the portable path. It is
38//! deliberately unbuilt: it needs `unsafe` wgpu-hal and is backend-specific
39//! (DX12/Metal have their own external-memory mechanisms), which would re-introduce
40//! exactly the per-backend coupling that removing CUDA just eliminated. Build it
41//! only when a real deployment needs it; never as the core storage dependency.
42//!
43//! All routines here are pure-scalar policy / planning logic — **zero heap**, no
44//! recursion, run anywhere.
45#![cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
46
47// ── 1. Unified-memory zero-copy strategy ───────────────────────────────────────
48
49/// How weight data reaches the GPU for a given device.
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51pub enum ZeroCopyStrategy {
52    /// CPU in-pool data, or host memory explicitly imported by a backend.
53    MmapDirect,
54    /// Upload through the portable GPU buffer path, including integrated GPUs.
55    StagingUpload,
56}
57
58impl ZeroCopyStrategy {
59    /// Topology alone cannot establish an imported host mapping.
60    pub fn for_device(_is_unified_memory: bool) -> Self {
61        Self::StagingUpload
62    }
63
64    /// Portable GPU allocations require uploads regardless of adapter class.
65    pub fn for_wgpu_device_type(_device_type: etalon::DeviceType) -> Self {
66        Self::StagingUpload
67    }
68
69    /// Whether a host→device copy is required (false on unified memory).
70    pub fn requires_host_copy(self) -> bool {
71        matches!(self, Self::StagingUpload)
72    }
73}
74
75// ── 2. Hardware-agnostic fallback dispatcher ────────────────────────────────────
76
77/// The compute backend a job is routed to.
78#[derive(Debug, Clone, Copy, PartialEq, Eq)]
79pub enum ComputeBackend {
80    /// Portable `wgpu` GPU path ([`super::gpu::WebGpuIntegrator`]).
81    Gpu,
82    /// Reserved for a production executor verified for the requested workload.
83    Npu,
84    /// CPU fallback — always available, never hard-fails.
85    Cpu,
86}
87
88/// What the host can offer the dispatcher.
89#[derive(Debug, Clone, Copy, PartialEq, Eq)]
90pub struct HostCapabilities {
91    pub gpu_available: bool,
92    /// Usable VRAM in bytes (e.g. `wgpu` `limits().max_buffer_size`).
93    pub vram_available: u64,
94    /// Discovery only; this flag does not establish a verified NPU executor.
95    pub npu_available: bool,
96    pub cpu_threads: u32,
97}
98
99/// Routes compute jobs across GPU / NPU / CPU and degrades gracefully under VRAM
100/// pressure instead of hard-failing (the behaviour the CUDA bridge lacked).
101#[derive(Debug, Clone, Copy)]
102pub struct HeterogeneousDispatcher {
103    caps: HostCapabilities,
104}
105
106impl HeterogeneousDispatcher {
107    /// Creates a default `HeterogeneousDispatcher`.
108    pub fn new(caps: HostCapabilities) -> Self {
109        Self { caps }
110    }
111
112    /// Choose a backend for a job needing `vram_required` bytes: GPU if present and
113    /// it fits → else CPU. NPU discovery alone cannot select an execution path. When the GPU is
114    /// present but the job is larger than VRAM, the caller should GPU-tile (see
115    /// [`Self::gpu_tiles`]) rather than fall straight to CPU.
116    pub fn select_backend(&self, vram_required: u64) -> ComputeBackend {
117        if self.caps.gpu_available {
118            if vram_required <= self.caps.vram_available || self.caps.vram_available > 0 {
119                // GPU present: it fits, or it can be tiled to fit (see gpu_tiles).
120                return ComputeBackend::Gpu;
121            }
122        }
123        ComputeBackend::Cpu
124    }
125
126    /// How many sequential tiles a `total_bytes` GPU job must be split into so each
127    /// tile fits in available VRAM — graceful degradation instead of an OOM hard
128    /// fail. Returns 1 when it already fits (or when there's no GPU/VRAM to tile
129    /// into, in which case the job runs on the CPU as a single pass).
130    pub fn gpu_tiles(&self, total_bytes: u64) -> u32 {
131        if !self.caps.gpu_available || self.caps.vram_available == 0 {
132            return 1;
133        }
134        // ceil(total / vram), clamped to ≥ 1.
135        let tiles = total_bytes.div_ceil(self.caps.vram_available);
136        tiles.max(1).min(u32::MAX as u64) as u32
137    }
138
139    /// Dispatch capabilities.
140    pub fn capabilities(&self) -> HostCapabilities {
141        self.caps
142    }
143}
144
145// ── 3. Kernel / stream fusion planning ──────────────────────────────────────────
146
147/// The fusability class of a tensor op in a dispatch sequence.
148#[derive(Debug, Clone, Copy, PartialEq, Eq)]
149pub enum TensorOpKind {
150    /// Element-wise (add, mul, activation…): fusable with adjacent same-shape
151    /// element-wise ops into one compute pass.
152    Elementwise,
153    /// A fusion barrier (reduction, matmul, reshape): forces a new pass.
154    Barrier,
155}
156
157/// One op in a planned dispatch sequence. `shape` is a shape token; element-wise ops
158/// only fuse with neighbours of the same shape.
159#[derive(Debug, Clone, Copy, PartialEq, Eq)]
160pub struct TensorOp {
161    pub kind: TensorOpKind,
162    pub shape: u64,
163}
164
165/// Plan stream fusion: the number of `wgpu` compute passes a sequence needs after
166/// fusing each maximal run of same-shape element-wise ops into a single pass.
167/// Barriers each take their own pass. Result is in `1..=ops.len()`; fewer passes ⇒
168/// fewer dispatch / PCIe round-trips. Zero-heap (single linear scan).
169pub fn plan_fusion(ops: &[TensorOp]) -> u32 {
170    let mut passes = 0u32;
171    let mut i = 0usize;
172    while i < ops.len() {
173        passes += 1;
174        if ops[i].kind == TensorOpKind::Barrier {
175            i += 1;
176            continue;
177        }
178        // Absorb the maximal run of same-shape element-wise ops into this pass.
179        let shape = ops[i].shape;
180        i += 1;
181        while i < ops.len() && ops[i].kind == TensorOpKind::Elementwise && ops[i].shape == shape {
182            i += 1;
183        }
184    }
185    passes
186}
187
188// ── 4. Mixed-precision policy ────────────────────────────────────────────────────
189
190/// Numeric precision for weights / activations.
191#[derive(Debug, Clone, Copy, PartialEq, Eq)]
192pub enum Precision {
193    F32,
194    F16,
195    Q8,
196    Q4,
197}
198
199impl Precision {
200    /// Bytes per weight element.
201    pub fn bytes_per_weight(self) -> f64 {
202        match self {
203            Self::F32 => 4.0,
204            Self::F16 => 2.0,
205            Self::Q8 => 1.0,
206            Self::Q4 => 0.5,
207        }
208    }
209}
210
211/// The host's power / thermal / memory envelope for the precision decision.
212#[derive(Debug, Clone, Copy, PartialEq)]
213pub struct PowerThermalBudget {
214    pub vram_budget_bytes: u64,
215    /// Available power headroom, milliwatts.
216    pub power_budget_mw: u32,
217    /// Degrees C below the thermal-throttle point.
218    pub thermal_headroom_c: f32,
219}
220
221/// Pick the precision tuned to the host's thermodynamic / power / memory constraints:
222/// choose the coarsest precision that still fits `param_count` weights in the VRAM
223/// budget, and — when power or thermal headroom is tight — never run the heavy
224/// precisions (f32/f16 draw more power and generate more heat). Returns `Q4` if even
225/// q4 overflows VRAM (the caller must then tile or offload).
226pub fn select_precision(param_count: u64, budget: &PowerThermalBudget) -> Precision {
227    let fits = |p: Precision| {
228        (param_count as f64 * p.bytes_per_weight()) as u64 <= budget.vram_budget_bytes
229    };
230    // Tight envelope ⇒ cap the *maximum* precision we'll consider (lower precision =
231    // fewer FLOPs = less heat/draw). Thresholds: < 5°C headroom or < 15 W budget.
232    let throttling = budget.thermal_headroom_c < 5.0 || budget.power_budget_mw < 15_000;
233    let candidates: &[Precision] = if throttling {
234        &[Precision::Q8, Precision::Q4]
235    } else {
236        &[Precision::F32, Precision::F16, Precision::Q8, Precision::Q4]
237    };
238    for &p in candidates {
239        if fits(p) {
240            return p;
241        }
242    }
243    Precision::Q4
244}
245
246#[cfg(test)]
247mod tests {
248    use super::*;
249
250    #[test]
251    fn gpu_topology_does_not_establish_host_memory_import() {
252        for unified in [false, true] {
253            let strategy = ZeroCopyStrategy::for_device(unified);
254            assert_eq!(strategy, ZeroCopyStrategy::StagingUpload);
255            assert!(strategy.requires_host_copy());
256        }
257        for device_type in [
258            etalon::DeviceType::IntegratedGpu,
259            etalon::DeviceType::DiscreteGpu,
260            etalon::DeviceType::Cpu,
261            etalon::DeviceType::VirtualGpu,
262            etalon::DeviceType::Other,
263        ] {
264            assert_eq!(
265                ZeroCopyStrategy::for_wgpu_device_type(device_type),
266                ZeroCopyStrategy::StagingUpload
267            );
268        }
269        assert!(!ZeroCopyStrategy::MmapDirect.requires_host_copy());
270    }
271
272    #[test]
273    fn dispatcher_routes_and_falls_back() {
274        // GPU present and job fits → GPU.
275        let d = HeterogeneousDispatcher::new(HostCapabilities {
276            gpu_available: true,
277            vram_available: 8 << 30,
278            npu_available: true,
279            cpu_threads: 16,
280        });
281        assert_eq!(d.select_backend(1 << 30), ComputeBackend::Gpu);
282
283        // Detection alone does not establish a production NPU executor.
284        let d2 = HeterogeneousDispatcher::new(HostCapabilities {
285            gpu_available: false,
286            vram_available: 0,
287            npu_available: true,
288            cpu_threads: 8,
289        });
290        assert_eq!(d2.select_backend(1 << 30), ComputeBackend::Cpu);
291
292        // No GPU, no NPU → CPU (always works, never hard-fails).
293        let d3 = HeterogeneousDispatcher::new(HostCapabilities {
294            gpu_available: false,
295            vram_available: 0,
296            npu_available: false,
297            cpu_threads: 4,
298        });
299        assert_eq!(d3.select_backend(1 << 30), ComputeBackend::Cpu);
300    }
301
302    #[test]
303    fn vram_exhaustion_tiles_instead_of_failing() {
304        let d = HeterogeneousDispatcher::new(HostCapabilities {
305            gpu_available: true,
306            vram_available: 2 << 30, // 2 GiB
307            npu_available: false,
308            cpu_threads: 8,
309        });
310        assert_eq!(d.gpu_tiles(1 << 30), 1, "fits in one tile");
311        assert_eq!(d.gpu_tiles(2 << 30), 1, "exactly fits");
312        assert_eq!(
313            d.gpu_tiles(5 << 30),
314            3,
315            "5 GiB / 2 GiB → 3 tiles, no OOM hard-fail"
316        );
317        // No GPU → single CPU pass.
318        let cpu = HeterogeneousDispatcher::new(HostCapabilities {
319            gpu_available: false,
320            vram_available: 0,
321            npu_available: false,
322            cpu_threads: 4,
323        });
324        assert_eq!(cpu.gpu_tiles(99 << 30), 1);
325    }
326
327    #[test]
328    fn fusion_collapses_elementwise_runs() {
329        let ew = |s| TensorOp {
330            kind: TensorOpKind::Elementwise,
331            shape: s,
332        };
333        let barrier = |s| TensorOp {
334            kind: TensorOpKind::Barrier,
335            shape: s,
336        };
337
338        // Three same-shape element-wise ops fuse into one pass.
339        assert_eq!(plan_fusion(&[ew(1), ew(1), ew(1)]), 1);
340        // A barrier (matmul/reduction) splits the run: EW | barrier | EW = 3 passes.
341        assert_eq!(plan_fusion(&[ew(1), barrier(1), ew(1)]), 3);
342        // Different shapes don't fuse.
343        assert_eq!(plan_fusion(&[ew(1), ew(2)]), 2);
344        // Mixed run: (EW EW) | barrier | (EW EW EW) = 3 passes.
345        assert_eq!(
346            plan_fusion(&[ew(1), ew(1), barrier(9), ew(2), ew(2), ew(2)]),
347            3
348        );
349        assert_eq!(plan_fusion(&[]), 0);
350    }
351
352    #[test]
353    fn precision_fits_budget_and_respects_thermals() {
354        let gib = 1u64 << 30;
355        // 1B params, roomy VRAM + headroom → f32.
356        let roomy = PowerThermalBudget {
357            vram_budget_bytes: 8 * gib,
358            power_budget_mw: 60_000,
359            thermal_headroom_c: 30.0,
360        };
361        assert_eq!(select_precision(1_000_000_000, &roomy), Precision::F32);
362
363        // 1B params, only ~1.5 GiB VRAM → must drop to q8 (1 GB) — f32/f16 overflow.
364        let tight_vram = PowerThermalBudget {
365            vram_budget_bytes: gib + gib / 2,
366            power_budget_mw: 60_000,
367            thermal_headroom_c: 30.0,
368        };
369        assert_eq!(select_precision(1_000_000_000, &tight_vram), Precision::Q8);
370
371        // Throttling (low thermal headroom) → never f32/f16 even with VRAM to spare.
372        let throttling = PowerThermalBudget {
373            vram_budget_bytes: 64 * gib,
374            power_budget_mw: 60_000,
375            thermal_headroom_c: 2.0,
376        };
377        assert_eq!(select_precision(1_000_000_000, &throttling), Precision::Q8);
378
379        // Model too big for any precision → Q4 (caller tiles/offloads).
380        let huge = PowerThermalBudget {
381            vram_budget_bytes: gib,
382            power_budget_mw: 60_000,
383            thermal_headroom_c: 30.0,
384        };
385        assert_eq!(select_precision(10_000_000_000, &huge), Precision::Q4);
386    }
387}