Skip to main content

qualia_core_db/inference/
inference_path_selector.rs

1//! Device-optimal inference path selection.
2//!
3//! Design intent (Timothy): a **benchmark utility** identifies the fastest method on
4//! *this* machine (wgpu DX12 / Vulkan / Metal / GL, optional CUDA lane, CPU) and the
5//! product **picks that path** — not a static hierarchy.
6//!
7//! ## What is selected
8//! | Axis | Candidates | Measured by |
9//! |------|------------|-------------|
10//! | **API backend** | dx12, vulkan, metal, gl | Hardware passport GEMV + decode-proxy |
11//! | **Compute lane** | portable (resident wgpu), cuda (forge TC + Q4 SoA GEMV) | Capability flags + optional micro-policy |
12//! | **Quant profile** | INT4 SoA weights, INT8 KV, quant-graph quality | A2000-class bandwidth + rights mode |
13//! | **Prefill vs decode** | TC GEMM when batch/dims allow; GEMV for m=1 decode | Shape policy, not GEMV score alone |
14//!
15//! Vulkan and DX12 both use the **same resident weight plan** (VRAM-sticky multi-weight);
16//! there is no separate "Vulkan multi-weight" — multi-weight without host RT is the
17//! **resident decode** path. CUDA multi-weight is the optional densify/Q4 device slab.
18//!
19//! Env pins always win: `QUALIA_WGPU_BACKEND`, `QUALIA_INFERENCE_MODE`, `QUALIA_PATH_AUTO=0`.
20
21#![cfg(not(target_arch = "wasm32"))]
22
23use std::sync::atomic::{AtomicBool, Ordering};
24use std::sync::OnceLock;
25
26use crate::inference_modes::{set_inference_mode, InferenceMode};
27use crate::llm_bench::{
28    set_coop_gemv, set_ffn_fusion, set_kv_int8, set_resident_decode, set_resident_prefill,
29    set_resident_weights,
30};
31
32/// How decode math is executed after the wgpu API backend is chosen.
33#[derive(Debug, Clone, Copy, PartialEq, Eq)]
34pub enum ComputeLane {
35    /// Single-fence resident decode/prefill on wgpu (DX12/Vulkan/Metal).
36    PortableResident,
37    /// Prefer CUDA TC prefill + Q4 device GEMV when eligible; else portable.
38    CudaAccelerated,
39}
40
41impl ComputeLane {
42    /// Static string.
43    pub fn as_str(self) -> &'static str {
44        match self {
45            Self::PortableResident => "portable-resident",
46            Self::CudaAccelerated => "cuda-accelerated",
47        }
48    }
49}
50
51/// Quantization / quality profile for consumer GPUs.
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub enum QuantProfile {
54    /// INT4 SoA weights + INT8 KV (A2000 bandwidth sweet spot).
55    Int4SoaInt8Kv,
56    /// Same as above + quant-graph hybrid quality net.
57    Int4SoaGraphHybrid,
58}
59
60impl QuantProfile {
61    /// Static string.
62    pub fn as_str(self) -> &'static str {
63        match self {
64            Self::Int4SoaInt8Kv => "int4-soa+int8-kv",
65            Self::Int4SoaGraphHybrid => "int4-soa+int8-kv+graph",
66        }
67    }
68}
69
70/// Full selected plan for this process / host.
71#[derive(Debug, Clone, PartialEq)]
72pub struct InferencePathPlan {
73    /// `dx12` | `vulkan` | `metal` | `gl` | unknown
74    pub wgpu_backend: Option<String>,
75    pub compute_lane: ComputeLane,
76    pub quant: QuantProfile,
77    /// Use tensor-core dense GEMM for prefill when dims allow (CUDA lane).
78    pub prefill_prefer_tc: bool,
79    /// Decode uses GEMV (m=1); never pretend TC replaces decode.
80    pub decode_is_gemv: bool,
81    /// Human-readable reason chain.
82    pub rationale: String,
83    /// Passport circuit label that won, if any.
84    pub winning_circuit: Option<String>,
85    pub decode_proxy_tok_s: Option<f64>,
86    pub gemv_ms: Option<f64>,
87}
88
89static APPLIED: AtomicBool = AtomicBool::new(false);
90static LAST_PLAN: OnceLock<std::sync::Mutex<Option<InferencePathPlan>>> = OnceLock::new();
91
92fn last_plan_slot() -> &'static std::sync::Mutex<Option<InferencePathPlan>> {
93    LAST_PLAN.get_or_init(|| std::sync::Mutex::new(None))
94}
95
96/// Last plan applied (or resolved without apply).
97pub fn last_inference_path_plan() -> Option<InferencePathPlan> {
98    last_plan_slot().lock().ok().and_then(|g| g.clone())
99}
100
101/// Whether auto path selection is enabled (`QUALIA_PATH_AUTO` default on).
102pub fn path_auto_enabled() -> bool {
103    match std::env::var("QUALIA_PATH_AUTO").ok().as_deref() {
104        Some("0") | Some("false") | Some("off") => false,
105        _ => true,
106    }
107}
108
109/// Resolve the optimal plan from passport + host capabilities (does not mutate process yet).
110pub fn resolve_inference_path_plan() -> InferencePathPlan {
111    let mut rationale = Vec::new();
112
113    // ── 1. wgpu API backend from passport (or env pin) ─────────────────────
114    let env_backend = std::env::var("QUALIA_WGPU_BACKEND")
115        .ok()
116        .map(|s| s.trim().to_ascii_lowercase())
117        .filter(|s| !s.is_empty());
118
119    let (wgpu_backend, winning, decode_tok, gemv_ms) = if let Some(ref b) = env_backend {
120        rationale.push(format!("wgpu backend pinned by QUALIA_WGPU_BACKEND={b}"));
121        (Some(b.clone()), None, None, None)
122    } else if let Some(p) =
123        crate::hardware_passport::read_passport(&crate::hardware_passport::default_cache_path())
124    {
125        let best = p.matrix.best().cloned();
126        let token = p.preferred_inference_backend.clone().or_else(|| {
127            best.as_ref().and_then(|c| {
128                crate::hardware_passport::backend_env_token(&c.backend).map(str::to_string)
129            })
130        });
131        if let Some(ref c) = best {
132            rationale.push(format!(
133                "passport winner: {} [{}/{}] gemv={:.3}ms decode={:?}",
134                c.label,
135                format!("{:?}", c.kind),
136                c.backend,
137                c.ms_per_gemv,
138                c.decode_proxy_tok_s
139            ));
140        } else {
141            rationale.push("passport present but empty matrix".into());
142        }
143        (
144            token,
145            best.as_ref().map(|c| c.label.clone()),
146            best.as_ref().and_then(|c| c.decode_proxy_tok_s),
147            best.as_ref().map(|c| c.ms_per_gemv),
148        )
149    } else {
150        rationale.push(
151            "no hardware passport — run `qualia-cli llm passport --reprobe --decode-proxy <model> --apply-env-hint`"
152                .into(),
153        );
154        // Platform defaults: Metal on Apple, else leave unset (gpu_context picks).
155        let def = if cfg!(target_os = "macos") || cfg!(target_os = "ios") {
156            rationale.push("default metal on Apple".into());
157            Some("metal".into())
158        } else {
159            None
160        };
161        (def, None, None, None)
162    };
163
164    // ── 2. Compute lane: CUDA only when toolkit present + not slower policy ─
165    let cuda_caps = cuda_toolkit_present();
166    let mode_pin = std::env::var("QUALIA_INFERENCE_MODE").ok();
167    let compute_lane = if let Some(ref m) = mode_pin {
168        if matches!(
169            m.to_ascii_lowercase().as_str(),
170            "cuda" | "cuda-tc" | "cudatc" | "tc" | "1"
171        ) {
172            rationale.push("compute lane pinned by QUALIA_INFERENCE_MODE=cuda".into());
173            ComputeLane::CudaAccelerated
174        } else {
175            rationale.push(format!("compute lane portable (mode pin {m})"));
176            ComputeLane::PortableResident
177        }
178    } else if cuda_caps && prefer_cuda_lane_heuristic() {
179        rationale.push(
180            "CUDA toolkit detected — lane=cuda-accelerated for prefill TC / Q4 device GEMV; decode still GEMV-primary"
181                .into(),
182        );
183        ComputeLane::CudaAccelerated
184    } else {
185        if cuda_caps {
186            rationale.push(
187                "CUDA present but portable-resident preferred (set QUALIA_INFERENCE_MODE=cuda to force)"
188                    .into(),
189            );
190        } else {
191            rationale.push("no CUDA — portable resident (wgpu multi-weight in VRAM)".into());
192        }
193        ComputeLane::PortableResident
194    };
195
196    // ── 3. Quant + quality ────────────────────────────────────────────────
197    let rights = crate::inference_modes::rights_mode_enabled()
198        || matches!(
199            mode_pin
200                .as_deref()
201                .map(|s| s.to_ascii_lowercase())
202                .as_deref(),
203            Some("quant-graph")
204                | Some("graph")
205                | Some("hybrid")
206                | Some("2")
207                | Some("fast-verify")
208                | Some("fast_verify")
209                | Some("3")
210        );
211    // Prefer FastVerify when operator asks for ollama-like + post heal without mid-decode tax.
212    if matches!(
213        mode_pin
214            .as_deref()
215            .map(|s| s.to_ascii_lowercase())
216            .as_deref(),
217        Some("fast-verify") | Some("fast_verify") | Some("ollama-like") | Some("3")
218    ) {
219        rationale.push(
220            "mode pin fast-verify: full-speed decode then post-turn CML/graph self-heal".into(),
221        );
222    }
223    let quant = if rights {
224        rationale.push("quant profile: INT4 SoA + INT8 KV + quant-graph hybrid".into());
225        QuantProfile::Int4SoaGraphHybrid
226    } else {
227        rationale.push("quant profile: INT4 SoA weights + INT8 KV (consumer bandwidth)".into());
228        QuantProfile::Int4SoaInt8Kv
229    };
230
231    let prefill_prefer_tc = matches!(compute_lane, ComputeLane::CudaAccelerated);
232    if prefill_prefer_tc {
233        rationale.push("prefill: prefer TC dense GEMM when m,n,k multiples of 16".into());
234    }
235    rationale
236        .push("decode: always GEMV (m=1) — tensor cores do not replace single-token GEMV".into());
237    rationale.push(
238        "multi-weight: wgpu resident plan keeps layer weights in VRAM (Vulkan/DX12/Metal); CUDA slab is optional densify path"
239            .into(),
240    );
241
242    InferencePathPlan {
243        wgpu_backend,
244        compute_lane,
245        quant,
246        prefill_prefer_tc,
247        decode_is_gemv: true,
248        rationale: rationale.join(" | "),
249        winning_circuit: winning,
250        decode_proxy_tok_s: decode_tok,
251        gemv_ms,
252    }
253}
254
255/// Forge CUDA capability probe. Absent `wgsl-forge`, CPU/portable selection still compiles.
256fn cuda_toolkit_present() -> bool {
257    #[cfg(feature = "wgsl-forge")]
258    {
259        crate::wgsl_forge::dispatch::caps().cuda
260    }
261    #[cfg(not(feature = "wgsl-forge"))]
262    {
263        false
264    }
265}
266
267/// Heuristic: enable CUDA lane when not explicitly disabled and runtime path exists.
268fn prefer_cuda_lane_heuristic() -> bool {
269    // Default OFF for CUDA as primary decode — measured A2000 still favored portable resident.
270    // Opt-in via QUALIA_PREFER_CUDA=1 or mode=cuda. Prefill can still use forge when mode=cuda.
271    matches!(
272        std::env::var("QUALIA_PREFER_CUDA").ok().as_deref(),
273        Some("1") | Some("true") | Some("on")
274    )
275}
276
277/// Apply plan to process toggles (once per process unless `force`).
278///
279/// Sets inference mode / bench flags. Does **not** set `QUALIA_WGPU_BACKEND` if already
280/// set (gpu_context already honors env + passport). Optionally writes passport preferred
281/// fields for the next process.
282pub fn apply_inference_path_plan(plan: &InferencePathPlan, force: bool) -> bool {
283    if force {
284        APPLIED.store(true, Ordering::SeqCst);
285    } else if APPLIED.swap(true, Ordering::SeqCst) {
286        return false; // already applied this process
287    }
288
289    // Resident always on for product path.
290    set_resident_decode(true);
291    set_resident_prefill(true);
292    set_resident_weights(true);
293    set_coop_gemv(true);
294    set_ffn_fusion(true);
295    set_kv_int8(true);
296
297    match plan.quant {
298        QuantProfile::Int4SoaInt8Kv => {
299            // Leave mode as portable unless compute lane says cuda.
300        }
301        QuantProfile::Int4SoaGraphHybrid => {
302            if std::env::var("QUALIA_INFERENCE_MODE").is_err() {
303                // Prefer post-turn FastVerify for consumer rights path when not pinned.
304                if crate::inference_modes::rights_mode_enabled()
305                    || matches!(
306                        std::env::var("QUALIA_PREFER_FAST_VERIFY").ok().as_deref(),
307                        Some("1") | Some("true")
308                    )
309                {
310                    set_inference_mode(InferenceMode::FastVerify);
311                } else {
312                    set_inference_mode(InferenceMode::QuantGraph);
313                }
314            }
315        }
316    }
317
318    match plan.compute_lane {
319        ComputeLane::PortableResident => {
320            if std::env::var("QUALIA_INFERENCE_MODE").is_err()
321                && !matches!(plan.quant, QuantProfile::Int4SoaGraphHybrid)
322            {
323                set_inference_mode(InferenceMode::Portable);
324            }
325        }
326        ComputeLane::CudaAccelerated => {
327            if std::env::var("QUALIA_INFERENCE_MODE").is_err() {
328                set_inference_mode(InferenceMode::CudaTc);
329            }
330            #[cfg(feature = "wgsl-forge")]
331            crate::wgsl_forge::dispatch::ensure_cuda_runtime_path();
332        }
333    }
334
335    // Soft-set backend for child processes / logs if unset.
336    if std::env::var("QUALIA_WGPU_BACKEND").is_err() {
337        if let Some(ref b) = plan.wgpu_backend {
338            // SAFETY: process-local config before GPU init; documented operator surface.
339            std::env::set_var("QUALIA_WGPU_BACKEND", b);
340            log::info!("path_select|set_env|QUALIA_WGPU_BACKEND={b}");
341        }
342    }
343
344    log::info!(
345        "path_select|applied|backend={:?}|lane={}|quant={}|prefill_tc={}|{}",
346        plan.wgpu_backend,
347        plan.compute_lane.as_str(),
348        plan.quant.as_str(),
349        plan.prefill_prefer_tc,
350        plan.rationale
351    );
352
353    if let Ok(mut g) = last_plan_slot().lock() {
354        *g = Some(plan.clone());
355    }
356    true
357}
358
359/// Resolve + apply if `QUALIA_PATH_AUTO` enabled. Called from `bootstrap_inference_mode`.
360pub fn bootstrap_optimal_inference_path() -> InferencePathPlan {
361    let plan = resolve_inference_path_plan();
362    if path_auto_enabled() {
363        apply_inference_path_plan(&plan, false);
364    } else {
365        log::info!("path_select|skipped|QUALIA_PATH_AUTO=0|{}", plan.rationale);
366        if let Ok(mut g) = last_plan_slot().lock() {
367            *g = Some(plan.clone());
368        }
369    }
370    plan
371}
372
373/// Operator-facing summary (CLI / doctor).
374pub fn format_path_plan(plan: &InferencePathPlan) -> String {
375    format!(
376        "InferencePathPlan\n  wgpu_backend:     {}\n  compute_lane:     {}\n  quant:            {}\n  prefill_TC:       {}\n  decode:           GEMV (m=1)\n  winning_circuit:  {}\n  decode_proxy:     {}\n  gemv_ms:          {}\n  rationale:\n    {}\n",
377        plan.wgpu_backend.as_deref().unwrap_or("(platform default)"),
378        plan.compute_lane.as_str(),
379        plan.quant.as_str(),
380        plan.prefill_prefer_tc,
381        plan.winning_circuit.as_deref().unwrap_or("—"),
382        plan.decode_proxy_tok_s
383            .map(|t| format!("{t:.2} tok/s"))
384            .unwrap_or_else(|| "—".into()),
385        plan.gemv_ms
386            .map(|m| format!("{m:.3}"))
387            .unwrap_or_else(|| "—".into()),
388        plan.rationale.replace(" | ", "\n    "),
389    )
390}
391
392/// Probe passport (optional re-probe) and print/apply plan. Used by CLI.
393pub fn run_path_select_cli(reprobe: bool, apply: bool) -> InferencePathPlan {
394    if reprobe {
395        let _ = crate::hardware_passport::load_or_probe(
396            &crate::hardware_passport::default_cache_path(),
397            crate::hardware_passport::PASSPORT_GEMV_N,
398        );
399    }
400    let plan = resolve_inference_path_plan();
401    if apply {
402        apply_inference_path_plan(&plan, true);
403        // Persist preferred backend into passport if we have one.
404        if let Some(mut p) =
405            crate::hardware_passport::read_passport(&crate::hardware_passport::default_cache_path())
406        {
407            if let Some(ref b) = plan.wgpu_backend {
408                p.preferred_inference_backend = Some(b.clone());
409            }
410            let _ = crate::hardware_passport::write_passport(
411                &p,
412                &crate::hardware_passport::default_cache_path(),
413            );
414        }
415    }
416    plan
417}
418
419#[cfg(test)]
420mod tests {
421    use super::*;
422
423    #[test]
424    #[serial_test::serial]
425    fn resolve_does_not_panic() {
426        let p = resolve_inference_path_plan();
427        assert!(p.decode_is_gemv);
428        assert!(!p.rationale.is_empty());
429    }
430
431    #[test]
432    fn format_contains_axes() {
433        let p = InferencePathPlan {
434            wgpu_backend: Some("dx12".into()),
435            compute_lane: ComputeLane::PortableResident,
436            quant: QuantProfile::Int4SoaInt8Kv,
437            prefill_prefer_tc: false,
438            decode_is_gemv: true,
439            rationale: "test".into(),
440            winning_circuit: Some("A2000".into()),
441            decode_proxy_tok_s: Some(2.5),
442            gemv_ms: Some(0.11),
443        };
444        let s = format_path_plan(&p);
445        assert!(s.contains("dx12"));
446        assert!(s.contains("portable-resident"));
447        assert!(s.contains("GEMV"));
448    }
449}