qualia_core_db/inference/
inference_path_selector.rs1#![cfg(not(target_arch = "wasm32"))]
22
23use std::sync::atomic::{AtomicBool, Ordering};
24use std::sync::OnceLock;
25
26use crate::inference_modes::{set_inference_mode, InferenceMode};
27use crate::llm_bench::{
28 set_coop_gemv, set_ffn_fusion, set_kv_int8, set_resident_decode, set_resident_prefill,
29 set_resident_weights,
30};
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
34pub enum ComputeLane {
35 PortableResident,
37 CudaAccelerated,
39}
40
41impl ComputeLane {
42 pub fn as_str(self) -> &'static str {
44 match self {
45 Self::PortableResident => "portable-resident",
46 Self::CudaAccelerated => "cuda-accelerated",
47 }
48 }
49}
50
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub enum QuantProfile {
54 Int4SoaInt8Kv,
56 Int4SoaGraphHybrid,
58}
59
60impl QuantProfile {
61 pub fn as_str(self) -> &'static str {
63 match self {
64 Self::Int4SoaInt8Kv => "int4-soa+int8-kv",
65 Self::Int4SoaGraphHybrid => "int4-soa+int8-kv+graph",
66 }
67 }
68}
69
70#[derive(Debug, Clone, PartialEq)]
72pub struct InferencePathPlan {
73 pub wgpu_backend: Option<String>,
75 pub compute_lane: ComputeLane,
76 pub quant: QuantProfile,
77 pub prefill_prefer_tc: bool,
79 pub decode_is_gemv: bool,
81 pub rationale: String,
83 pub winning_circuit: Option<String>,
85 pub decode_proxy_tok_s: Option<f64>,
86 pub gemv_ms: Option<f64>,
87}
88
89static APPLIED: AtomicBool = AtomicBool::new(false);
90static LAST_PLAN: OnceLock<std::sync::Mutex<Option<InferencePathPlan>>> = OnceLock::new();
91
92fn last_plan_slot() -> &'static std::sync::Mutex<Option<InferencePathPlan>> {
93 LAST_PLAN.get_or_init(|| std::sync::Mutex::new(None))
94}
95
96pub fn last_inference_path_plan() -> Option<InferencePathPlan> {
98 last_plan_slot().lock().ok().and_then(|g| g.clone())
99}
100
101pub fn path_auto_enabled() -> bool {
103 match std::env::var("QUALIA_PATH_AUTO").ok().as_deref() {
104 Some("0") | Some("false") | Some("off") => false,
105 _ => true,
106 }
107}
108
109pub fn resolve_inference_path_plan() -> InferencePathPlan {
111 let mut rationale = Vec::new();
112
113 let env_backend = std::env::var("QUALIA_WGPU_BACKEND")
115 .ok()
116 .map(|s| s.trim().to_ascii_lowercase())
117 .filter(|s| !s.is_empty());
118
119 let (wgpu_backend, winning, decode_tok, gemv_ms) = if let Some(ref b) = env_backend {
120 rationale.push(format!("wgpu backend pinned by QUALIA_WGPU_BACKEND={b}"));
121 (Some(b.clone()), None, None, None)
122 } else if let Some(p) =
123 crate::hardware_passport::read_passport(&crate::hardware_passport::default_cache_path())
124 {
125 let best = p.matrix.best().cloned();
126 let token = p.preferred_inference_backend.clone().or_else(|| {
127 best.as_ref().and_then(|c| {
128 crate::hardware_passport::backend_env_token(&c.backend).map(str::to_string)
129 })
130 });
131 if let Some(ref c) = best {
132 rationale.push(format!(
133 "passport winner: {} [{}/{}] gemv={:.3}ms decode={:?}",
134 c.label,
135 format!("{:?}", c.kind),
136 c.backend,
137 c.ms_per_gemv,
138 c.decode_proxy_tok_s
139 ));
140 } else {
141 rationale.push("passport present but empty matrix".into());
142 }
143 (
144 token,
145 best.as_ref().map(|c| c.label.clone()),
146 best.as_ref().and_then(|c| c.decode_proxy_tok_s),
147 best.as_ref().map(|c| c.ms_per_gemv),
148 )
149 } else {
150 rationale.push(
151 "no hardware passport — run `qualia-cli llm passport --reprobe --decode-proxy <model> --apply-env-hint`"
152 .into(),
153 );
154 let def = if cfg!(target_os = "macos") || cfg!(target_os = "ios") {
156 rationale.push("default metal on Apple".into());
157 Some("metal".into())
158 } else {
159 None
160 };
161 (def, None, None, None)
162 };
163
164 let cuda_caps = cuda_toolkit_present();
166 let mode_pin = std::env::var("QUALIA_INFERENCE_MODE").ok();
167 let compute_lane = if let Some(ref m) = mode_pin {
168 if matches!(
169 m.to_ascii_lowercase().as_str(),
170 "cuda" | "cuda-tc" | "cudatc" | "tc" | "1"
171 ) {
172 rationale.push("compute lane pinned by QUALIA_INFERENCE_MODE=cuda".into());
173 ComputeLane::CudaAccelerated
174 } else {
175 rationale.push(format!("compute lane portable (mode pin {m})"));
176 ComputeLane::PortableResident
177 }
178 } else if cuda_caps && prefer_cuda_lane_heuristic() {
179 rationale.push(
180 "CUDA toolkit detected — lane=cuda-accelerated for prefill TC / Q4 device GEMV; decode still GEMV-primary"
181 .into(),
182 );
183 ComputeLane::CudaAccelerated
184 } else {
185 if cuda_caps {
186 rationale.push(
187 "CUDA present but portable-resident preferred (set QUALIA_INFERENCE_MODE=cuda to force)"
188 .into(),
189 );
190 } else {
191 rationale.push("no CUDA — portable resident (wgpu multi-weight in VRAM)".into());
192 }
193 ComputeLane::PortableResident
194 };
195
196 let rights = crate::inference_modes::rights_mode_enabled()
198 || matches!(
199 mode_pin
200 .as_deref()
201 .map(|s| s.to_ascii_lowercase())
202 .as_deref(),
203 Some("quant-graph")
204 | Some("graph")
205 | Some("hybrid")
206 | Some("2")
207 | Some("fast-verify")
208 | Some("fast_verify")
209 | Some("3")
210 );
211 if matches!(
213 mode_pin
214 .as_deref()
215 .map(|s| s.to_ascii_lowercase())
216 .as_deref(),
217 Some("fast-verify") | Some("fast_verify") | Some("ollama-like") | Some("3")
218 ) {
219 rationale.push(
220 "mode pin fast-verify: full-speed decode then post-turn CML/graph self-heal".into(),
221 );
222 }
223 let quant = if rights {
224 rationale.push("quant profile: INT4 SoA + INT8 KV + quant-graph hybrid".into());
225 QuantProfile::Int4SoaGraphHybrid
226 } else {
227 rationale.push("quant profile: INT4 SoA weights + INT8 KV (consumer bandwidth)".into());
228 QuantProfile::Int4SoaInt8Kv
229 };
230
231 let prefill_prefer_tc = matches!(compute_lane, ComputeLane::CudaAccelerated);
232 if prefill_prefer_tc {
233 rationale.push("prefill: prefer TC dense GEMM when m,n,k multiples of 16".into());
234 }
235 rationale
236 .push("decode: always GEMV (m=1) — tensor cores do not replace single-token GEMV".into());
237 rationale.push(
238 "multi-weight: wgpu resident plan keeps layer weights in VRAM (Vulkan/DX12/Metal); CUDA slab is optional densify path"
239 .into(),
240 );
241
242 InferencePathPlan {
243 wgpu_backend,
244 compute_lane,
245 quant,
246 prefill_prefer_tc,
247 decode_is_gemv: true,
248 rationale: rationale.join(" | "),
249 winning_circuit: winning,
250 decode_proxy_tok_s: decode_tok,
251 gemv_ms,
252 }
253}
254
255fn cuda_toolkit_present() -> bool {
257 #[cfg(feature = "wgsl-forge")]
258 {
259 crate::wgsl_forge::dispatch::caps().cuda
260 }
261 #[cfg(not(feature = "wgsl-forge"))]
262 {
263 false
264 }
265}
266
267fn prefer_cuda_lane_heuristic() -> bool {
269 matches!(
272 std::env::var("QUALIA_PREFER_CUDA").ok().as_deref(),
273 Some("1") | Some("true") | Some("on")
274 )
275}
276
277pub fn apply_inference_path_plan(plan: &InferencePathPlan, force: bool) -> bool {
283 if force {
284 APPLIED.store(true, Ordering::SeqCst);
285 } else if APPLIED.swap(true, Ordering::SeqCst) {
286 return false; }
288
289 set_resident_decode(true);
291 set_resident_prefill(true);
292 set_resident_weights(true);
293 set_coop_gemv(true);
294 set_ffn_fusion(true);
295 set_kv_int8(true);
296
297 match plan.quant {
298 QuantProfile::Int4SoaInt8Kv => {
299 }
301 QuantProfile::Int4SoaGraphHybrid => {
302 if std::env::var("QUALIA_INFERENCE_MODE").is_err() {
303 if crate::inference_modes::rights_mode_enabled()
305 || matches!(
306 std::env::var("QUALIA_PREFER_FAST_VERIFY").ok().as_deref(),
307 Some("1") | Some("true")
308 )
309 {
310 set_inference_mode(InferenceMode::FastVerify);
311 } else {
312 set_inference_mode(InferenceMode::QuantGraph);
313 }
314 }
315 }
316 }
317
318 match plan.compute_lane {
319 ComputeLane::PortableResident => {
320 if std::env::var("QUALIA_INFERENCE_MODE").is_err()
321 && !matches!(plan.quant, QuantProfile::Int4SoaGraphHybrid)
322 {
323 set_inference_mode(InferenceMode::Portable);
324 }
325 }
326 ComputeLane::CudaAccelerated => {
327 if std::env::var("QUALIA_INFERENCE_MODE").is_err() {
328 set_inference_mode(InferenceMode::CudaTc);
329 }
330 #[cfg(feature = "wgsl-forge")]
331 crate::wgsl_forge::dispatch::ensure_cuda_runtime_path();
332 }
333 }
334
335 if std::env::var("QUALIA_WGPU_BACKEND").is_err() {
337 if let Some(ref b) = plan.wgpu_backend {
338 std::env::set_var("QUALIA_WGPU_BACKEND", b);
340 log::info!("path_select|set_env|QUALIA_WGPU_BACKEND={b}");
341 }
342 }
343
344 log::info!(
345 "path_select|applied|backend={:?}|lane={}|quant={}|prefill_tc={}|{}",
346 plan.wgpu_backend,
347 plan.compute_lane.as_str(),
348 plan.quant.as_str(),
349 plan.prefill_prefer_tc,
350 plan.rationale
351 );
352
353 if let Ok(mut g) = last_plan_slot().lock() {
354 *g = Some(plan.clone());
355 }
356 true
357}
358
359pub fn bootstrap_optimal_inference_path() -> InferencePathPlan {
361 let plan = resolve_inference_path_plan();
362 if path_auto_enabled() {
363 apply_inference_path_plan(&plan, false);
364 } else {
365 log::info!("path_select|skipped|QUALIA_PATH_AUTO=0|{}", plan.rationale);
366 if let Ok(mut g) = last_plan_slot().lock() {
367 *g = Some(plan.clone());
368 }
369 }
370 plan
371}
372
373pub fn format_path_plan(plan: &InferencePathPlan) -> String {
375 format!(
376 "InferencePathPlan\n wgpu_backend: {}\n compute_lane: {}\n quant: {}\n prefill_TC: {}\n decode: GEMV (m=1)\n winning_circuit: {}\n decode_proxy: {}\n gemv_ms: {}\n rationale:\n {}\n",
377 plan.wgpu_backend.as_deref().unwrap_or("(platform default)"),
378 plan.compute_lane.as_str(),
379 plan.quant.as_str(),
380 plan.prefill_prefer_tc,
381 plan.winning_circuit.as_deref().unwrap_or("—"),
382 plan.decode_proxy_tok_s
383 .map(|t| format!("{t:.2} tok/s"))
384 .unwrap_or_else(|| "—".into()),
385 plan.gemv_ms
386 .map(|m| format!("{m:.3}"))
387 .unwrap_or_else(|| "—".into()),
388 plan.rationale.replace(" | ", "\n "),
389 )
390}
391
392pub fn run_path_select_cli(reprobe: bool, apply: bool) -> InferencePathPlan {
394 if reprobe {
395 let _ = crate::hardware_passport::load_or_probe(
396 &crate::hardware_passport::default_cache_path(),
397 crate::hardware_passport::PASSPORT_GEMV_N,
398 );
399 }
400 let plan = resolve_inference_path_plan();
401 if apply {
402 apply_inference_path_plan(&plan, true);
403 if let Some(mut p) =
405 crate::hardware_passport::read_passport(&crate::hardware_passport::default_cache_path())
406 {
407 if let Some(ref b) = plan.wgpu_backend {
408 p.preferred_inference_backend = Some(b.clone());
409 }
410 let _ = crate::hardware_passport::write_passport(
411 &p,
412 &crate::hardware_passport::default_cache_path(),
413 );
414 }
415 }
416 plan
417}
418
419#[cfg(test)]
420mod tests {
421 use super::*;
422
423 #[test]
424 #[serial_test::serial]
425 fn resolve_does_not_panic() {
426 let p = resolve_inference_path_plan();
427 assert!(p.decode_is_gemv);
428 assert!(!p.rationale.is_empty());
429 }
430
431 #[test]
432 fn format_contains_axes() {
433 let p = InferencePathPlan {
434 wgpu_backend: Some("dx12".into()),
435 compute_lane: ComputeLane::PortableResident,
436 quant: QuantProfile::Int4SoaInt8Kv,
437 prefill_prefer_tc: false,
438 decode_is_gemv: true,
439 rationale: "test".into(),
440 winning_circuit: Some("A2000".into()),
441 decode_proxy_tok_s: Some(2.5),
442 gemv_ms: Some(0.11),
443 };
444 let s = format_path_plan(&p);
445 assert!(s.contains("dx12"));
446 assert!(s.contains("portable-resident"));
447 assert!(s.contains("GEMV"));
448 }
449}