Skip to main content

qualia_core_db/inference/
inference_modes.rs

1//! Multi-mode inference — keep parallel approaches, do not dump one pipeline.
2//!
3//! Timothy (2026-07-09): different *modes* rather than replacing the portable
4//! resident path. Modes are selected by env / API; each has a honest role:
5//!
6//! | Mode | Role |
7//! |------|------|
8//! | **Portable** | wgpu resident decode/prefill (DX12/Vulkan/Metal) — default product path |
9//! | **CudaTc** | Prefer forge CUDA WMMA / TC GEMM when dims allow; fall back to Portable |
10//! | **QuantGraph** | Aggressive INT4/INT8 + **mid** hybrid hints + post graph repair |
11//! | **FastVerify** | Ollama-like full-speed decode (no mid-token Sentinel tax) → **post-turn** CML/graph self-heal + HTML |
12//!
13//! Modes compose toggles already in `inference_bench`; they do not invent a
14//! second engine. Graph-hybrid quality recovery is neuro-symbolic: LLM proposes,
15//! graph + Logic VM grounds, domain engines compute exact subproblems.
16
17use std::sync::atomic::{AtomicU8, Ordering};
18
19/// Active inference approach for this process.
20#[repr(u8)]
21#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
22pub enum InferenceMode {
23    /// wgpu single-fence resident decode/prefill — default, portable.
24    Portable = 0,
25    /// Prefer CUDA tensor-core dense GEMM via forge when eligible; else Portable.
26    CudaTc = 1,
27    /// Quantized front-end + graph/Webizen grounding for quality recovery.
28    QuantGraph = 2,
29    /// Full-speed decode (skip mid-token Sentinel rings) then post-turn verify/heal.
30    FastVerify = 3,
31}
32
33impl InferenceMode {
34    pub const ALL: [InferenceMode; 4] = [
35        InferenceMode::Portable,
36        InferenceMode::CudaTc,
37        InferenceMode::QuantGraph,
38        InferenceMode::FastVerify,
39    ];
40
41    /// Stable string form.
42    pub fn as_str(self) -> &'static str {
43        match self {
44            InferenceMode::Portable => "portable",
45            InferenceMode::CudaTc => "cuda",
46            InferenceMode::QuantGraph => "quant-graph",
47            InferenceMode::FastVerify => "fast-verify",
48        }
49    }
50
51    /// Parses input; error on malformed data.
52    pub fn parse(s: &str) -> Option<Self> {
53        match s.trim().to_ascii_lowercase().as_str() {
54            "portable" | "wgpu" | "default" | "0" => Some(Self::Portable),
55            "cuda" | "cuda-tc" | "cudatc" | "tc" | "1" => Some(Self::CudaTc),
56            "quant-graph" | "quant_graph" | "graph" | "hybrid" | "neuro-symbolic" | "2" => {
57                Some(Self::QuantGraph)
58            }
59            "fast-verify" | "fast_verify" | "fastverify" | "ollama-like" | "post-verify"
60            | "verify" | "3" => Some(Self::FastVerify),
61            _ => None,
62        }
63    }
64
65    /// Human-readable description of the mode.
66    pub fn description(self) -> &'static str {
67        match self {
68            InferenceMode::Portable => {
69                "wgpu resident decode/prefill (DX12/Vulkan/Metal); Q4_K SoA INT4 coop GEMV; INT8 KV default; no second DirectML device unless QUALIA_DIRECTML=1"
70            }
71            InferenceMode::CudaTc => {
72                "CUDA-capable lane: resident wgpu decode by default (coherent); Q4_K SoA device GEMV when applicable; dense densify+TC decode GEMV is opt-in QUALIA_LLM_CUDA_TC_DECODE=1 (lab — was incoherent when default-on)"
73            }
74            InferenceMode::QuantGraph => {
75                "INT4 (Q4_K SoA) front-end — A2000 bandwidth sweet spot — + graph/Webizen verify to recover quality; INT2 experimental with graph net"
76            }
77            InferenceMode::FastVerify => {
78                "Ollama-like full-speed decode (no mid-token Sentinel) → post-turn quant-graph + CML/HTML self-heal before turn finalises"
79            }
80        }
81    }
82}
83
84static MODE: AtomicU8 = AtomicU8::new(InferenceMode::Portable as u8);
85
86#[inline]
87pub(crate) fn atomic_inference_mode() -> InferenceMode {
88    #[cfg(not(target_arch = "wasm32"))]
89    if let Some(mode) =
90        crate::llm_bench::prepared_settings::read(|settings| settings.inference_mode)
91    {
92        return mode;
93    }
94    match MODE.load(Ordering::Relaxed) {
95        1 => InferenceMode::CudaTc,
96        2 => InferenceMode::QuantGraph,
97        3 => InferenceMode::FastVerify,
98        _ => InferenceMode::Portable,
99    }
100}
101
102/// Resolve mode from `QUALIA_INFERENCE_MODE` at the cold configuration boundary.
103///
104/// A successful environment parse is published to `MODE`. Per-token predicates must read
105/// that atomic via `atomic_inference_mode` instead of allocating a fresh environment string.
106pub fn active_inference_mode() -> InferenceMode {
107    #[cfg(not(target_arch = "wasm32"))]
108    if let Some(mode) =
109        crate::llm_bench::prepared_settings::read(|settings| settings.inference_mode)
110    {
111        return mode;
112    }
113    // Env can override the configured atomic when this cold-boundary API is invoked.
114    if let Ok(s) = std::env::var("QUALIA_INFERENCE_MODE") {
115        if let Some(m) = InferenceMode::parse(&s) {
116            MODE.store(m as u8, Ordering::Relaxed);
117            return m;
118        }
119    }
120    atomic_inference_mode()
121}
122
123/// Set process mode and apply associated toggles. Env still wins on next read if set.
124pub fn set_inference_mode(mode: InferenceMode) {
125    #[cfg(all(feature = "cuda", not(target_arch = "wasm32")))]
126    crate::inference::cuda_lane::install_execution_policy();
127    MODE.store(mode as u8, Ordering::Relaxed);
128    apply_mode_toggles(mode);
129    log::info!("LLM_MODE|active|{}|{}", mode.as_str(), mode.description());
130}
131
132/// Apply mode-specific defaults without changing the mode atom (used at first infer).
133pub fn apply_mode_toggles(mode: InferenceMode) {
134    #[cfg(not(target_arch = "wasm32"))]
135    {
136        use crate::llm_bench::{
137            set_coop_gemv, set_resident_decode, set_resident_prefill, set_resident_weights,
138        };
139        // All modes keep resident paths ON by default; they differ in accel / grounding.
140        set_resident_decode(true);
141        set_resident_prefill(true);
142        set_resident_weights(true);
143        set_coop_gemv(true);
144        match mode {
145            InferenceMode::Portable => {
146                // Explicit: do not force CUDA TC for dense forge calls.
147            }
148            InferenceMode::CudaTc => {
149                #[cfg(feature = "wgsl-forge")]
150                crate::wgsl_forge::dispatch::ensure_cuda_runtime_path();
151                // Default: resident mega-pass ON (measured ~6.5–7 tok/s on 3B).
152                // QUALIA_LLM_CUDA_DECODE=1 opts into the layer-by-layer CUDA SoA path
153                // (P4: device RoPE/KV/SDPA + sticky Q4_K_SOA). Lab / A-B only until
154                // it beats resident. Device SDPA requires f32 KV (int8 indices differ).
155                match std::env::var("QUALIA_LLM_CUDA_DECODE").ok().as_deref() {
156                    Some("1") | Some("true") => {
157                        // Keep resident_decode ON — the wgpu resident path handles mixed
158                        // quant types (Q4_K_M has Q8_0 V, Q6_K down) and serves as fallback
159                        // when the CUDA mega-pass can't run (requires all-Q4_K_SOA weights).
160                        // The mega-pass is only attempted when the resident path returns None.
161                        // Force f32 KV so device SDPA index formula matches host layout.
162                        crate::llm_bench::set_kv_int8(false);
163                        // SAFETY: process-local lab toggle; decode-proxy is single-threaded measure.
164                        std::env::set_var("QUALIA_LLM_KV_INT8", "0");
165                        log::info!(
166                            "LLM_MODE|cuda|resident_decode=on|cuda_mega_pass_fallback|f32_kv|device_sdpa|lab_path"
167                        );
168                    }
169                    _ => {
170                        // Default excellence path: resident mega-pass + wgpu GEMV.
171                        // Dense densify+TC for decode GEMV is OFF unless QUALIA_LLM_CUDA_TC_DECODE=1
172                        // (that path was measured incoherent — garbage tokens — 2026-07-24).
173                        log::info!(
174                            "LLM_MODE|cuda|resident_decode=on|dense_tc_decode=off|use_q4k_soa_device_when_present"
175                        );
176                    }
177                }
178            }
179            InferenceMode::QuantGraph => {
180                // Prefer bandwidth-friendly layouts; graph verify is opt-in at agent layer.
181                // FFN f16 promote stays off (measured slower on A2000 for Q4).
182                // Refresh fact graph from bundled TSV / QUALIA_GROUNDING_FACTS.
183                let n = crate::quant_graph_grounding::seed_facts_from_bundled();
184                log::info!("LLM_MODE|quant-graph|facts_seeded|{n}");
185            }
186            InferenceMode::FastVerify => {
187                // Same weight path as portable; quality is *post-turn* only.
188                let n = crate::quant_graph_grounding::seed_facts_from_bundled();
189                log::info!("LLM_MODE|fast-verify|post_turn_only|facts_seeded|{n}");
190            }
191        }
192    }
193    let _ = mode;
194}
195
196/// True when dense forge GEMM should try tensor-core path first.
197#[inline]
198pub fn prefer_tensor_core_gemm() -> bool {
199    matches!(atomic_inference_mode(), InferenceMode::CudaTc)
200}
201
202/// True when agent should run graph / Webizen grounding after proposals.
203/// FastVerify also grounds — but only post-turn (see `post_turn_verify_enabled`).
204#[inline]
205pub fn quant_graph_grounding_enabled() -> bool {
206    matches!(
207        active_inference_mode(),
208        InferenceMode::QuantGraph | InferenceMode::FastVerify
209    )
210}
211
212/// Mid-decode Webizen Sentinel / logit-ring governance active?
213/// Off in FastVerify so decode matches Ollama-style uninterrupted generation.
214#[inline]
215pub fn sentinel_mid_decode_enabled() -> bool {
216    // Explicit override.
217    match std::env::var("QUALIA_SENTINEL_MID").ok().as_deref() {
218        Some("1") | Some("true") | Some("on") => return true,
219        Some("0") | Some("false") | Some("off") => return false,
220        _ => {}
221    }
222    !matches!(active_inference_mode(), InferenceMode::FastVerify)
223}
224
225/// Post-turn verify + self-heal (graph/CML/HTML) before finalising the turn.
226#[inline]
227pub fn post_turn_verify_enabled() -> bool {
228    matches!(
229        active_inference_mode(),
230        InferenceMode::FastVerify | InferenceMode::QuantGraph
231    ) || matches!(
232        std::env::var("QUALIA_POST_VERIFY").ok().as_deref(),
233        Some("1") | Some("true") | Some("on")
234    )
235}
236
237/// FastVerify defaults to returning plain healed text; HTML when this is true.
238#[inline]
239pub fn fast_verify_html_default() -> bool {
240    matches!(
241        std::env::var("QUALIA_RETURN_VERIFY_HTML").ok().as_deref(),
242        Some("1") | Some("true") | Some("on")
243    )
244}
245
246/// One-shot: resolve env, apply toggles, return mode (call from agent entry).
247///
248/// Order:
249/// 1. Device path selector (passport-ranked backend + lane + quant) when `QUALIA_PATH_AUTO`
250/// 2. Rights-mode quant-graph if still unset
251/// 3. Active mode toggles (resident, coop GEMV, INT8 KV, …)
252pub fn bootstrap_inference_mode() -> InferenceMode {
253    #[cfg(not(target_arch = "wasm32"))]
254    {
255        let _plan = crate::inference_path_selector::bootstrap_optimal_inference_path();
256        // Application profile (interactive / live-fast / batch overnight).
257        let _app = crate::application_profile::bootstrap_application_profile();
258    }
259    // Rights-grade consumer default: FastVerify (speed + post-heal) unless pinned.
260    if matches!(
261        std::env::var("QUALIA_RIGHTS_MODE").ok().as_deref(),
262        Some("1") | Some("true") | Some("on")
263    ) {
264        if std::env::var("QUALIA_INFERENCE_MODE").is_err()
265            && std::env::var("QUALIA_APP_PROFILE").is_err()
266        {
267            set_inference_mode(InferenceMode::FastVerify);
268        }
269        log::info!("LLM_MODE|rights|fast-verify|post_turn_heal");
270    }
271    // Unpinned default on NVIDIA: FastVerify (smol ~60+ tok/s). Operator can pin
272    // QUALIA_INFERENCE_MODE=cuda for large SoA models (3B ~7 tok/s measured).
273    if std::env::var("QUALIA_INFERENCE_MODE").is_err()
274        && std::env::var("QUALIA_APP_PROFILE").is_err()
275        && std::env::var("QUALIA_RIGHTS_MODE").is_err()
276    {
277        set_inference_mode(InferenceMode::FastVerify);
278        log::info!("LLM_MODE|default|fast-verify|consumer_speed");
279    }
280    let m = active_inference_mode();
281    apply_mode_toggles(m);
282    m
283}
284
285/// True when operator asked for rights-grade defaults (`QUALIA_RIGHTS_MODE`).
286#[inline]
287pub fn rights_mode_enabled() -> bool {
288    matches!(
289        std::env::var("QUALIA_RIGHTS_MODE").ok().as_deref(),
290        Some("1") | Some("true") | Some("on")
291    )
292}
293
294#[cfg(test)]
295mod tests {
296    use super::*;
297
298    #[test]
299    fn parse_mode_names() {
300        assert_eq!(
301            InferenceMode::parse("portable"),
302            Some(InferenceMode::Portable)
303        );
304        assert_eq!(InferenceMode::parse("CUDA"), Some(InferenceMode::CudaTc));
305        assert_eq!(
306            InferenceMode::parse("quant-graph"),
307            Some(InferenceMode::QuantGraph)
308        );
309        assert_eq!(
310            InferenceMode::parse("hybrid"),
311            Some(InferenceMode::QuantGraph)
312        );
313        assert_eq!(
314            InferenceMode::parse("fast-verify"),
315            Some(InferenceMode::FastVerify)
316        );
317        assert_eq!(
318            InferenceMode::parse("ollama-like"),
319            Some(InferenceMode::FastVerify)
320        );
321        assert!(InferenceMode::parse("nope").is_none());
322    }
323
324    #[test]
325    #[serial_test::serial]
326    fn fast_verify_disables_mid_sentinel() {
327        if std::env::var("QUALIA_INFERENCE_MODE").is_ok()
328            || std::env::var("QUALIA_SENTINEL_MID").is_ok()
329        {
330            return;
331        }
332        set_inference_mode(InferenceMode::FastVerify);
333        assert!(!sentinel_mid_decode_enabled());
334        assert!(post_turn_verify_enabled());
335        set_inference_mode(InferenceMode::Portable);
336        assert!(sentinel_mid_decode_enabled());
337    }
338
339    #[test]
340    #[serial_test::serial]
341    fn set_and_read_without_env() {
342        // Do not assert env-free if the machine has QUALIA_INFERENCE_MODE set.
343        if std::env::var("QUALIA_INFERENCE_MODE").is_ok() {
344            return;
345        }
346        set_inference_mode(InferenceMode::CudaTc);
347        assert_eq!(active_inference_mode(), InferenceMode::CudaTc);
348        set_inference_mode(InferenceMode::Portable);
349        assert_eq!(active_inference_mode(), InferenceMode::Portable);
350    }
351
352    #[test]
353    #[serial_test::serial]
354    fn cold_env_publish_makes_hot_cuda_guard_zero_allocation() {
355        let previous = std::env::var("QUALIA_INFERENCE_MODE").ok();
356        std::env::set_var("QUALIA_INFERENCE_MODE", "cuda");
357        set_inference_mode(InferenceMode::Portable);
358        assert_eq!(active_inference_mode(), InferenceMode::CudaTc);
359        crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc(
360            "atomic_cuda_mode_guard",
361            || assert!(prefer_tensor_core_gemm()),
362        );
363        match previous {
364            Some(value) => {
365                std::env::set_var("QUALIA_INFERENCE_MODE", value);
366                let _ = active_inference_mode();
367            }
368            None => {
369                std::env::remove_var("QUALIA_INFERENCE_MODE");
370                set_inference_mode(InferenceMode::Portable);
371            }
372        }
373    }
374}