qualia_core_db/inference/
inference_modes.rs1use std::sync::atomic::{AtomicU8, Ordering};
18
19#[repr(u8)]
21#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
22pub enum InferenceMode {
23 Portable = 0,
25 CudaTc = 1,
27 QuantGraph = 2,
29 FastVerify = 3,
31}
32
33impl InferenceMode {
34 pub const ALL: [InferenceMode; 4] = [
35 InferenceMode::Portable,
36 InferenceMode::CudaTc,
37 InferenceMode::QuantGraph,
38 InferenceMode::FastVerify,
39 ];
40
41 pub fn as_str(self) -> &'static str {
43 match self {
44 InferenceMode::Portable => "portable",
45 InferenceMode::CudaTc => "cuda",
46 InferenceMode::QuantGraph => "quant-graph",
47 InferenceMode::FastVerify => "fast-verify",
48 }
49 }
50
51 pub fn parse(s: &str) -> Option<Self> {
53 match s.trim().to_ascii_lowercase().as_str() {
54 "portable" | "wgpu" | "default" | "0" => Some(Self::Portable),
55 "cuda" | "cuda-tc" | "cudatc" | "tc" | "1" => Some(Self::CudaTc),
56 "quant-graph" | "quant_graph" | "graph" | "hybrid" | "neuro-symbolic" | "2" => {
57 Some(Self::QuantGraph)
58 }
59 "fast-verify" | "fast_verify" | "fastverify" | "ollama-like" | "post-verify"
60 | "verify" | "3" => Some(Self::FastVerify),
61 _ => None,
62 }
63 }
64
65 pub fn description(self) -> &'static str {
67 match self {
68 InferenceMode::Portable => {
69 "wgpu resident decode/prefill (DX12/Vulkan/Metal); Q4_K SoA INT4 coop GEMV; INT8 KV default; no second DirectML device unless QUALIA_DIRECTML=1"
70 }
71 InferenceMode::CudaTc => {
72 "CUDA-capable lane: resident wgpu decode by default (coherent); Q4_K SoA device GEMV when applicable; dense densify+TC decode GEMV is opt-in QUALIA_LLM_CUDA_TC_DECODE=1 (lab — was incoherent when default-on)"
73 }
74 InferenceMode::QuantGraph => {
75 "INT4 (Q4_K SoA) front-end — A2000 bandwidth sweet spot — + graph/Webizen verify to recover quality; INT2 experimental with graph net"
76 }
77 InferenceMode::FastVerify => {
78 "Ollama-like full-speed decode (no mid-token Sentinel) → post-turn quant-graph + CML/HTML self-heal before turn finalises"
79 }
80 }
81 }
82}
83
84static MODE: AtomicU8 = AtomicU8::new(InferenceMode::Portable as u8);
85
86#[inline]
87pub(crate) fn atomic_inference_mode() -> InferenceMode {
88 #[cfg(not(target_arch = "wasm32"))]
89 if let Some(mode) =
90 crate::llm_bench::prepared_settings::read(|settings| settings.inference_mode)
91 {
92 return mode;
93 }
94 match MODE.load(Ordering::Relaxed) {
95 1 => InferenceMode::CudaTc,
96 2 => InferenceMode::QuantGraph,
97 3 => InferenceMode::FastVerify,
98 _ => InferenceMode::Portable,
99 }
100}
101
102pub fn active_inference_mode() -> InferenceMode {
107 #[cfg(not(target_arch = "wasm32"))]
108 if let Some(mode) =
109 crate::llm_bench::prepared_settings::read(|settings| settings.inference_mode)
110 {
111 return mode;
112 }
113 if let Ok(s) = std::env::var("QUALIA_INFERENCE_MODE") {
115 if let Some(m) = InferenceMode::parse(&s) {
116 MODE.store(m as u8, Ordering::Relaxed);
117 return m;
118 }
119 }
120 atomic_inference_mode()
121}
122
123pub fn set_inference_mode(mode: InferenceMode) {
125 #[cfg(all(feature = "cuda", not(target_arch = "wasm32")))]
126 crate::inference::cuda_lane::install_execution_policy();
127 MODE.store(mode as u8, Ordering::Relaxed);
128 apply_mode_toggles(mode);
129 log::info!("LLM_MODE|active|{}|{}", mode.as_str(), mode.description());
130}
131
132pub fn apply_mode_toggles(mode: InferenceMode) {
134 #[cfg(not(target_arch = "wasm32"))]
135 {
136 use crate::llm_bench::{
137 set_coop_gemv, set_resident_decode, set_resident_prefill, set_resident_weights,
138 };
139 set_resident_decode(true);
141 set_resident_prefill(true);
142 set_resident_weights(true);
143 set_coop_gemv(true);
144 match mode {
145 InferenceMode::Portable => {
146 }
148 InferenceMode::CudaTc => {
149 #[cfg(feature = "wgsl-forge")]
150 crate::wgsl_forge::dispatch::ensure_cuda_runtime_path();
151 match std::env::var("QUALIA_LLM_CUDA_DECODE").ok().as_deref() {
156 Some("1") | Some("true") => {
157 crate::llm_bench::set_kv_int8(false);
163 std::env::set_var("QUALIA_LLM_KV_INT8", "0");
165 log::info!(
166 "LLM_MODE|cuda|resident_decode=on|cuda_mega_pass_fallback|f32_kv|device_sdpa|lab_path"
167 );
168 }
169 _ => {
170 log::info!(
174 "LLM_MODE|cuda|resident_decode=on|dense_tc_decode=off|use_q4k_soa_device_when_present"
175 );
176 }
177 }
178 }
179 InferenceMode::QuantGraph => {
180 let n = crate::quant_graph_grounding::seed_facts_from_bundled();
184 log::info!("LLM_MODE|quant-graph|facts_seeded|{n}");
185 }
186 InferenceMode::FastVerify => {
187 let n = crate::quant_graph_grounding::seed_facts_from_bundled();
189 log::info!("LLM_MODE|fast-verify|post_turn_only|facts_seeded|{n}");
190 }
191 }
192 }
193 let _ = mode;
194}
195
196#[inline]
198pub fn prefer_tensor_core_gemm() -> bool {
199 matches!(atomic_inference_mode(), InferenceMode::CudaTc)
200}
201
202#[inline]
205pub fn quant_graph_grounding_enabled() -> bool {
206 matches!(
207 active_inference_mode(),
208 InferenceMode::QuantGraph | InferenceMode::FastVerify
209 )
210}
211
212#[inline]
215pub fn sentinel_mid_decode_enabled() -> bool {
216 match std::env::var("QUALIA_SENTINEL_MID").ok().as_deref() {
218 Some("1") | Some("true") | Some("on") => return true,
219 Some("0") | Some("false") | Some("off") => return false,
220 _ => {}
221 }
222 !matches!(active_inference_mode(), InferenceMode::FastVerify)
223}
224
225#[inline]
227pub fn post_turn_verify_enabled() -> bool {
228 matches!(
229 active_inference_mode(),
230 InferenceMode::FastVerify | InferenceMode::QuantGraph
231 ) || matches!(
232 std::env::var("QUALIA_POST_VERIFY").ok().as_deref(),
233 Some("1") | Some("true") | Some("on")
234 )
235}
236
237#[inline]
239pub fn fast_verify_html_default() -> bool {
240 matches!(
241 std::env::var("QUALIA_RETURN_VERIFY_HTML").ok().as_deref(),
242 Some("1") | Some("true") | Some("on")
243 )
244}
245
246pub fn bootstrap_inference_mode() -> InferenceMode {
253 #[cfg(not(target_arch = "wasm32"))]
254 {
255 let _plan = crate::inference_path_selector::bootstrap_optimal_inference_path();
256 let _app = crate::application_profile::bootstrap_application_profile();
258 }
259 if matches!(
261 std::env::var("QUALIA_RIGHTS_MODE").ok().as_deref(),
262 Some("1") | Some("true") | Some("on")
263 ) {
264 if std::env::var("QUALIA_INFERENCE_MODE").is_err()
265 && std::env::var("QUALIA_APP_PROFILE").is_err()
266 {
267 set_inference_mode(InferenceMode::FastVerify);
268 }
269 log::info!("LLM_MODE|rights|fast-verify|post_turn_heal");
270 }
271 if std::env::var("QUALIA_INFERENCE_MODE").is_err()
274 && std::env::var("QUALIA_APP_PROFILE").is_err()
275 && std::env::var("QUALIA_RIGHTS_MODE").is_err()
276 {
277 set_inference_mode(InferenceMode::FastVerify);
278 log::info!("LLM_MODE|default|fast-verify|consumer_speed");
279 }
280 let m = active_inference_mode();
281 apply_mode_toggles(m);
282 m
283}
284
285#[inline]
287pub fn rights_mode_enabled() -> bool {
288 matches!(
289 std::env::var("QUALIA_RIGHTS_MODE").ok().as_deref(),
290 Some("1") | Some("true") | Some("on")
291 )
292}
293
294#[cfg(test)]
295mod tests {
296 use super::*;
297
298 #[test]
299 fn parse_mode_names() {
300 assert_eq!(
301 InferenceMode::parse("portable"),
302 Some(InferenceMode::Portable)
303 );
304 assert_eq!(InferenceMode::parse("CUDA"), Some(InferenceMode::CudaTc));
305 assert_eq!(
306 InferenceMode::parse("quant-graph"),
307 Some(InferenceMode::QuantGraph)
308 );
309 assert_eq!(
310 InferenceMode::parse("hybrid"),
311 Some(InferenceMode::QuantGraph)
312 );
313 assert_eq!(
314 InferenceMode::parse("fast-verify"),
315 Some(InferenceMode::FastVerify)
316 );
317 assert_eq!(
318 InferenceMode::parse("ollama-like"),
319 Some(InferenceMode::FastVerify)
320 );
321 assert!(InferenceMode::parse("nope").is_none());
322 }
323
324 #[test]
325 #[serial_test::serial]
326 fn fast_verify_disables_mid_sentinel() {
327 if std::env::var("QUALIA_INFERENCE_MODE").is_ok()
328 || std::env::var("QUALIA_SENTINEL_MID").is_ok()
329 {
330 return;
331 }
332 set_inference_mode(InferenceMode::FastVerify);
333 assert!(!sentinel_mid_decode_enabled());
334 assert!(post_turn_verify_enabled());
335 set_inference_mode(InferenceMode::Portable);
336 assert!(sentinel_mid_decode_enabled());
337 }
338
339 #[test]
340 #[serial_test::serial]
341 fn set_and_read_without_env() {
342 if std::env::var("QUALIA_INFERENCE_MODE").is_ok() {
344 return;
345 }
346 set_inference_mode(InferenceMode::CudaTc);
347 assert_eq!(active_inference_mode(), InferenceMode::CudaTc);
348 set_inference_mode(InferenceMode::Portable);
349 assert_eq!(active_inference_mode(), InferenceMode::Portable);
350 }
351
352 #[test]
353 #[serial_test::serial]
354 fn cold_env_publish_makes_hot_cuda_guard_zero_allocation() {
355 let previous = std::env::var("QUALIA_INFERENCE_MODE").ok();
356 std::env::set_var("QUALIA_INFERENCE_MODE", "cuda");
357 set_inference_mode(InferenceMode::Portable);
358 assert_eq!(active_inference_mode(), InferenceMode::CudaTc);
359 crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc(
360 "atomic_cuda_mode_guard",
361 || assert!(prefer_tensor_core_gemm()),
362 );
363 match previous {
364 Some(value) => {
365 std::env::set_var("QUALIA_INFERENCE_MODE", value);
366 let _ = active_inference_mode();
367 }
368 None => {
369 std::env::remove_var("QUALIA_INFERENCE_MODE");
370 set_inference_mode(InferenceMode::Portable);
371 }
372 }
373 }
374}