qualia_core_db/inference/inference_bench/toggles.rs
1//! Runtime-config toggles read by the real inference path (decode / prefill /
2//! attention / GEMM selection). Each is a process-global flag with a `set_*` /
3//! `*_enabled` pair; env vars override where documented. Also the GEMM backend
4//! selector. Pure code motion — behaviour unchanged.
5
6use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
7use std::sync::Mutex;
8
9// ── Decode budget override ────────────────────────────────────────────────────
10// A bounded, fixed decode-token count gives stable, comparable tok/s. The real
11// decode loop reads this once per call; 0 = use the production `DECODE_TOKEN_BUDGET`.
12
13static DECODE_BUDGET_OVERRIDE: AtomicU32 = AtomicU32::new(0);
14
15// Application-profile bootstrap is intentionally allowed to reset the normal
16// override to its production default. A benchmark is different: it requires a
17// fixed, reproducible budget for its complete cold/warm scope. Keep that
18// request separate so profile startup in the inference thread cannot silently
19// turn a requested 64-token measurement back into the 256-token UI default.
20static BENCHMARK_DECODE_BUDGET_OVERRIDE: AtomicU32 = AtomicU32::new(0);
21static BENCHMARK_DECODE_SCOPE: Mutex<()> = Mutex::new(());
22
23/// Serializes and installs a benchmark-only decode budget.
24///
25/// The returned guard restores the production behaviour when dropped. This
26/// is process-global because the underlying inference toggles are global too;
27/// concurrent benchmarks are therefore deliberately serialized.
28pub struct BenchmarkDecodeBudgetGuard {
29 _scope: std::sync::MutexGuard<'static, ()>,
30}
31
32impl Drop for BenchmarkDecodeBudgetGuard {
33 fn drop(&mut self) {
34 BENCHMARK_DECODE_BUDGET_OVERRIDE.store(0, Ordering::Release);
35 }
36}
37
38/// RAII guard installing a temporary decode budget; restores the previous value on drop.
39pub fn benchmark_decode_budget_scope(n: u32) -> BenchmarkDecodeBudgetGuard {
40 let scope = BENCHMARK_DECODE_SCOPE
41 .lock()
42 .unwrap_or_else(|poisoned| poisoned.into_inner());
43 BENCHMARK_DECODE_BUDGET_OVERRIDE.store(n, Ordering::Release);
44 BenchmarkDecodeBudgetGuard { _scope: scope }
45}
46
47/// Set a fixed decode-token budget for benchmarking (0 = production default).
48#[inline]
49pub fn set_decode_budget_override(n: u32) {
50 DECODE_BUDGET_OVERRIDE.store(n, Ordering::Relaxed);
51}
52
53/// When budget override is active, ignore EOS so A/B runs a fixed token count
54/// (prevents early-stop from inflating/deflating tok/s on short prompts).
55#[inline]
56pub fn decode_budget_fixed_tokens() -> bool {
57 decode_budget_override() > 0
58}
59
60/// Current decode-budget override (0 = none).
61#[inline]
62pub fn decode_budget_override() -> u32 {
63 if let Some(value) = super::prepared_settings::read(|settings| settings.decode_budget_override)
64 {
65 return value;
66 }
67 let benchmark = BENCHMARK_DECODE_BUDGET_OVERRIDE.load(Ordering::Acquire);
68 if benchmark > 0 {
69 benchmark
70 } else {
71 DECODE_BUDGET_OVERRIDE.load(Ordering::Relaxed)
72 }
73}
74
75// ── Wall-clock inference timeout override (batch / overnight jobs) ────────────
76// 0 = use production `INFERENCE_TIMEOUT_MS` (30s interactive). Batch profile may
77// raise this to hours for multi-system health differential analysis overnight.
78static INFERENCE_TIMEOUT_OVERRIDE_MS: AtomicU64 = AtomicU64::new(0);
79
80/// Set wall-clock decode timeout in ms (0 = production default 30_000).
81#[inline]
82pub fn set_inference_timeout_override_ms(ms: u64) {
83 INFERENCE_TIMEOUT_OVERRIDE_MS.store(ms, Ordering::Relaxed);
84}
85
86/// Effective timeout: override if non-zero, else env `QUALIA_INFERENCE_TIMEOUT_MS`, else 30s.
87#[inline]
88pub fn inference_timeout_ms() -> u64 {
89 if let Some(value) = super::prepared_settings::read(|settings| settings.inference_timeout_ms) {
90 return value;
91 }
92 let o = INFERENCE_TIMEOUT_OVERRIDE_MS.load(Ordering::Relaxed);
93 if o > 0 {
94 return o;
95 }
96 if let Ok(s) = std::env::var("QUALIA_INFERENCE_TIMEOUT_MS") {
97 if let Ok(v) = s.parse::<u64>() {
98 if v > 0 {
99 return v;
100 }
101 }
102 }
103 30_000
104}
105
106// ── A1a GPU top-k toggle (D18) ────────────────────────────────────────────────
107// Default ON after widening the output gate and adding the allocation-free top-1 path. The decode
108// loop reads only block winners from the GPU instead of full vocabulary chunks when no sieve mask is
109// active.
110// Set `QUALIA_LLM_GPU_TOPK=0` to force the full-logit argmax fallback.
111static GPU_TOPK: AtomicBool = AtomicBool::new(true);
112
113/// Enable/disable the GPU top-k decode path (`QUALIA_LLM_GPU_TOPK`).
114#[inline]
115pub fn set_gpu_topk(on: bool) {
116 GPU_TOPK.store(on, Ordering::Relaxed);
117}
118
119/// Whether the GPU top-k decode path is active. The env var overrides the flag in BOTH directions
120/// (`0`/`false` → off, `1`/`true` → on); otherwise the process default (ON) applies.
121#[inline]
122pub fn gpu_topk_enabled() -> bool {
123 if let Some(value) = super::prepared_settings::read(|settings| settings.gpu_topk_enabled) {
124 return value;
125 }
126 match std::env::var("QUALIA_LLM_GPU_TOPK").ok().as_deref() {
127 Some("0") | Some("false") => false,
128 Some("1") | Some("true") => true,
129 _ => GPU_TOPK.load(Ordering::Relaxed),
130 }
131}
132
133// ── A1b ternary-FFN toggle (D3/D7) ────────────────────────────────────────────
134// Additive, default-OFF: when a `.q42` ternary container is booted, routes its FFN
135// GEMMs through the resident 2-bit GPU kernel (`TernaryFfnResident`). OFF runs the
136// SAME ternary weights via the CPU oracle — so ON-vs-OFF isolates the GPU-kernel win
137// on identical weights, and ternary-container-vs-Q8 (a0) is the headline FFN number.
138static TERNARY_FFN: AtomicBool = AtomicBool::new(false);
139
140/// Enable/disable the resident 2-bit GPU ternary-FFN path (`QUALIA_LLM_TERNARY_FFN`).
141#[inline]
142pub fn set_ternary_ffn(on: bool) {
143 TERNARY_FFN.store(on, Ordering::Relaxed);
144}
145
146/// Whether the GPU ternary-FFN path is active (atomic flag OR the env var). When false, ternary
147/// FFN GEMMs fall back to the CPU oracle (correct, slower) — the toggle's OFF baseline.
148#[inline]
149pub fn ternary_ffn_enabled() -> bool {
150 if let Some(value) = super::prepared_settings::read(|settings| settings.ternary_ffn_enabled) {
151 return value;
152 }
153 TERNARY_FFN.load(Ordering::Relaxed)
154 || matches!(
155 std::env::var("QUALIA_LLM_TERNARY_FFN").ok().as_deref(),
156 Some("1") | Some("true")
157 )
158}
159
160// Native attention projection split: K/V matmuls run through cooperative GEMV, then the attention
161// shader consumes the projected rows via `proj_row_stride` only for RoPE + KV-cache writes. Default
162// ON after the no-readback fused K/V path replaced the old diagnostic readback implementation.
163// Set `QUALIA_LLM_PREPROJECT_ATTN=0` to force the legacy in-attention projection path.
164static ATTN_PREPROJECT: AtomicBool = AtomicBool::new(true);
165
166#[inline]
167/// Enables/disables the attention pre-projection bench toggle (atomic).
168pub fn set_attention_preproject(on: bool) {
169 ATTN_PREPROJECT.store(on, Ordering::Relaxed);
170}
171
172#[inline]
173/// Current attention pre-projection toggle state.
174pub fn attention_preproject_enabled() -> bool {
175 if let Some(value) =
176 super::prepared_settings::read(|settings| settings.attention_preproject_enabled)
177 {
178 return value;
179 }
180 // W5b Phase 4b: the dict-coded write happens in `write_kv_head` (the attention pass); the fused
181 // pre-projection bypasses it, so force it off in dict mode.
182 if kv_dict_enabled() {
183 return false;
184 }
185 match std::env::var("QUALIA_LLM_PREPROJECT_ATTN").ok().as_deref() {
186 Some("0") | Some("false") => false,
187 Some("1") | Some("true") => true,
188 _ => ATTN_PREPROJECT.load(Ordering::Relaxed),
189 }
190}
191
192// Native attention tail fusion: Q-attention writes its output to a GPU buffer, and o_proj consumes
193// that buffer directly after K and V are both present in the KV cache. Default ON (native): removes
194// one submit->wait round-trip per layer while preserving token identity against the proven readback
195// path. Set `QUALIA_LLM_FUSE_ATTN_O=0` to force the older Q-readback + o_proj path.
196static ATTN_O_FUSE: AtomicBool = AtomicBool::new(true);
197
198#[inline]
199/// Enables/disables the attention-o fusion bench toggle.
200pub fn set_attention_o_fuse(on: bool) {
201 ATTN_O_FUSE.store(on, Ordering::Relaxed);
202}
203
204#[inline]
205/// Current attention-o fusion state.
206pub fn attention_o_fuse_enabled() -> bool {
207 if let Some(value) =
208 super::prepared_settings::read(|settings| settings.attention_o_fuse_enabled)
209 {
210 return value;
211 }
212 // W5b Phase 4b: the fused Q+O tail reads K/V from the cache on the GPU path; keep the plain
213 // read_k/read_v (dict-aware) path in dict mode.
214 if kv_dict_enabled() {
215 return false;
216 }
217 match std::env::var("QUALIA_LLM_FUSE_ATTN_O").ok().as_deref() {
218 Some("0") | Some("false") => false,
219 Some("1") | Some("true") => true,
220 _ => ATTN_O_FUSE.load(Ordering::Relaxed),
221 }
222}
223
224// Certified Forge WGSL attention: dispatches Naga-validated Forge causal attention
225// directly on wgpu device during decode step.
226static FORGE_ATTENTION: AtomicBool = AtomicBool::new(false);
227
228#[inline]
229/// Enables/disables the forge-attention bench toggle.
230pub fn set_forge_attention(on: bool) {
231 FORGE_ATTENTION.store(on, Ordering::Relaxed);
232}
233
234#[inline]
235/// Current forge-attention state.
236pub fn forge_attention_enabled() -> bool {
237 if let Some(value) = super::prepared_settings::read(|settings| settings.forge_attention_enabled)
238 {
239 return value;
240 }
241 match std::env::var("QUALIA_LLM_FORGE_ATTENTION").ok().as_deref() {
242 Some("1") | Some("true") => true,
243 Some("0") | Some("false") => false,
244 _ => FORGE_ATTENTION.load(Ordering::Relaxed),
245 }
246}
247
248// ── Phase 2: resident weights toggle ──────────────────────────────────────────
249// Default ON (native). Each layer's q/k/v/o/gate/up/down weight is uploaded to its own resident
250// VRAM buffer once (keyed by the GGUF tensor byte_offset) and reused every token, instead of
251// re-`write_buffer`ing the (up to ~50 MB for a 3B FFN tensor) weight into the shared GEMM buffer
252// on every GEMM, every token. For a 3B F16 model that re-upload is ~5 GB/token of PCIe traffic —
253// the decode bottleneck. Set `QUALIA_LLM_RESIDENT_WEIGHTS=0` to force the per-token re-upload (the
254// A/B OFF baseline) — useful for measuring the win or on VRAM-constrained GPUs.
255static RESIDENT_WEIGHTS: AtomicBool = AtomicBool::new(true);
256
257/// Enable/disable the resident per-tensor weight buffers (`QUALIA_LLM_RESIDENT_WEIGHTS`).
258#[inline]
259pub fn set_resident_weights(on: bool) {
260 RESIDENT_WEIGHTS.store(on, Ordering::Relaxed);
261}
262
263/// Whether native GEMM should bind resident per-tensor weight buffers (upload-once) rather than
264/// re-uploading the weight every token. Env forces either direction; otherwise the atomic flag.
265#[inline]
266pub fn resident_weights_enabled() -> bool {
267 if let Some(value) =
268 super::prepared_settings::read(|settings| settings.resident_weights_enabled)
269 {
270 return value;
271 }
272 match std::env::var("QUALIA_LLM_RESIDENT_WEIGHTS").ok().as_deref() {
273 Some("0") | Some("false") => false,
274 Some("1") | Some("true") => true,
275 _ => RESIDENT_WEIGHTS.load(Ordering::Relaxed),
276 }
277}
278
279// ── Resident-token decode toggle (single fence per token) ─────────────────────
280// Default ON (native). Keeps the hidden state in VRAM for the WHOLE token: all 32 layers
281// (RMSNorm/residuals as GPU elem ops) + output norm + chunked logits top-1 are encoded into ONE
282// command submit with ONE blocking fence and a ~400 B candidate readback — replacing the legacy
283// ~107 submit→wait round-trips/token (measured ~24% pure fence time on SmolLM2-360M, A2000,
284// Vulkan). Any per-model ineligibility (unsupported quant, no resident logits, sieve mask, CPU
285// attention) falls back to the legacy per-layer path. `QUALIA_LLM_RESIDENT_DECODE=0` forces the
286// legacy path (the A/B baseline + the differential-test comparator).
287static RESIDENT_DECODE: AtomicBool = AtomicBool::new(true);
288
289/// Enable/disable the resident-token single-fence decode (`QUALIA_LLM_RESIDENT_DECODE`).
290#[inline]
291pub fn set_resident_decode(on: bool) {
292 RESIDENT_DECODE.store(on, Ordering::Relaxed);
293}
294
295/// Whether native decode should run the GPU-resident single-fence token path.
296#[inline]
297pub fn resident_decode_enabled() -> bool {
298 if let Some(value) = super::prepared_settings::read(|settings| settings.resident_decode_enabled)
299 {
300 return value;
301 }
302 match std::env::var("QUALIA_LLM_RESIDENT_DECODE").ok().as_deref() {
303 Some("0") | Some("false") => false,
304 Some("1") | Some("true") => true,
305 _ => RESIDENT_DECODE.load(Ordering::Relaxed),
306 }
307}
308
309// ── W3: resident single-fence-per-chunk prefill toggle ────────────────────────
310// Default ON (verified). When on (and the model is eligible — GPU-eligible weights, coop GEMV, no
311// active sparse-attention route), each prefill chunk of ≤PREFILL_CHUNK_SIZE prompt tokens populates
312// the KV cache in ONE command submit / ONE fence (all 32 layers batched + resident hidden state)
313// instead of the legacy per-layer + per-token Q/FFN loop (~640 submit→wait round-trips for a 10-token
314// prompt). Delivers TTFT and the batched-forward primitive W6a-verify needs; on a fast discrete GPU
315// the steady-state win is latent (prefill is compute-bound), the fence win lands on
316// edge/mobile/under-load. Passed the `a3a` gate on SmolLM2-360M Q8 (A2000): the batched RMSNorm
317// reduces in the same sequential order as the legacy CPU path, so the KV it writes is BYTE-IDENTICAL
318// → decode-identical, with the int8 KV cache both ON and OFF. Any ineligibility falls back to the
319// legacy `dispatch_prefill_chunk` path unchanged. `QUALIA_LLM_RESIDENT_PREFILL=0` forces legacy.
320static RESIDENT_PREFILL: AtomicBool = AtomicBool::new(true);
321
322/// Enable/disable the resident single-fence-per-chunk prefill path (`QUALIA_LLM_RESIDENT_PREFILL`).
323#[inline]
324pub fn set_resident_prefill(on: bool) {
325 RESIDENT_PREFILL.store(on, Ordering::Relaxed);
326}
327
328/// Whether native prefill should run the GPU-resident single-fence-per-chunk arena.
329#[inline]
330pub fn resident_prefill_enabled() -> bool {
331 if let Some(value) =
332 super::prepared_settings::read(|settings| settings.resident_prefill_enabled)
333 {
334 return value;
335 }
336 match std::env::var("QUALIA_LLM_RESIDENT_PREFILL").ok().as_deref() {
337 Some("0") | Some("false") => false,
338 Some("1") | Some("true") => true,
339 _ => RESIDENT_PREFILL.load(Ordering::Relaxed),
340 }
341}
342
343// ── W6a: prompt-lookup speculative decode toggle (ADR 0010) ───────────────────
344// Default ON (ADR 0010, directed by Timothy). When on (and no sieve/sampler/route is active), the
345// decode loop drafts the next few tokens by n-gram prompt-lookup, verifies them in ONE batched
346// forward (`verify_draft_batch`), and emits the longest greedily-agreeing prefix + the model's own
347// correction token. Output matches ordinary decode everywhere except rare genuine near-ties (the a1a
348// phenomenon: the model is ambivalent, both tokens equally valid, and a ULP-level difference between
349// the batched and single-token forwards flips the pick). The win is pure latency on repetitive /
350// quoting / structured / code text (several tokens per forward, measured ~3–12×); on novel text it
351// drafts little and costs ~nothing.
352//
353// This is the MODE SWITCH. Change modes three ways: (1) env `QUALIA_LLM_SPEC_DECODE=0` (off) / `=1`
354// (on) at launch — the desktop/daemon reads it; (2) `set_spec_decode(bool)` at runtime (the UI /
355// host calls this); (3) `spec_decode_enabled()` to read the effective mode. The env var, when set,
356// overrides the runtime flag in BOTH directions. See ADR 0010 for the rationale.
357static SPEC_DECODE: AtomicBool = AtomicBool::new(true);
358
359/// Enable/disable prompt-lookup speculative decode (`QUALIA_LLM_SPEC_DECODE`). Runtime mode switch —
360/// the desktop UI / host calls this to flip between exact single-token decode and speculative decode.
361#[inline]
362pub fn set_spec_decode(on: bool) {
363 SPEC_DECODE.store(on, Ordering::Relaxed);
364}
365
366/// Whether the decode loop should run prompt-lookup speculative decode (the effective mode: env var
367/// wins if set, else the runtime flag). Read this to reflect the current mode in a UI.
368#[inline]
369pub fn spec_decode_enabled() -> bool {
370 if let Some(value) = super::prepared_settings::read(|settings| settings.spec_decode_enabled) {
371 return value;
372 }
373 match std::env::var("QUALIA_LLM_SPEC_DECODE").ok().as_deref() {
374 Some("0") | Some("false") => false,
375 Some("1") | Some("true") => true,
376 _ => SPEC_DECODE.load(Ordering::Relaxed),
377 }
378}
379
380// ── W5a: int8 KV cache toggle ─────────────────────────────────────────────────
381// Default ON (verified). When on (and head_dim % 4 == 0), the KV cache is stored as packed int8
382// lanes + one f32 scale per (slot, kv_head) instead of f32 — ~3.8× less KV memory (80→21 MiB @
383// ctx 1024) and ~3.8× less attention memory bandwidth (the decode bottleneck). Passed the gate on
384// SmolLM2-360M Q8: ΔPPL +0.05% (≪ the 5% bar), coherent, Vulkan-parity tok/s (see `w5a_int8_kv`).
385// Read once at model load (`ensure_kv_cache`), so set it BEFORE the model loads. Models with
386// head_dim not a multiple of 4 transparently fall back to the f32 layout. `QUALIA_LLM_KV_INT8=0`
387// forces the f32 KV cache (the A/B baseline).
388static KV_INT8: AtomicBool = AtomicBool::new(true);
389
390/// Enable/disable the int8 KV cache (`QUALIA_LLM_KV_INT8`).
391#[inline]
392pub fn set_kv_int8(on: bool) {
393 KV_INT8.store(on, Ordering::Relaxed);
394}
395
396/// Whether the KV cache should be int8-quantized.
397#[inline]
398pub fn kv_int8_enabled() -> bool {
399 if let Some(value) = super::prepared_settings::read(|settings| settings.kv_int8_enabled) {
400 return value;
401 }
402 match std::env::var("QUALIA_LLM_KV_INT8").ok().as_deref() {
403 Some("0") | Some("false") => false,
404 Some("1") | Some("true") => true,
405 _ => KV_INT8.load(Ordering::Relaxed),
406 }
407}
408
409// ── W5b Phase 4b: sparse-dictionary KV cache ──────────────────────────────────
410// Store each KV vector as its k-sparse dictionary code instead of f32/int8, reconstructing in the
411// attention shader on read (~3-4× smaller than int8). Read once at model load (`ensure_kv_cache`); a
412// certified dictionary must be installed (`kv_dict_runtime::load_certified`) whose head_dim matches the
413// model, else the layout transparently falls back. Default OFF — it trades memory for reconstruct
414// compute, so it only wins on memory-bound / long-context targets (see the Phase 4b plan).
415static KV_DICT: AtomicBool = AtomicBool::new(false);
416
417/// Enable/disable the sparse-dictionary KV cache (`QUALIA_LLM_KV_DICT`).
418#[inline]
419pub fn set_kv_dict(on: bool) {
420 KV_DICT.store(on, Ordering::Relaxed);
421}
422
423/// Whether the KV cache should use the installed sparse dictionary.
424#[inline]
425pub fn kv_dict_enabled() -> bool {
426 if let Some(value) = super::prepared_settings::read(|settings| settings.kv_dict_enabled) {
427 return value;
428 }
429 match std::env::var("QUALIA_LLM_KV_DICT").ok().as_deref() {
430 Some("0") | Some("false") => false,
431 Some("1") | Some("true") => true,
432 _ => KV_DICT.load(Ordering::Relaxed),
433 }
434}
435
436// ── W2: exact sampler config ──────────────────────────────────────────────────
437// Process-global sampler config, read ONCE at decode start (like the decode budget). `None` ⇒
438// greedy argmax (the pre-W2 default; a1a/a1c/a1d byte-identical). `Some(cfg)` with cfg.temperature
439// > 0 activates the CPU sampling chain in `crate::sampler`. Set per-request by the host/MCP layer.
440static SAMPLER_CONFIG: Mutex<Option<crate::sampler::SamplerConfig>> = Mutex::new(None);
441
442/// Install the decode sampler config (`None` restores greedy argmax).
443#[inline]
444pub fn set_sampler_config(cfg: Option<crate::sampler::SamplerConfig>) {
445 if let Ok(mut g) = SAMPLER_CONFIG.lock() {
446 // A greedy config is equivalent to None — normalize so the decode loop can skip the
447 // full-logits readback entirely when nothing non-greedy is requested.
448 *g = cfg.filter(|c| !c.is_greedy());
449 }
450}
451
452/// The active decode sampler config, if a non-greedy one is installed.
453#[inline]
454pub fn sampler_config() -> Option<crate::sampler::SamplerConfig> {
455 if let Some(value) = super::prepared_settings::read(|settings| settings.sampler_config) {
456 return value;
457 }
458 SAMPLER_CONFIG.lock().ok().and_then(|g| *g)
459}
460
461// ── R9: DOMINO constrained-decoding toggle ────────────────────────────────────
462// Process-global DOMINO masker. When active, the decode loop applies the
463// grammar constraint mask before sampling. The masker owns a token trie
464// built from the vocabulary (cold path). Set per-request by the host layer
465// when VibeScript constrained generation is requested.
466static DOMINO_MASKER: Mutex<Option<crate::inference::DominoMasker>> = Mutex::new(None);
467
468/// Install a DOMINO masker for constrained decoding. The masker must be
469/// built from the tokenizer vocabulary. Pass `None` to disable.
470pub fn set_domino_masker(masker: Option<crate::inference::DominoMasker>) {
471 if super::prepared_domino::invalidate() {
472 return;
473 }
474 if let Ok(mut g) = DOMINO_MASKER.lock() {
475 *g = masker;
476 }
477}
478
479/// Whether DOMINO constrained decoding is active.
480pub fn domino_active() -> bool {
481 if let Some(active) = super::prepared_domino::with(|grammar| {
482 grammar.as_ref().is_some_and(|grammar| grammar.active())
483 }) {
484 return active;
485 }
486 DOMINO_MASKER
487 .lock()
488 .ok()
489 .and_then(|g| g.as_ref().map(|m| m.is_active()))
490 .unwrap_or(false)
491}
492
493/// Apply the DOMINO mask to logits if a masker is installed and active.
494/// Returns the sampled token using `sample_constrained`, or `None` if no
495/// masker is installed (caller should use plain `sample`).
496pub fn domino_sample(
497 sampler: &mut crate::sampler::SamplerState,
498 logits: &mut [f32],
499 ctx: &[u32],
500) -> Option<u32> {
501 if let Some(token) = super::prepared_domino::with(|grammar| {
502 grammar
503 .as_mut()
504 .and_then(|grammar| grammar.sample(sampler, logits, ctx))
505 }) {
506 return token;
507 }
508 let mut g = DOMINO_MASKER.lock().ok()?;
509 let masker = g.as_mut()?;
510 if !masker.is_active() {
511 return None;
512 }
513 Some(sampler.sample_constrained(logits, ctx, masker))
514}
515
516/// Feed a decoded token's bytes back into the DOMINO grammar state machine
517/// so the grammar advances for the next token's mask. No-op when no masker
518/// is installed or it is inactive (T53/W11).
519///
520/// This MUST be called after every token selected under constrained decoding,
521/// otherwise the grammar state never advances and every token gets masked
522/// against the initial state.
523pub fn domino_feed_token(bytes: &[u8]) {
524 if super::prepared_domino::with(|grammar| {
525 if let Some(grammar) = grammar {
526 grammar.feed(bytes);
527 }
528 })
529 .is_some()
530 {
531 return;
532 }
533 if let Ok(mut g) = DOMINO_MASKER.lock() {
534 if let Some(masker) = g.as_mut() {
535 masker.feed_token(bytes);
536 }
537 }
538}
539
540/// Reset the DOMINO grammar state to the initial state (e.g. for a new
541/// generation turn). No-op when no masker is installed (T53/W11).
542pub fn domino_reset() {
543 if super::prepared_domino::with(|grammar| {
544 if let Some(grammar) = grammar {
545 grammar.reset();
546 }
547 })
548 .is_some()
549 {
550 return;
551 }
552 if let Ok(mut g) = DOMINO_MASKER.lock() {
553 if let Some(masker) = g.as_mut() {
554 masker.reset();
555 }
556 }
557}
558
559/// Cold request-local grammar ownership. The process template is only read;
560/// no provisional or accepted grammar mutation can alter future requests.
561pub fn prepare_domino_scope(
562 budget: crate::inference::speculative_decode::DominoPreparationBudget,
563) -> Result<super::prepared_domino::PreparedDominoScope, &'static str> {
564 let grammar = DOMINO_MASKER
565 .lock()
566 .map_err(|_| "DOMINO template poisoned")?
567 .as_ref()
568 .map(|grammar| grammar.prepare_owned(budget))
569 .transpose()?;
570 super::prepared_domino::PreparedDominoScope::enter(grammar)
571}
572
573/// Reserve private grammar capacity before a candidate can be accepted.
574pub fn domino_admit_piece(raw_bytes: usize) -> bool {
575 super::prepared_domino::with(|grammar| {
576 grammar
577 .as_mut()
578 .is_none_or(|grammar| grammar.admit_raw_piece(raw_bytes))
579 })
580 .unwrap_or(true)
581}
582
583// ── Phase 3: FFN fusion toggle ────────────────────────────────────────────────
584// Default ON (native). Runs the whole pre-norm FFN — gate GEMM, up GEMM, GPU SiLU·mul,
585// down GEMM — in ONE command submit with intermediates kept in VRAM, so a layer costs ONE
586// submit→wait round-trip instead of three (the gate/up/down readbacks + CPU SiLU·mul between
587// them). Requires resident weights (it binds resident weight buffers); falls back to the
588// per-GEMM path when resident is off or a tensor is GPU-ineligible. `QUALIA_LLM_FFN_FUSION=0`
589// forces the per-GEMM path (the A/B OFF baseline).
590static FFN_FUSION: AtomicBool = AtomicBool::new(true);
591
592/// Enable/disable the fused single-submit FFN path (`QUALIA_LLM_FFN_FUSION`).
593#[inline]
594pub fn set_ffn_fusion(on: bool) {
595 FFN_FUSION.store(on, Ordering::Relaxed);
596}
597
598/// Whether the native FFN should run fused (one submit/layer) rather than three GEMM round-trips.
599#[inline]
600pub fn ffn_fusion_enabled() -> bool {
601 if let Some(value) = super::prepared_settings::read(|settings| settings.ffn_fusion_enabled) {
602 return value;
603 }
604 match std::env::var("QUALIA_LLM_FFN_FUSION").ok().as_deref() {
605 Some("0") | Some("false") => false,
606 Some("1") | Some("true") => true,
607 _ => FFN_FUSION.load(Ordering::Relaxed),
608 }
609}
610
611/// Set when a resident decode plan was built with `fused_ffn.wgsl` in the mega-pass (T-A1).
612static FFN_FUSION_IN_RESIDENT: AtomicBool = AtomicBool::new(false);
613
614#[inline]
615/// Sets `ffn-fusion-in-resident`.
616pub fn set_ffn_fusion_in_resident(on: bool) {
617 FFN_FUSION_IN_RESIDENT.store(on, Ordering::Relaxed);
618}
619
620/// True when the last-built resident plan wires fused FFN expansion (not just the flag).
621#[inline]
622pub fn ffn_fusion_in_resident() -> bool {
623 FFN_FUSION_IN_RESIDENT.load(Ordering::Relaxed)
624}
625
626// ── FFN quant → f16 promotion (opt-in; bandwidth vs dequant trade-off) ─
627// Default OFF. Microbench on small GEMMs favoured f16, but full Llama-3.2-3B on
628// A2000 12GB measured **slower** with FFN f16 (~2.1 tok/s) than Q4_K SoA (~2.6)
629// — 4× weight traffic outweighs dequant savings when memory-bound. Opt in with
630// `QUALIA_LLM_FFN_F16=1` on higher-bandwidth GPUs / smaller FFN dims.
631static FFN_F16: AtomicBool = AtomicBool::new(false);
632
633/// Enable/disable FFN quant→f16 promotion at resident-plan build (`QUALIA_LLM_FFN_F16`).
634#[inline]
635pub fn set_ffn_f16(on: bool) {
636 FFN_F16.store(on, Ordering::Relaxed);
637}
638
639/// Whether FFN weights should be promoted to f16 in VRAM for decode/prefill GEMV.
640#[inline]
641pub fn ffn_f16_enabled() -> bool {
642 if let Some(value) = super::prepared_settings::read(|settings| settings.ffn_f16_enabled) {
643 return value;
644 }
645 match std::env::var("QUALIA_LLM_FFN_F16").ok().as_deref() {
646 Some("0") | Some("false") => false,
647 Some("1") | Some("true") => true,
648 _ => FFN_F16.load(Ordering::Relaxed),
649 }
650}
651
652// ── 0.0.21: cooperative GEMV kernel toggle ────────────────────────────────────
653// Default ON (native), verified. Routes native GEMV work through the cooperative
654// one-workgroup-per-row kernel (`coop_gemv`: coalesced reads + per-thread dequant +
655// shared-memory reduction) instead of the naive 1-thread/row `main`. The fused FFN path also selects
656// this cooperative entry point for gate/up/down GEMMs, so decode keeps one FFN readback per layer
657// while using the faster row reducer. The naive GEMV is the measured decode bottleneck
658// (compute/ALU-bound: uncoalesced strided reads + serial accumulate; it also makes Q4_K slower than
659// F16). `QUALIA_LLM_COOP_GEMV=0` forces the naive kernel (the A/B OFF baseline).
660// Earlier A2000 / Llama-3.2-3B-F16 per-GEMM verification: 2.39→3.22 tok/s (+35% over naive).
661static COOP_GEMV: AtomicBool = AtomicBool::new(true);
662
663/// Enable/disable the cooperative GEMV decode path (`QUALIA_LLM_COOP_GEMV`).
664#[inline]
665pub fn set_coop_gemv(on: bool) {
666 COOP_GEMV.store(on, Ordering::Relaxed);
667}
668
669/// Whether native GEMM should run the cooperative `coop_gemv` kernel rather than the naive
670/// 1-thread/row `main`. Env forces either direction; otherwise the atomic flag (default OFF).
671#[inline]
672pub fn coop_gemv_enabled() -> bool {
673 if let Some(value) = super::prepared_settings::read(|settings| settings.coop_gemv_enabled) {
674 return value;
675 }
676 match std::env::var("QUALIA_LLM_COOP_GEMV").ok().as_deref() {
677 Some("0") | Some("false") => false,
678 Some("1") | Some("true") => true,
679 _ => COOP_GEMV.load(Ordering::Relaxed),
680 }
681}
682
683/// Rows per workgroup for multi-row coop GEMV (`coop_gemv_mr` in fused_transformer.wgsl).
684/// Must stay in lock-step with WGSL `COOP_ROWS`.
685pub const COOP_GEMV_ROWS: u32 = 8;
686
687/// Workgroup count for coop GEMV dispatch: `ceil(n_out / COOP_GEMV_ROWS)`.
688#[inline]
689pub fn coop_gemv_workgroups(n_out: u32) -> u32 {
690 n_out.div_ceil(COOP_GEMV_ROWS).max(1)
691}
692
693// ── W8: coopmat (tensor-core) GEMM selection seam ─────────────────────────────
694// Default OFF. The forge already has the self-activating coopmat path
695// (`wgsl_forge::gemm_f32_tc` → `coopmat_usable()` runtime probe): on wgpu 29.0.3 the WGSL coopmat
696// multiply returns zeros (#9741), so `coopmat_usable()` is `false` and the tier stays dormant; it
697// self-activates the moment a wgpu release / soft-fork carries the fix. This toggle is the
698// INFERENCE-side seam: when on AND coopmat is genuinely usable AND the matmul dims fit the 8×8×8 tile
699// (m,n,k multiples of 8 — so batched prefill, not the m=1 decode GEMV), the GEMM backend selector
700// reports `Coopmat`. Until the inference-side coopmat kernel is wired (which needs the wgpu fix), an
701// eligible `Coopmat` selection logs its readiness and falls back to `CoopGemv` — the plumbing is ready
702// and visible, self-activating with the forge. `QUALIA_LLM_COOPMAT=1` arms it.
703static COOPMAT_GEMM: AtomicBool = AtomicBool::new(false);
704
705/// Arm/disarm the coopmat (tensor-core) GEMM selection seam (`QUALIA_LLM_COOPMAT`).
706#[inline]
707pub fn set_coopmat_gemm(on: bool) {
708 COOPMAT_GEMM.store(on, Ordering::Relaxed);
709}
710
711/// Whether the coopmat GEMM seam is armed (env forces either direction; else the flag). Being armed
712/// does not mean coopmat runs — see [`coopmat_gemm_usable`], which also requires the hardware probe.
713#[inline]
714pub fn coopmat_gemm_enabled() -> bool {
715 if let Some(value) = super::prepared_settings::read(|settings| settings.coopmat_gemm_enabled) {
716 return value;
717 }
718 match std::env::var("QUALIA_LLM_COOPMAT").ok().as_deref() {
719 Some("0") | Some("false") => false,
720 Some("1") | Some("true") => true,
721 _ => COOPMAT_GEMM.load(Ordering::Relaxed),
722 }
723}
724
725/// Whether coopmat is BOTH armed and genuinely usable on this device right now (the forge runtime
726/// probe passes — i.e. the wgpu #9741 fix is present). `false` on wgpu 29.0.3. Feature-guarded: with
727/// `wgsl-forge` off there is no probe, so coopmat is never usable.
728#[inline]
729pub fn coopmat_gemm_usable() -> bool {
730 if !coopmat_gemm_enabled() {
731 return false;
732 }
733 #[cfg(all(feature = "wgsl-forge", not(target_arch = "wasm32")))]
734 {
735 crate::wgsl_forge::coopmat_usable()
736 }
737 #[cfg(not(all(feature = "wgsl-forge", not(target_arch = "wasm32"))))]
738 {
739 false
740 }
741}
742
743/// GEMM backend the inference layer selects for a given matmul shape.
744#[derive(Debug, Clone, Copy, PartialEq, Eq)]
745pub enum GemmBackend {
746 /// Naive 1-thread/row `main` kernel (the correctness floor / A-B baseline).
747 Naive,
748 /// Cooperative one-workgroup-per-row `coop_gemv` (coalesced reads + shared-mem reduce).
749 CoopGemv,
750 /// WGSL cooperative-matrix (tensor-core) tile — selected only when armed, usable, and the dims
751 /// fit the 8×8×8 tile. Self-activates via the forge when wgpu #9741 lands.
752 Coopmat,
753}
754
755/// Select the GEMM backend for an `m×k×n` matmul: coopmat when armed+usable and all dims are 8-mult
756/// (the tensor-core tile — batched prefill, not the m=1 decode GEMV); else cooperative GEMV when
757/// enabled; else naive. Pure + total, so it is unit-tested without a GPU.
758#[inline]
759pub fn select_gemm_backend(m: usize, k: usize, n: usize) -> GemmBackend {
760 let tile_fits = m % 8 == 0 && k % 8 == 0 && n % 8 == 0 && m.min(k).min(n) > 0;
761 if coopmat_gemm_usable() && tile_fits {
762 GemmBackend::Coopmat
763 } else if coop_gemv_enabled() {
764 GemmBackend::CoopGemv
765 } else {
766 GemmBackend::Naive
767 }
768}
769
770// ── 0.0.21: resident-activation decode toggle ─────────────────────────────────
771// readback happens, after the final layer — replacing the legacy 2 readbacks/layer (each forced by
772// ── #48 correctness path: CPU attention reference ─────────────────────────────
773// Route native attention through the wasm-proven CPU SDPA (`cpu_attention_pass`) instead of the
774// GPU attention shader (whose output is currently unbounded). Correct-but-slower; opt-in.
775static CPU_ATTENTION: AtomicBool = AtomicBool::new(false);
776
777/// Enable/disable the native CPU-attention reference path (`QUALIA_LLM_CPU_ATTENTION`).
778#[inline]
779pub fn set_cpu_attention(on: bool) {
780 CPU_ATTENTION.store(on, Ordering::Relaxed);
781}
782
783/// Whether native attention should use the CPU reference.
784///
785/// **Default OFF** (use the GPU attention path) — as of #49 the GPU path also honors `norm_weight`
786/// for prefill K/V and produces coherent output, and it is faster. The CPU SDPA reference remains
787/// available as a correctness fallback / cross-check via `QUALIA_LLM_CPU_ATTENTION=1` or
788/// [`set_cpu_attention`].
789#[inline]
790pub fn cpu_attention_enabled() -> bool {
791 if let Some(value) = super::prepared_settings::read(|settings| settings.cpu_attention_enabled) {
792 return value;
793 }
794 CPU_ATTENTION.load(Ordering::Relaxed)
795 || matches!(
796 std::env::var("QUALIA_LLM_CPU_ATTENTION").ok().as_deref(),
797 Some("1") | Some("true")
798 )
799}
800
801// ── Prefix cache enabled toggle ───────────────────────────────────────────────
802static PREFIX_CACHE_ENABLED: AtomicBool = AtomicBool::new(true);
803
804#[inline]
805/// Sets the prefix-cache flag (atomic) — F22: previously a no-op whose getter always read `true`.
806pub fn set_prefix_cache_enabled(on: bool) {
807 PREFIX_CACHE_ENABLED.store(on, Ordering::Relaxed);
808}
809
810#[inline]
811/// Current prefix-cache flag — real atomic read since F22 fix.
812pub fn prefix_cache_enabled() -> bool {
813 if let Some(value) = super::prepared_settings::read(|settings| settings.prefix_cache_enabled) {
814 return value;
815 }
816 PREFIX_CACHE_ENABLED.load(Ordering::Relaxed)
817}
818
819// ── KV pool budget override in MB (0 = default) ──────────────────────────────
820static KV_POOL_BUDGET_MB: AtomicU32 = AtomicU32::new(0);
821
822#[inline]
823/// Sets the KV-pool budget in MiB.
824pub fn set_kv_pool_budget_mb(mb: u32) {
825 KV_POOL_BUDGET_MB.store(mb, Ordering::Relaxed);
826}
827
828#[inline]
829/// Configured KV-pool budget in MiB.
830pub fn kv_pool_budget_mb() -> u32 {
831 if let Some(value) = super::prepared_settings::read(|settings| settings.kv_pool_budget_mb) {
832 return value;
833 }
834 KV_POOL_BUDGET_MB.load(Ordering::Relaxed)
835}
836
837// ── Prefill chunk size override (0 = default PREFILL_CHUNK_SIZE) ──────────────
838static PREFILL_CHUNK_SIZE_OVERRIDE: AtomicU32 = AtomicU32::new(0);
839
840#[inline]
841/// Sets the prefill chunk-size override.
842pub fn set_prefill_chunk_size_override(sz: u32) {
843 PREFILL_CHUNK_SIZE_OVERRIDE.store(sz, Ordering::Relaxed);
844}
845
846#[inline]
847/// Current prefill chunk-size override.
848pub fn prefill_chunk_size_override() -> u32 {
849 if let Some(value) =
850 super::prepared_settings::read(|settings| settings.prefill_chunk_size_override)
851 {
852 return value;
853 }
854 PREFILL_CHUNK_SIZE_OVERRIDE.load(Ordering::Relaxed)
855}
856
857// ── Prompt precision mode setting ─────────────────────────────────────────────
858static PROMPT_PRECISION_MODE: Mutex<String> = Mutex::new(String::new());
859
860/// Sets the prompt precision mode string.
861pub fn set_prompt_precision_mode(mode: &str) {
862 if let Ok(mut m) = PROMPT_PRECISION_MODE.lock() {
863 m.clear();
864 m.push_str(mode);
865 }
866}
867
868/// Current prompt precision mode.
869pub fn prompt_precision_mode() -> String {
870 if let Some(value) =
871 super::prepared_settings::read(|settings| settings.prompt_precision_mode.clone())
872 {
873 return value;
874 }
875 PROMPT_PRECISION_MODE
876 .lock()
877 .map(|m| m.clone())
878 .unwrap_or_default()
879}
880
881// ── Lifecycle stage telemetry (inference-lifecycle.n3 `stage:*`) ─────────────
882// Per-turn StageTrace recording inside the live decode loop. Recording is a
883// fixed 16-slot stack structure coalesced per stage kind — no hot allocations.
884// Default ON; `QUALIA_STAGE_TELEMETRY=0` disables the instrumentation entirely.
885static STAGE_TELEMETRY: AtomicBool = AtomicBool::new(true);
886
887/// Enable/disable per-turn lifecycle stage telemetry (`QUALIA_STAGE_TELEMETRY`).
888#[inline]
889pub fn set_stage_telemetry(on: bool) {
890 STAGE_TELEMETRY.store(on, Ordering::Relaxed);
891}
892
893/// Whether lifecycle stage telemetry is active. The env var overrides the flag
894/// in BOTH directions (`0`/`false` → off, `1`/`true` → on); otherwise the
895/// process default (ON) applies.
896#[inline]
897pub fn stage_telemetry_enabled() -> bool {
898 if let Some(value) = super::prepared_settings::read(|settings| settings.stage_telemetry_enabled)
899 {
900 return value;
901 }
902 match std::env::var("QUALIA_STAGE_TELEMETRY").ok().as_deref() {
903 Some("0") | Some("false") => false,
904 Some("1") | Some("true") => true,
905 _ => STAGE_TELEMETRY.load(Ordering::Relaxed),
906 }
907}