Skip to main content

qualia_core_db/inference/inference_bench/
toggles.rs

1//! Runtime-config toggles read by the real inference path (decode / prefill /
2//! attention / GEMM selection). Each is a process-global flag with a `set_*` /
3//! `*_enabled` pair; env vars override where documented. Also the GEMM backend
4//! selector. Pure code motion — behaviour unchanged.
5
6use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
7use std::sync::Mutex;
8
9// ── Decode budget override ────────────────────────────────────────────────────
10// A bounded, fixed decode-token count gives stable, comparable tok/s. The real
11// decode loop reads this once per call; 0 = use the production `DECODE_TOKEN_BUDGET`.
12
13static DECODE_BUDGET_OVERRIDE: AtomicU32 = AtomicU32::new(0);
14
15// Application-profile bootstrap is intentionally allowed to reset the normal
16// override to its production default.  A benchmark is different: it requires a
17// fixed, reproducible budget for its complete cold/warm scope.  Keep that
18// request separate so profile startup in the inference thread cannot silently
19// turn a requested 64-token measurement back into the 256-token UI default.
20static BENCHMARK_DECODE_BUDGET_OVERRIDE: AtomicU32 = AtomicU32::new(0);
21static BENCHMARK_DECODE_SCOPE: Mutex<()> = Mutex::new(());
22
23/// Serializes and installs a benchmark-only decode budget.
24///
25/// The returned guard restores the production behaviour when dropped.  This
26/// is process-global because the underlying inference toggles are global too;
27/// concurrent benchmarks are therefore deliberately serialized.
28pub struct BenchmarkDecodeBudgetGuard {
29    _scope: std::sync::MutexGuard<'static, ()>,
30}
31
32impl Drop for BenchmarkDecodeBudgetGuard {
33    fn drop(&mut self) {
34        BENCHMARK_DECODE_BUDGET_OVERRIDE.store(0, Ordering::Release);
35    }
36}
37
38/// RAII guard installing a temporary decode budget; restores the previous value on drop.
39pub fn benchmark_decode_budget_scope(n: u32) -> BenchmarkDecodeBudgetGuard {
40    let scope = BENCHMARK_DECODE_SCOPE
41        .lock()
42        .unwrap_or_else(|poisoned| poisoned.into_inner());
43    BENCHMARK_DECODE_BUDGET_OVERRIDE.store(n, Ordering::Release);
44    BenchmarkDecodeBudgetGuard { _scope: scope }
45}
46
47/// Set a fixed decode-token budget for benchmarking (0 = production default).
48#[inline]
49pub fn set_decode_budget_override(n: u32) {
50    DECODE_BUDGET_OVERRIDE.store(n, Ordering::Relaxed);
51}
52
53/// When budget override is active, ignore EOS so A/B runs a fixed token count
54/// (prevents early-stop from inflating/deflating tok/s on short prompts).
55#[inline]
56pub fn decode_budget_fixed_tokens() -> bool {
57    decode_budget_override() > 0
58}
59
60/// Current decode-budget override (0 = none).
61#[inline]
62pub fn decode_budget_override() -> u32 {
63    if let Some(value) = super::prepared_settings::read(|settings| settings.decode_budget_override)
64    {
65        return value;
66    }
67    let benchmark = BENCHMARK_DECODE_BUDGET_OVERRIDE.load(Ordering::Acquire);
68    if benchmark > 0 {
69        benchmark
70    } else {
71        DECODE_BUDGET_OVERRIDE.load(Ordering::Relaxed)
72    }
73}
74
75// ── Wall-clock inference timeout override (batch / overnight jobs) ────────────
76// 0 = use production `INFERENCE_TIMEOUT_MS` (30s interactive). Batch profile may
77// raise this to hours for multi-system health differential analysis overnight.
78static INFERENCE_TIMEOUT_OVERRIDE_MS: AtomicU64 = AtomicU64::new(0);
79
80/// Set wall-clock decode timeout in ms (0 = production default 30_000).
81#[inline]
82pub fn set_inference_timeout_override_ms(ms: u64) {
83    INFERENCE_TIMEOUT_OVERRIDE_MS.store(ms, Ordering::Relaxed);
84}
85
86/// Effective timeout: override if non-zero, else env `QUALIA_INFERENCE_TIMEOUT_MS`, else 30s.
87#[inline]
88pub fn inference_timeout_ms() -> u64 {
89    if let Some(value) = super::prepared_settings::read(|settings| settings.inference_timeout_ms) {
90        return value;
91    }
92    let o = INFERENCE_TIMEOUT_OVERRIDE_MS.load(Ordering::Relaxed);
93    if o > 0 {
94        return o;
95    }
96    if let Ok(s) = std::env::var("QUALIA_INFERENCE_TIMEOUT_MS") {
97        if let Ok(v) = s.parse::<u64>() {
98            if v > 0 {
99                return v;
100            }
101        }
102    }
103    30_000
104}
105
106// ── A1a GPU top-k toggle (D18) ────────────────────────────────────────────────
107// Default ON after widening the output gate and adding the allocation-free top-1 path. The decode
108// loop reads only block winners from the GPU instead of full vocabulary chunks when no sieve mask is
109// active.
110// Set `QUALIA_LLM_GPU_TOPK=0` to force the full-logit argmax fallback.
111static GPU_TOPK: AtomicBool = AtomicBool::new(true);
112
113/// Enable/disable the GPU top-k decode path (`QUALIA_LLM_GPU_TOPK`).
114#[inline]
115pub fn set_gpu_topk(on: bool) {
116    GPU_TOPK.store(on, Ordering::Relaxed);
117}
118
119/// Whether the GPU top-k decode path is active. The env var overrides the flag in BOTH directions
120/// (`0`/`false` → off, `1`/`true` → on); otherwise the process default (ON) applies.
121#[inline]
122pub fn gpu_topk_enabled() -> bool {
123    if let Some(value) = super::prepared_settings::read(|settings| settings.gpu_topk_enabled) {
124        return value;
125    }
126    match std::env::var("QUALIA_LLM_GPU_TOPK").ok().as_deref() {
127        Some("0") | Some("false") => false,
128        Some("1") | Some("true") => true,
129        _ => GPU_TOPK.load(Ordering::Relaxed),
130    }
131}
132
133// ── A1b ternary-FFN toggle (D3/D7) ────────────────────────────────────────────
134// Additive, default-OFF: when a `.q42` ternary container is booted, routes its FFN
135// GEMMs through the resident 2-bit GPU kernel (`TernaryFfnResident`). OFF runs the
136// SAME ternary weights via the CPU oracle — so ON-vs-OFF isolates the GPU-kernel win
137// on identical weights, and ternary-container-vs-Q8 (a0) is the headline FFN number.
138static TERNARY_FFN: AtomicBool = AtomicBool::new(false);
139
140/// Enable/disable the resident 2-bit GPU ternary-FFN path (`QUALIA_LLM_TERNARY_FFN`).
141#[inline]
142pub fn set_ternary_ffn(on: bool) {
143    TERNARY_FFN.store(on, Ordering::Relaxed);
144}
145
146/// Whether the GPU ternary-FFN path is active (atomic flag OR the env var). When false, ternary
147/// FFN GEMMs fall back to the CPU oracle (correct, slower) — the toggle's OFF baseline.
148#[inline]
149pub fn ternary_ffn_enabled() -> bool {
150    if let Some(value) = super::prepared_settings::read(|settings| settings.ternary_ffn_enabled) {
151        return value;
152    }
153    TERNARY_FFN.load(Ordering::Relaxed)
154        || matches!(
155            std::env::var("QUALIA_LLM_TERNARY_FFN").ok().as_deref(),
156            Some("1") | Some("true")
157        )
158}
159
160// Native attention projection split: K/V matmuls run through cooperative GEMV, then the attention
161// shader consumes the projected rows via `proj_row_stride` only for RoPE + KV-cache writes. Default
162// ON after the no-readback fused K/V path replaced the old diagnostic readback implementation.
163// Set `QUALIA_LLM_PREPROJECT_ATTN=0` to force the legacy in-attention projection path.
164static ATTN_PREPROJECT: AtomicBool = AtomicBool::new(true);
165
166#[inline]
167/// Enables/disables the attention pre-projection bench toggle (atomic).
168pub fn set_attention_preproject(on: bool) {
169    ATTN_PREPROJECT.store(on, Ordering::Relaxed);
170}
171
172#[inline]
173/// Current attention pre-projection toggle state.
174pub fn attention_preproject_enabled() -> bool {
175    if let Some(value) =
176        super::prepared_settings::read(|settings| settings.attention_preproject_enabled)
177    {
178        return value;
179    }
180    // W5b Phase 4b: the dict-coded write happens in `write_kv_head` (the attention pass); the fused
181    // pre-projection bypasses it, so force it off in dict mode.
182    if kv_dict_enabled() {
183        return false;
184    }
185    match std::env::var("QUALIA_LLM_PREPROJECT_ATTN").ok().as_deref() {
186        Some("0") | Some("false") => false,
187        Some("1") | Some("true") => true,
188        _ => ATTN_PREPROJECT.load(Ordering::Relaxed),
189    }
190}
191
192// Native attention tail fusion: Q-attention writes its output to a GPU buffer, and o_proj consumes
193// that buffer directly after K and V are both present in the KV cache. Default ON (native): removes
194// one submit->wait round-trip per layer while preserving token identity against the proven readback
195// path. Set `QUALIA_LLM_FUSE_ATTN_O=0` to force the older Q-readback + o_proj path.
196static ATTN_O_FUSE: AtomicBool = AtomicBool::new(true);
197
198#[inline]
199/// Enables/disables the attention-o fusion bench toggle.
200pub fn set_attention_o_fuse(on: bool) {
201    ATTN_O_FUSE.store(on, Ordering::Relaxed);
202}
203
204#[inline]
205/// Current attention-o fusion state.
206pub fn attention_o_fuse_enabled() -> bool {
207    if let Some(value) =
208        super::prepared_settings::read(|settings| settings.attention_o_fuse_enabled)
209    {
210        return value;
211    }
212    // W5b Phase 4b: the fused Q+O tail reads K/V from the cache on the GPU path; keep the plain
213    // read_k/read_v (dict-aware) path in dict mode.
214    if kv_dict_enabled() {
215        return false;
216    }
217    match std::env::var("QUALIA_LLM_FUSE_ATTN_O").ok().as_deref() {
218        Some("0") | Some("false") => false,
219        Some("1") | Some("true") => true,
220        _ => ATTN_O_FUSE.load(Ordering::Relaxed),
221    }
222}
223
224// Certified Forge WGSL attention: dispatches Naga-validated Forge causal attention
225// directly on wgpu device during decode step.
226static FORGE_ATTENTION: AtomicBool = AtomicBool::new(false);
227
228#[inline]
229/// Enables/disables the forge-attention bench toggle.
230pub fn set_forge_attention(on: bool) {
231    FORGE_ATTENTION.store(on, Ordering::Relaxed);
232}
233
234#[inline]
235/// Current forge-attention state.
236pub fn forge_attention_enabled() -> bool {
237    if let Some(value) = super::prepared_settings::read(|settings| settings.forge_attention_enabled)
238    {
239        return value;
240    }
241    match std::env::var("QUALIA_LLM_FORGE_ATTENTION").ok().as_deref() {
242        Some("1") | Some("true") => true,
243        Some("0") | Some("false") => false,
244        _ => FORGE_ATTENTION.load(Ordering::Relaxed),
245    }
246}
247
248// ── Phase 2: resident weights toggle ──────────────────────────────────────────
249// Default ON (native). Each layer's q/k/v/o/gate/up/down weight is uploaded to its own resident
250// VRAM buffer once (keyed by the GGUF tensor byte_offset) and reused every token, instead of
251// re-`write_buffer`ing the (up to ~50 MB for a 3B FFN tensor) weight into the shared GEMM buffer
252// on every GEMM, every token. For a 3B F16 model that re-upload is ~5 GB/token of PCIe traffic —
253// the decode bottleneck. Set `QUALIA_LLM_RESIDENT_WEIGHTS=0` to force the per-token re-upload (the
254// A/B OFF baseline) — useful for measuring the win or on VRAM-constrained GPUs.
255static RESIDENT_WEIGHTS: AtomicBool = AtomicBool::new(true);
256
257/// Enable/disable the resident per-tensor weight buffers (`QUALIA_LLM_RESIDENT_WEIGHTS`).
258#[inline]
259pub fn set_resident_weights(on: bool) {
260    RESIDENT_WEIGHTS.store(on, Ordering::Relaxed);
261}
262
263/// Whether native GEMM should bind resident per-tensor weight buffers (upload-once) rather than
264/// re-uploading the weight every token. Env forces either direction; otherwise the atomic flag.
265#[inline]
266pub fn resident_weights_enabled() -> bool {
267    if let Some(value) =
268        super::prepared_settings::read(|settings| settings.resident_weights_enabled)
269    {
270        return value;
271    }
272    match std::env::var("QUALIA_LLM_RESIDENT_WEIGHTS").ok().as_deref() {
273        Some("0") | Some("false") => false,
274        Some("1") | Some("true") => true,
275        _ => RESIDENT_WEIGHTS.load(Ordering::Relaxed),
276    }
277}
278
279// ── Resident-token decode toggle (single fence per token) ─────────────────────
280// Default ON (native). Keeps the hidden state in VRAM for the WHOLE token: all 32 layers
281// (RMSNorm/residuals as GPU elem ops) + output norm + chunked logits top-1 are encoded into ONE
282// command submit with ONE blocking fence and a ~400 B candidate readback — replacing the legacy
283// ~107 submit→wait round-trips/token (measured ~24% pure fence time on SmolLM2-360M, A2000,
284// Vulkan). Any per-model ineligibility (unsupported quant, no resident logits, sieve mask, CPU
285// attention) falls back to the legacy per-layer path. `QUALIA_LLM_RESIDENT_DECODE=0` forces the
286// legacy path (the A/B baseline + the differential-test comparator).
287static RESIDENT_DECODE: AtomicBool = AtomicBool::new(true);
288
289/// Enable/disable the resident-token single-fence decode (`QUALIA_LLM_RESIDENT_DECODE`).
290#[inline]
291pub fn set_resident_decode(on: bool) {
292    RESIDENT_DECODE.store(on, Ordering::Relaxed);
293}
294
295/// Whether native decode should run the GPU-resident single-fence token path.
296#[inline]
297pub fn resident_decode_enabled() -> bool {
298    if let Some(value) = super::prepared_settings::read(|settings| settings.resident_decode_enabled)
299    {
300        return value;
301    }
302    match std::env::var("QUALIA_LLM_RESIDENT_DECODE").ok().as_deref() {
303        Some("0") | Some("false") => false,
304        Some("1") | Some("true") => true,
305        _ => RESIDENT_DECODE.load(Ordering::Relaxed),
306    }
307}
308
309// ── W3: resident single-fence-per-chunk prefill toggle ────────────────────────
310// Default ON (verified). When on (and the model is eligible — GPU-eligible weights, coop GEMV, no
311// active sparse-attention route), each prefill chunk of ≤PREFILL_CHUNK_SIZE prompt tokens populates
312// the KV cache in ONE command submit / ONE fence (all 32 layers batched + resident hidden state)
313// instead of the legacy per-layer + per-token Q/FFN loop (~640 submit→wait round-trips for a 10-token
314// prompt). Delivers TTFT and the batched-forward primitive W6a-verify needs; on a fast discrete GPU
315// the steady-state win is latent (prefill is compute-bound), the fence win lands on
316// edge/mobile/under-load. Passed the `a3a` gate on SmolLM2-360M Q8 (A2000): the batched RMSNorm
317// reduces in the same sequential order as the legacy CPU path, so the KV it writes is BYTE-IDENTICAL
318// → decode-identical, with the int8 KV cache both ON and OFF. Any ineligibility falls back to the
319// legacy `dispatch_prefill_chunk` path unchanged. `QUALIA_LLM_RESIDENT_PREFILL=0` forces legacy.
320static RESIDENT_PREFILL: AtomicBool = AtomicBool::new(true);
321
322/// Enable/disable the resident single-fence-per-chunk prefill path (`QUALIA_LLM_RESIDENT_PREFILL`).
323#[inline]
324pub fn set_resident_prefill(on: bool) {
325    RESIDENT_PREFILL.store(on, Ordering::Relaxed);
326}
327
328/// Whether native prefill should run the GPU-resident single-fence-per-chunk arena.
329#[inline]
330pub fn resident_prefill_enabled() -> bool {
331    if let Some(value) =
332        super::prepared_settings::read(|settings| settings.resident_prefill_enabled)
333    {
334        return value;
335    }
336    match std::env::var("QUALIA_LLM_RESIDENT_PREFILL").ok().as_deref() {
337        Some("0") | Some("false") => false,
338        Some("1") | Some("true") => true,
339        _ => RESIDENT_PREFILL.load(Ordering::Relaxed),
340    }
341}
342
343// ── W6a: prompt-lookup speculative decode toggle (ADR 0010) ───────────────────
344// Default ON (ADR 0010, directed by Timothy). When on (and no sieve/sampler/route is active), the
345// decode loop drafts the next few tokens by n-gram prompt-lookup, verifies them in ONE batched
346// forward (`verify_draft_batch`), and emits the longest greedily-agreeing prefix + the model's own
347// correction token. Output matches ordinary decode everywhere except rare genuine near-ties (the a1a
348// phenomenon: the model is ambivalent, both tokens equally valid, and a ULP-level difference between
349// the batched and single-token forwards flips the pick). The win is pure latency on repetitive /
350// quoting / structured / code text (several tokens per forward, measured ~3–12×); on novel text it
351// drafts little and costs ~nothing.
352//
353// This is the MODE SWITCH. Change modes three ways: (1) env `QUALIA_LLM_SPEC_DECODE=0` (off) / `=1`
354// (on) at launch — the desktop/daemon reads it; (2) `set_spec_decode(bool)` at runtime (the UI /
355// host calls this); (3) `spec_decode_enabled()` to read the effective mode. The env var, when set,
356// overrides the runtime flag in BOTH directions. See ADR 0010 for the rationale.
357static SPEC_DECODE: AtomicBool = AtomicBool::new(true);
358
359/// Enable/disable prompt-lookup speculative decode (`QUALIA_LLM_SPEC_DECODE`). Runtime mode switch —
360/// the desktop UI / host calls this to flip between exact single-token decode and speculative decode.
361#[inline]
362pub fn set_spec_decode(on: bool) {
363    SPEC_DECODE.store(on, Ordering::Relaxed);
364}
365
366/// Whether the decode loop should run prompt-lookup speculative decode (the effective mode: env var
367/// wins if set, else the runtime flag). Read this to reflect the current mode in a UI.
368#[inline]
369pub fn spec_decode_enabled() -> bool {
370    if let Some(value) = super::prepared_settings::read(|settings| settings.spec_decode_enabled) {
371        return value;
372    }
373    match std::env::var("QUALIA_LLM_SPEC_DECODE").ok().as_deref() {
374        Some("0") | Some("false") => false,
375        Some("1") | Some("true") => true,
376        _ => SPEC_DECODE.load(Ordering::Relaxed),
377    }
378}
379
380// ── W5a: int8 KV cache toggle ─────────────────────────────────────────────────
381// Default ON (verified). When on (and head_dim % 4 == 0), the KV cache is stored as packed int8
382// lanes + one f32 scale per (slot, kv_head) instead of f32 — ~3.8× less KV memory (80→21 MiB @
383// ctx 1024) and ~3.8× less attention memory bandwidth (the decode bottleneck). Passed the gate on
384// SmolLM2-360M Q8: ΔPPL +0.05% (≪ the 5% bar), coherent, Vulkan-parity tok/s (see `w5a_int8_kv`).
385// Read once at model load (`ensure_kv_cache`), so set it BEFORE the model loads. Models with
386// head_dim not a multiple of 4 transparently fall back to the f32 layout. `QUALIA_LLM_KV_INT8=0`
387// forces the f32 KV cache (the A/B baseline).
388static KV_INT8: AtomicBool = AtomicBool::new(true);
389
390/// Enable/disable the int8 KV cache (`QUALIA_LLM_KV_INT8`).
391#[inline]
392pub fn set_kv_int8(on: bool) {
393    KV_INT8.store(on, Ordering::Relaxed);
394}
395
396/// Whether the KV cache should be int8-quantized.
397#[inline]
398pub fn kv_int8_enabled() -> bool {
399    if let Some(value) = super::prepared_settings::read(|settings| settings.kv_int8_enabled) {
400        return value;
401    }
402    match std::env::var("QUALIA_LLM_KV_INT8").ok().as_deref() {
403        Some("0") | Some("false") => false,
404        Some("1") | Some("true") => true,
405        _ => KV_INT8.load(Ordering::Relaxed),
406    }
407}
408
409// ── W5b Phase 4b: sparse-dictionary KV cache ──────────────────────────────────
410// Store each KV vector as its k-sparse dictionary code instead of f32/int8, reconstructing in the
411// attention shader on read (~3-4× smaller than int8). Read once at model load (`ensure_kv_cache`); a
412// certified dictionary must be installed (`kv_dict_runtime::load_certified`) whose head_dim matches the
413// model, else the layout transparently falls back. Default OFF — it trades memory for reconstruct
414// compute, so it only wins on memory-bound / long-context targets (see the Phase 4b plan).
415static KV_DICT: AtomicBool = AtomicBool::new(false);
416
417/// Enable/disable the sparse-dictionary KV cache (`QUALIA_LLM_KV_DICT`).
418#[inline]
419pub fn set_kv_dict(on: bool) {
420    KV_DICT.store(on, Ordering::Relaxed);
421}
422
423/// Whether the KV cache should use the installed sparse dictionary.
424#[inline]
425pub fn kv_dict_enabled() -> bool {
426    if let Some(value) = super::prepared_settings::read(|settings| settings.kv_dict_enabled) {
427        return value;
428    }
429    match std::env::var("QUALIA_LLM_KV_DICT").ok().as_deref() {
430        Some("0") | Some("false") => false,
431        Some("1") | Some("true") => true,
432        _ => KV_DICT.load(Ordering::Relaxed),
433    }
434}
435
436// ── W2: exact sampler config ──────────────────────────────────────────────────
437// Process-global sampler config, read ONCE at decode start (like the decode budget). `None` ⇒
438// greedy argmax (the pre-W2 default; a1a/a1c/a1d byte-identical). `Some(cfg)` with cfg.temperature
439// > 0 activates the CPU sampling chain in `crate::sampler`. Set per-request by the host/MCP layer.
440static SAMPLER_CONFIG: Mutex<Option<crate::sampler::SamplerConfig>> = Mutex::new(None);
441
442/// Install the decode sampler config (`None` restores greedy argmax).
443#[inline]
444pub fn set_sampler_config(cfg: Option<crate::sampler::SamplerConfig>) {
445    if let Ok(mut g) = SAMPLER_CONFIG.lock() {
446        // A greedy config is equivalent to None — normalize so the decode loop can skip the
447        // full-logits readback entirely when nothing non-greedy is requested.
448        *g = cfg.filter(|c| !c.is_greedy());
449    }
450}
451
452/// The active decode sampler config, if a non-greedy one is installed.
453#[inline]
454pub fn sampler_config() -> Option<crate::sampler::SamplerConfig> {
455    if let Some(value) = super::prepared_settings::read(|settings| settings.sampler_config) {
456        return value;
457    }
458    SAMPLER_CONFIG.lock().ok().and_then(|g| *g)
459}
460
461// ── R9: DOMINO constrained-decoding toggle ────────────────────────────────────
462// Process-global DOMINO masker. When active, the decode loop applies the
463// grammar constraint mask before sampling. The masker owns a token trie
464// built from the vocabulary (cold path). Set per-request by the host layer
465// when VibeScript constrained generation is requested.
466static DOMINO_MASKER: Mutex<Option<crate::inference::DominoMasker>> = Mutex::new(None);
467
468/// Install a DOMINO masker for constrained decoding. The masker must be
469/// built from the tokenizer vocabulary. Pass `None` to disable.
470pub fn set_domino_masker(masker: Option<crate::inference::DominoMasker>) {
471    if super::prepared_domino::invalidate() {
472        return;
473    }
474    if let Ok(mut g) = DOMINO_MASKER.lock() {
475        *g = masker;
476    }
477}
478
479/// Whether DOMINO constrained decoding is active.
480pub fn domino_active() -> bool {
481    if let Some(active) = super::prepared_domino::with(|grammar| {
482        grammar.as_ref().is_some_and(|grammar| grammar.active())
483    }) {
484        return active;
485    }
486    DOMINO_MASKER
487        .lock()
488        .ok()
489        .and_then(|g| g.as_ref().map(|m| m.is_active()))
490        .unwrap_or(false)
491}
492
493/// Apply the DOMINO mask to logits if a masker is installed and active.
494/// Returns the sampled token using `sample_constrained`, or `None` if no
495/// masker is installed (caller should use plain `sample`).
496pub fn domino_sample(
497    sampler: &mut crate::sampler::SamplerState,
498    logits: &mut [f32],
499    ctx: &[u32],
500) -> Option<u32> {
501    if let Some(token) = super::prepared_domino::with(|grammar| {
502        grammar
503            .as_mut()
504            .and_then(|grammar| grammar.sample(sampler, logits, ctx))
505    }) {
506        return token;
507    }
508    let mut g = DOMINO_MASKER.lock().ok()?;
509    let masker = g.as_mut()?;
510    if !masker.is_active() {
511        return None;
512    }
513    Some(sampler.sample_constrained(logits, ctx, masker))
514}
515
516/// Feed a decoded token's bytes back into the DOMINO grammar state machine
517/// so the grammar advances for the next token's mask. No-op when no masker
518/// is installed or it is inactive (T53/W11).
519///
520/// This MUST be called after every token selected under constrained decoding,
521/// otherwise the grammar state never advances and every token gets masked
522/// against the initial state.
523pub fn domino_feed_token(bytes: &[u8]) {
524    if super::prepared_domino::with(|grammar| {
525        if let Some(grammar) = grammar {
526            grammar.feed(bytes);
527        }
528    })
529    .is_some()
530    {
531        return;
532    }
533    if let Ok(mut g) = DOMINO_MASKER.lock() {
534        if let Some(masker) = g.as_mut() {
535            masker.feed_token(bytes);
536        }
537    }
538}
539
540/// Reset the DOMINO grammar state to the initial state (e.g. for a new
541/// generation turn). No-op when no masker is installed (T53/W11).
542pub fn domino_reset() {
543    if super::prepared_domino::with(|grammar| {
544        if let Some(grammar) = grammar {
545            grammar.reset();
546        }
547    })
548    .is_some()
549    {
550        return;
551    }
552    if let Ok(mut g) = DOMINO_MASKER.lock() {
553        if let Some(masker) = g.as_mut() {
554            masker.reset();
555        }
556    }
557}
558
559/// Cold request-local grammar ownership. The process template is only read;
560/// no provisional or accepted grammar mutation can alter future requests.
561pub fn prepare_domino_scope(
562    budget: crate::inference::speculative_decode::DominoPreparationBudget,
563) -> Result<super::prepared_domino::PreparedDominoScope, &'static str> {
564    let grammar = DOMINO_MASKER
565        .lock()
566        .map_err(|_| "DOMINO template poisoned")?
567        .as_ref()
568        .map(|grammar| grammar.prepare_owned(budget))
569        .transpose()?;
570    super::prepared_domino::PreparedDominoScope::enter(grammar)
571}
572
573/// Reserve private grammar capacity before a candidate can be accepted.
574pub fn domino_admit_piece(raw_bytes: usize) -> bool {
575    super::prepared_domino::with(|grammar| {
576        grammar
577            .as_mut()
578            .is_none_or(|grammar| grammar.admit_raw_piece(raw_bytes))
579    })
580    .unwrap_or(true)
581}
582
583// ── Phase 3: FFN fusion toggle ────────────────────────────────────────────────
584// Default ON (native). Runs the whole pre-norm FFN — gate GEMM, up GEMM, GPU SiLU·mul,
585// down GEMM — in ONE command submit with intermediates kept in VRAM, so a layer costs ONE
586// submit→wait round-trip instead of three (the gate/up/down readbacks + CPU SiLU·mul between
587// them). Requires resident weights (it binds resident weight buffers); falls back to the
588// per-GEMM path when resident is off or a tensor is GPU-ineligible. `QUALIA_LLM_FFN_FUSION=0`
589// forces the per-GEMM path (the A/B OFF baseline).
590static FFN_FUSION: AtomicBool = AtomicBool::new(true);
591
592/// Enable/disable the fused single-submit FFN path (`QUALIA_LLM_FFN_FUSION`).
593#[inline]
594pub fn set_ffn_fusion(on: bool) {
595    FFN_FUSION.store(on, Ordering::Relaxed);
596}
597
598/// Whether the native FFN should run fused (one submit/layer) rather than three GEMM round-trips.
599#[inline]
600pub fn ffn_fusion_enabled() -> bool {
601    if let Some(value) = super::prepared_settings::read(|settings| settings.ffn_fusion_enabled) {
602        return value;
603    }
604    match std::env::var("QUALIA_LLM_FFN_FUSION").ok().as_deref() {
605        Some("0") | Some("false") => false,
606        Some("1") | Some("true") => true,
607        _ => FFN_FUSION.load(Ordering::Relaxed),
608    }
609}
610
611/// Set when a resident decode plan was built with `fused_ffn.wgsl` in the mega-pass (T-A1).
612static FFN_FUSION_IN_RESIDENT: AtomicBool = AtomicBool::new(false);
613
614#[inline]
615/// Sets `ffn-fusion-in-resident`.
616pub fn set_ffn_fusion_in_resident(on: bool) {
617    FFN_FUSION_IN_RESIDENT.store(on, Ordering::Relaxed);
618}
619
620/// True when the last-built resident plan wires fused FFN expansion (not just the flag).
621#[inline]
622pub fn ffn_fusion_in_resident() -> bool {
623    FFN_FUSION_IN_RESIDENT.load(Ordering::Relaxed)
624}
625
626// ── FFN quant → f16 promotion (opt-in; bandwidth vs dequant trade-off) ─
627// Default OFF. Microbench on small GEMMs favoured f16, but full Llama-3.2-3B on
628// A2000 12GB measured **slower** with FFN f16 (~2.1 tok/s) than Q4_K SoA (~2.6)
629// — 4× weight traffic outweighs dequant savings when memory-bound. Opt in with
630// `QUALIA_LLM_FFN_F16=1` on higher-bandwidth GPUs / smaller FFN dims.
631static FFN_F16: AtomicBool = AtomicBool::new(false);
632
633/// Enable/disable FFN quant→f16 promotion at resident-plan build (`QUALIA_LLM_FFN_F16`).
634#[inline]
635pub fn set_ffn_f16(on: bool) {
636    FFN_F16.store(on, Ordering::Relaxed);
637}
638
639/// Whether FFN weights should be promoted to f16 in VRAM for decode/prefill GEMV.
640#[inline]
641pub fn ffn_f16_enabled() -> bool {
642    if let Some(value) = super::prepared_settings::read(|settings| settings.ffn_f16_enabled) {
643        return value;
644    }
645    match std::env::var("QUALIA_LLM_FFN_F16").ok().as_deref() {
646        Some("0") | Some("false") => false,
647        Some("1") | Some("true") => true,
648        _ => FFN_F16.load(Ordering::Relaxed),
649    }
650}
651
652// ── 0.0.21: cooperative GEMV kernel toggle ────────────────────────────────────
653// Default ON (native), verified. Routes native GEMV work through the cooperative
654// one-workgroup-per-row kernel (`coop_gemv`: coalesced reads + per-thread dequant +
655// shared-memory reduction) instead of the naive 1-thread/row `main`. The fused FFN path also selects
656// this cooperative entry point for gate/up/down GEMMs, so decode keeps one FFN readback per layer
657// while using the faster row reducer. The naive GEMV is the measured decode bottleneck
658// (compute/ALU-bound: uncoalesced strided reads + serial accumulate; it also makes Q4_K slower than
659// F16). `QUALIA_LLM_COOP_GEMV=0` forces the naive kernel (the A/B OFF baseline).
660// Earlier A2000 / Llama-3.2-3B-F16 per-GEMM verification: 2.39→3.22 tok/s (+35% over naive).
661static COOP_GEMV: AtomicBool = AtomicBool::new(true);
662
663/// Enable/disable the cooperative GEMV decode path (`QUALIA_LLM_COOP_GEMV`).
664#[inline]
665pub fn set_coop_gemv(on: bool) {
666    COOP_GEMV.store(on, Ordering::Relaxed);
667}
668
669/// Whether native GEMM should run the cooperative `coop_gemv` kernel rather than the naive
670/// 1-thread/row `main`. Env forces either direction; otherwise the atomic flag (default OFF).
671#[inline]
672pub fn coop_gemv_enabled() -> bool {
673    if let Some(value) = super::prepared_settings::read(|settings| settings.coop_gemv_enabled) {
674        return value;
675    }
676    match std::env::var("QUALIA_LLM_COOP_GEMV").ok().as_deref() {
677        Some("0") | Some("false") => false,
678        Some("1") | Some("true") => true,
679        _ => COOP_GEMV.load(Ordering::Relaxed),
680    }
681}
682
683/// Rows per workgroup for multi-row coop GEMV (`coop_gemv_mr` in fused_transformer.wgsl).
684/// Must stay in lock-step with WGSL `COOP_ROWS`.
685pub const COOP_GEMV_ROWS: u32 = 8;
686
687/// Workgroup count for coop GEMV dispatch: `ceil(n_out / COOP_GEMV_ROWS)`.
688#[inline]
689pub fn coop_gemv_workgroups(n_out: u32) -> u32 {
690    n_out.div_ceil(COOP_GEMV_ROWS).max(1)
691}
692
693// ── W8: coopmat (tensor-core) GEMM selection seam ─────────────────────────────
694// Default OFF. The forge already has the self-activating coopmat path
695// (`wgsl_forge::gemm_f32_tc` → `coopmat_usable()` runtime probe): on wgpu 29.0.3 the WGSL coopmat
696// multiply returns zeros (#9741), so `coopmat_usable()` is `false` and the tier stays dormant; it
697// self-activates the moment a wgpu release / soft-fork carries the fix. This toggle is the
698// INFERENCE-side seam: when on AND coopmat is genuinely usable AND the matmul dims fit the 8×8×8 tile
699// (m,n,k multiples of 8 — so batched prefill, not the m=1 decode GEMV), the GEMM backend selector
700// reports `Coopmat`. Until the inference-side coopmat kernel is wired (which needs the wgpu fix), an
701// eligible `Coopmat` selection logs its readiness and falls back to `CoopGemv` — the plumbing is ready
702// and visible, self-activating with the forge. `QUALIA_LLM_COOPMAT=1` arms it.
703static COOPMAT_GEMM: AtomicBool = AtomicBool::new(false);
704
705/// Arm/disarm the coopmat (tensor-core) GEMM selection seam (`QUALIA_LLM_COOPMAT`).
706#[inline]
707pub fn set_coopmat_gemm(on: bool) {
708    COOPMAT_GEMM.store(on, Ordering::Relaxed);
709}
710
711/// Whether the coopmat GEMM seam is armed (env forces either direction; else the flag). Being armed
712/// does not mean coopmat runs — see [`coopmat_gemm_usable`], which also requires the hardware probe.
713#[inline]
714pub fn coopmat_gemm_enabled() -> bool {
715    if let Some(value) = super::prepared_settings::read(|settings| settings.coopmat_gemm_enabled) {
716        return value;
717    }
718    match std::env::var("QUALIA_LLM_COOPMAT").ok().as_deref() {
719        Some("0") | Some("false") => false,
720        Some("1") | Some("true") => true,
721        _ => COOPMAT_GEMM.load(Ordering::Relaxed),
722    }
723}
724
725/// Whether coopmat is BOTH armed and genuinely usable on this device right now (the forge runtime
726/// probe passes — i.e. the wgpu #9741 fix is present). `false` on wgpu 29.0.3. Feature-guarded: with
727/// `wgsl-forge` off there is no probe, so coopmat is never usable.
728#[inline]
729pub fn coopmat_gemm_usable() -> bool {
730    if !coopmat_gemm_enabled() {
731        return false;
732    }
733    #[cfg(all(feature = "wgsl-forge", not(target_arch = "wasm32")))]
734    {
735        crate::wgsl_forge::coopmat_usable()
736    }
737    #[cfg(not(all(feature = "wgsl-forge", not(target_arch = "wasm32"))))]
738    {
739        false
740    }
741}
742
743/// GEMM backend the inference layer selects for a given matmul shape.
744#[derive(Debug, Clone, Copy, PartialEq, Eq)]
745pub enum GemmBackend {
746    /// Naive 1-thread/row `main` kernel (the correctness floor / A-B baseline).
747    Naive,
748    /// Cooperative one-workgroup-per-row `coop_gemv` (coalesced reads + shared-mem reduce).
749    CoopGemv,
750    /// WGSL cooperative-matrix (tensor-core) tile — selected only when armed, usable, and the dims
751    /// fit the 8×8×8 tile. Self-activates via the forge when wgpu #9741 lands.
752    Coopmat,
753}
754
755/// Select the GEMM backend for an `m×k×n` matmul: coopmat when armed+usable and all dims are 8-mult
756/// (the tensor-core tile — batched prefill, not the m=1 decode GEMV); else cooperative GEMV when
757/// enabled; else naive. Pure + total, so it is unit-tested without a GPU.
758#[inline]
759pub fn select_gemm_backend(m: usize, k: usize, n: usize) -> GemmBackend {
760    let tile_fits = m % 8 == 0 && k % 8 == 0 && n % 8 == 0 && m.min(k).min(n) > 0;
761    if coopmat_gemm_usable() && tile_fits {
762        GemmBackend::Coopmat
763    } else if coop_gemv_enabled() {
764        GemmBackend::CoopGemv
765    } else {
766        GemmBackend::Naive
767    }
768}
769
770// ── 0.0.21: resident-activation decode toggle ─────────────────────────────────
771// readback happens, after the final layer — replacing the legacy 2 readbacks/layer (each forced by
772// ── #48 correctness path: CPU attention reference ─────────────────────────────
773// Route native attention through the wasm-proven CPU SDPA (`cpu_attention_pass`) instead of the
774// GPU attention shader (whose output is currently unbounded). Correct-but-slower; opt-in.
775static CPU_ATTENTION: AtomicBool = AtomicBool::new(false);
776
777/// Enable/disable the native CPU-attention reference path (`QUALIA_LLM_CPU_ATTENTION`).
778#[inline]
779pub fn set_cpu_attention(on: bool) {
780    CPU_ATTENTION.store(on, Ordering::Relaxed);
781}
782
783/// Whether native attention should use the CPU reference.
784///
785/// **Default OFF** (use the GPU attention path) — as of #49 the GPU path also honors `norm_weight`
786/// for prefill K/V and produces coherent output, and it is faster. The CPU SDPA reference remains
787/// available as a correctness fallback / cross-check via `QUALIA_LLM_CPU_ATTENTION=1` or
788/// [`set_cpu_attention`].
789#[inline]
790pub fn cpu_attention_enabled() -> bool {
791    if let Some(value) = super::prepared_settings::read(|settings| settings.cpu_attention_enabled) {
792        return value;
793    }
794    CPU_ATTENTION.load(Ordering::Relaxed)
795        || matches!(
796            std::env::var("QUALIA_LLM_CPU_ATTENTION").ok().as_deref(),
797            Some("1") | Some("true")
798        )
799}
800
801// ── Prefix cache enabled toggle ───────────────────────────────────────────────
802static PREFIX_CACHE_ENABLED: AtomicBool = AtomicBool::new(true);
803
804#[inline]
805/// Sets the prefix-cache flag (atomic) — F22: previously a no-op whose getter always read `true`.
806pub fn set_prefix_cache_enabled(on: bool) {
807    PREFIX_CACHE_ENABLED.store(on, Ordering::Relaxed);
808}
809
810#[inline]
811/// Current prefix-cache flag — real atomic read since F22 fix.
812pub fn prefix_cache_enabled() -> bool {
813    if let Some(value) = super::prepared_settings::read(|settings| settings.prefix_cache_enabled) {
814        return value;
815    }
816    PREFIX_CACHE_ENABLED.load(Ordering::Relaxed)
817}
818
819// ── KV pool budget override in MB (0 = default) ──────────────────────────────
820static KV_POOL_BUDGET_MB: AtomicU32 = AtomicU32::new(0);
821
822#[inline]
823/// Sets the KV-pool budget in MiB.
824pub fn set_kv_pool_budget_mb(mb: u32) {
825    KV_POOL_BUDGET_MB.store(mb, Ordering::Relaxed);
826}
827
828#[inline]
829/// Configured KV-pool budget in MiB.
830pub fn kv_pool_budget_mb() -> u32 {
831    if let Some(value) = super::prepared_settings::read(|settings| settings.kv_pool_budget_mb) {
832        return value;
833    }
834    KV_POOL_BUDGET_MB.load(Ordering::Relaxed)
835}
836
837// ── Prefill chunk size override (0 = default PREFILL_CHUNK_SIZE) ──────────────
838static PREFILL_CHUNK_SIZE_OVERRIDE: AtomicU32 = AtomicU32::new(0);
839
840#[inline]
841/// Sets the prefill chunk-size override.
842pub fn set_prefill_chunk_size_override(sz: u32) {
843    PREFILL_CHUNK_SIZE_OVERRIDE.store(sz, Ordering::Relaxed);
844}
845
846#[inline]
847/// Current prefill chunk-size override.
848pub fn prefill_chunk_size_override() -> u32 {
849    if let Some(value) =
850        super::prepared_settings::read(|settings| settings.prefill_chunk_size_override)
851    {
852        return value;
853    }
854    PREFILL_CHUNK_SIZE_OVERRIDE.load(Ordering::Relaxed)
855}
856
857// ── Prompt precision mode setting ─────────────────────────────────────────────
858static PROMPT_PRECISION_MODE: Mutex<String> = Mutex::new(String::new());
859
860/// Sets the prompt precision mode string.
861pub fn set_prompt_precision_mode(mode: &str) {
862    if let Ok(mut m) = PROMPT_PRECISION_MODE.lock() {
863        m.clear();
864        m.push_str(mode);
865    }
866}
867
868/// Current prompt precision mode.
869pub fn prompt_precision_mode() -> String {
870    if let Some(value) =
871        super::prepared_settings::read(|settings| settings.prompt_precision_mode.clone())
872    {
873        return value;
874    }
875    PROMPT_PRECISION_MODE
876        .lock()
877        .map(|m| m.clone())
878        .unwrap_or_default()
879}
880
881// ── Lifecycle stage telemetry (inference-lifecycle.n3 `stage:*`) ─────────────
882// Per-turn StageTrace recording inside the live decode loop. Recording is a
883// fixed 16-slot stack structure coalesced per stage kind — no hot allocations.
884// Default ON; `QUALIA_STAGE_TELEMETRY=0` disables the instrumentation entirely.
885static STAGE_TELEMETRY: AtomicBool = AtomicBool::new(true);
886
887/// Enable/disable per-turn lifecycle stage telemetry (`QUALIA_STAGE_TELEMETRY`).
888#[inline]
889pub fn set_stage_telemetry(on: bool) {
890    STAGE_TELEMETRY.store(on, Ordering::Relaxed);
891}
892
893/// Whether lifecycle stage telemetry is active. The env var overrides the flag
894/// in BOTH directions (`0`/`false` → off, `1`/`true` → on); otherwise the
895/// process default (ON) applies.
896#[inline]
897pub fn stage_telemetry_enabled() -> bool {
898    if let Some(value) = super::prepared_settings::read(|settings| settings.stage_telemetry_enabled)
899    {
900        return value;
901    }
902    match std::env::var("QUALIA_STAGE_TELEMETRY").ok().as_deref() {
903        Some("0") | Some("false") => false,
904        Some("1") | Some("true") => true,
905        _ => STAGE_TELEMETRY.load(Ordering::Relaxed),
906    }
907}