Skip to main content

qualia_core_db/inference/inference_bench/
probes.rs

1//! Correctness / parity / evaluation probes that drive the real engine on a
2//! dedicated runtime: top-k A/B decode, single-decode metrics, GEMM parity
3//! (Q8/F16), native perplexity, GPU KV readback, the AWQ α-sweep, exact-sampler
4//! decode, and the speculative-decode verify probe. Pure code motion — unchanged.
5
6use crate::llm_agent::{AgentBackend, LocalLlmAgent};
7
8use super::metrics::tok_per_s;
9use super::*;
10
11/// A1a correctness: decode the same prompt with the GPU top-k path **off** then **on** (same resident
12/// model, deterministic argmax) and return both strings. Since k=1 top-k == argmax, the texts must be
13/// byte-identical — this verifies the GEMM→top-k wiring, not just the kernel (which is oracle-tested).
14pub fn compare_topk_decode(
15    model_path: &str,
16    prompt: &str,
17    decode_tokens: u32,
18) -> Result<(String, String), String> {
19    if !std::path::Path::new(model_path).exists() {
20        return Err(format!("model not found: {model_path}"));
21    }
22    let agent = LocalLlmAgent::with_local_backend(
23        "did:qualia:bench",
24        AgentBackend::Local {
25            model_path: model_path.to_string(),
26            context_window: 4096,
27            quantization: "auto".into(),
28            vision_projector_path: None,
29            modality: "text".into(),
30            architecture: None,
31        },
32    );
33    let _benchmark_budget = benchmark_decode_budget_scope(decode_tokens);
34    let model_id = crate::q_hash(model_path);
35    let _ = crate::resident_model::mount_resident_gguf(model_id, model_path, false);
36
37    set_gpu_topk(false);
38    let (off_text, _, _, _) = agent.infer_local_model_with_execution::<fn(String)>(
39        prompt,
40        "",
41        crate::llm_agent::NativeDecodeRequest::Developer,
42        None,
43        None,
44    );
45    set_gpu_topk(true);
46    let (on_text, _, _, _) = agent.infer_local_model_with_execution::<fn(String)>(
47        prompt,
48        "",
49        crate::llm_agent::NativeDecodeRequest::Developer,
50        None,
51        None,
52    );
53
54    set_gpu_topk(false);
55    crate::resident_model::clear_resident_model();
56    Ok((off_text, on_text))
57}
58
59/// `compare_topk_decode` inside a fresh multi-thread runtime (residency mount needs `block_in_place`).
60pub fn compare_topk_decode_blocking(
61    model_path: &str,
62    prompt: &str,
63    decode_tokens: u32,
64) -> Result<(String, String), String> {
65    let rt = tokio::runtime::Builder::new_multi_thread()
66        .worker_threads(2)
67        .enable_all()
68        .build()
69        .map_err(|e| e.to_string())?;
70    rt.block_on(async { compare_topk_decode(model_path, prompt, decode_tokens) })
71}
72
73/// A1b: mount a model (auto-detecting `P64` vs GGUF by magic) and run ONE decode of `prompt` for
74/// `decode_tokens`, returning `(text, decode_tok_s)`. For a ternary `.q42` the FFN routing follows
75/// the global `set_ternary_ffn` toggle, so a caller can measure GPU-ON vs CPU-OFF on identical
76/// weights. Caller sets the toggle before invoking. (Use the `_blocking` wrapper from sync code.)
77#[cfg(not(target_arch = "wasm32"))]
78pub fn decode_with_metrics(
79    model_path: &str,
80    prompt: &str,
81    decode_tokens: u32,
82) -> Result<(String, f64), String> {
83    if !std::path::Path::new(model_path).exists() {
84        return Err(format!("model not found: {model_path}"));
85    }
86    let is_q42 = {
87        use std::io::Read;
88        let mut buf = [0u8; 4];
89        std::fs::File::open(model_path)
90            .and_then(|mut f| f.read_exact(&mut buf))
91            .map(|_| &buf == b"p64\0")
92            .unwrap_or(false)
93    };
94    let agent = LocalLlmAgent::with_local_backend(
95        "did:qualia:bench",
96        AgentBackend::Local {
97            model_path: model_path.to_string(),
98            context_window: 4096,
99            quantization: "auto".into(),
100            vision_projector_path: None,
101            modality: "text".into(),
102            architecture: None,
103        },
104    );
105    let _benchmark_budget = benchmark_decode_budget_scope(decode_tokens);
106    let model_id = crate::q_hash(model_path);
107    if is_q42 {
108        crate::resident_model::mount_resident_q42(model_id, model_path, false)?;
109    } else {
110        let _ = crate::resident_model::mount_resident_gguf(model_id, model_path, false);
111    }
112    reset_phase_metrics();
113    let (text, _, _, _) = agent.infer_local_model_with_execution::<fn(String)>(
114        prompt,
115        "",
116        crate::llm_agent::NativeDecodeRequest::Developer,
117        None,
118        None,
119    );
120    let snap = phase_snapshot();
121    let decode_tok_s = tok_per_s(snap.decode_tokens, snap.decode_ns);
122    crate::resident_model::clear_resident_model();
123    Ok((text, decode_tok_s))
124}
125
126/// W3 — GPU↔CPU GEMM parity probe (test/diagnostic). Builds a fresh engine, synthesizes a random
127/// Q8_0 weight matrix (`n_out` rows × `n_in`; `n_in` must be a multiple of 32) + input from `seed`,
128/// runs the GPU kernel and the CPU reference on **identical** bytes, and returns
129/// `(max_abs_err, mean_abs_err, max_ulp, gpu_gemm_passes_profiled)`. A non-zero pass count proves the
130/// GPU path actually executed — the engine readback falls back to CPU when no tokio handle is present,
131/// so the `rt.enter()` below installs one to force the real GPU path.
132#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
133pub fn gemm_parity_probe_blocking(
134    n_in: usize,
135    n_out: usize,
136    seed: u64,
137) -> Result<(f32, f64, u64, u64), String> {
138    use crate::gguf_sharder::GgufTensorInfo;
139    if n_in == 0 || n_out == 0 || n_in % 32 != 0 {
140        return Err("n_in must be a non-zero multiple of 32 (Q8_0 block size); n_out > 0".into());
141    }
142    let rt = tokio::runtime::Builder::new_multi_thread()
143        .worker_threads(2)
144        .enable_all()
145        .build()
146        .map_err(|e| e.to_string())?;
147    let mut engine = rt
148        .block_on(crate::gguf_bridge::QTensorEngine::try_new())
149        .map_err(|e| format!("engine init: {e}"))?;
150    let _guard = rt.enter(); // install a tokio handle on this thread so the GPU readback path runs
151
152    // Deterministic LCG → values in [-1, 1).
153    let mut s = seed | 1;
154    let mut rng = move || -> f32 {
155        s = s
156            .wrapping_mul(6364136223846793005)
157            .wrapping_add(1442695040888963407);
158        ((s >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0
159    };
160
161    let row_bytes = crate::llm_kernel_parity::q8_0_bytes(n_in);
162    let mut raw = vec![0u8; row_bytes * n_out];
163    let mut row_f32 = vec![0f32; n_in];
164    for r in 0..n_out {
165        for x in row_f32.iter_mut() {
166            *x = rng();
167        }
168        if !crate::llm_kernel_parity::quantize_q8_0_from_f32(
169            &row_f32,
170            &mut raw[r * row_bytes..(r + 1) * row_bytes],
171        ) {
172            return Err("q8_0 quantize failed".into());
173        }
174    }
175    let input: Vec<f32> = (0..n_in).map(|_| rng()).collect();
176
177    let info = GgufTensorInfo {
178        dims: [n_in as u64, n_out as u64, 1, 1],
179        n_dims: 2,
180        ggml_type: crate::ggml_quants::GGML_TYPE_Q8_0,
181        byte_offset: 0,
182    };
183
184    crate::llm_gpu_profiler::set_enabled(true);
185    crate::llm_gpu_profiler::reset();
186    let mut gpu_out = vec![0f32; n_out];
187    let mut cpu_out = vec![0f32; n_out];
188    let ok = engine.gemm_parity_probe(&info, &raw, &input, &mut gpu_out, &mut cpu_out, n_in, n_out);
189    let calls = crate::llm_gpu_profiler::snapshot()
190        .iter()
191        .find(|t| matches!(t.phase, crate::llm_gpu_profiler::Phase::Gemm))
192        .map(|t| t.calls)
193        .unwrap_or(0);
194    crate::llm_gpu_profiler::set_enabled(false);
195    if !ok {
196        return Err("gemm_parity_probe: GPU or CPU path returned false".into());
197    }
198    Ok((
199        crate::llm_kernel_parity::max_abs_err(&gpu_out, &cpu_out),
200        crate::llm_kernel_parity::mean_abs_err(&gpu_out, &cpu_out),
201        crate::llm_kernel_parity::max_ulp_diff(&gpu_out, &cpu_out),
202        calls,
203    ))
204}
205
206#[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
207/// `?::gemm_parity_probe_blocking`.
208pub fn gemm_parity_probe_blocking(
209    n_in: usize,
210    n_out: usize,
211    seed: u64,
212) -> Result<(f32, f64, u64, u64), String> {
213    let _ = (n_in, n_out, seed);
214    Err("GEMM parity probe requires the `gpu-runtime` feature; refusing to stub GPU success".into())
215}
216
217/// W3/F16 — GPU↔CPU parity for the new **F16** GEMM path (`unpack2x16float` in the shader vs the CPU
218/// `dequant_f16` reference). Synthesizes a random F16 weight matrix (`n_out` rows × `n_in`; no block
219/// constraint) + input from `seed`, runs both on identical bytes, returns
220/// `(max_abs_err, mean_abs_err, max_ulp, gpu_gemm_passes)`. Same witness rule as the Q8 probe.
221#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
222pub fn gemm_parity_probe_f16_blocking(
223    n_in: usize,
224    n_out: usize,
225    seed: u64,
226) -> Result<(f32, f64, u64, u64), String> {
227    use crate::gguf_sharder::GgufTensorInfo;
228    if n_in == 0 || n_out == 0 {
229        return Err("n_in and n_out must be > 0".into());
230    }
231    let rt = tokio::runtime::Builder::new_multi_thread()
232        .worker_threads(2)
233        .enable_all()
234        .build()
235        .map_err(|e| e.to_string())?;
236    let mut engine = rt
237        .block_on(crate::gguf_bridge::QTensorEngine::try_new())
238        .map_err(|e| format!("engine init: {e}"))?;
239    let _guard = rt.enter();
240
241    let mut s = seed | 1;
242    let mut rng = move || -> f32 {
243        s = s
244            .wrapping_mul(6364136223846793005)
245            .wrapping_add(1442695040888963407);
246        ((s >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0
247    };
248
249    let row_bytes = crate::llm_kernel_parity::f16_bytes(n_in);
250    let mut raw = vec![0u8; row_bytes * n_out];
251    let mut row_f32 = vec![0f32; n_in];
252    for r in 0..n_out {
253        for x in row_f32.iter_mut() {
254            *x = rng();
255        }
256        if !crate::llm_kernel_parity::quantize_f16_from_f32(
257            &row_f32,
258            &mut raw[r * row_bytes..(r + 1) * row_bytes],
259        ) {
260            return Err("f16 quantize failed".into());
261        }
262    }
263    let input: Vec<f32> = (0..n_in).map(|_| rng()).collect();
264
265    let info = GgufTensorInfo {
266        dims: [n_in as u64, n_out as u64, 1, 1],
267        n_dims: 2,
268        ggml_type: crate::ggml_quants::GGML_TYPE_F16,
269        byte_offset: 0,
270    };
271
272    crate::llm_gpu_profiler::set_enabled(true);
273    crate::llm_gpu_profiler::reset();
274    let mut gpu_out = vec![0f32; n_out];
275    let mut cpu_out = vec![0f32; n_out];
276    let ok = engine.gemm_parity_probe(&info, &raw, &input, &mut gpu_out, &mut cpu_out, n_in, n_out);
277    let calls = crate::llm_gpu_profiler::snapshot()
278        .iter()
279        .find(|t| matches!(t.phase, crate::llm_gpu_profiler::Phase::Gemm))
280        .map(|t| t.calls)
281        .unwrap_or(0);
282    crate::llm_gpu_profiler::set_enabled(false);
283    if !ok {
284        return Err("gemm_parity_probe (f16): GPU or CPU path returned false".into());
285    }
286    Ok((
287        crate::llm_kernel_parity::max_abs_err(&gpu_out, &cpu_out),
288        crate::llm_kernel_parity::mean_abs_err(&gpu_out, &cpu_out),
289        crate::llm_kernel_parity::max_ulp_diff(&gpu_out, &cpu_out),
290        calls,
291    ))
292}
293
294#[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
295/// `?::gemm_parity_probe_f16_blocking`.
296pub fn gemm_parity_probe_f16_blocking(
297    n_in: usize,
298    n_out: usize,
299    seed: u64,
300) -> Result<(f32, f64, u64, u64), String> {
301    let _ = (n_in, n_out, seed);
302    Err(
303        "F16 GEMM parity probe requires the `gpu-runtime` feature; refusing to stub GPU success"
304            .into(),
305    )
306}
307
308/// W1 — teacher-forced perplexity of `model_path` over the eval corpus, run through Qualia's **native**
309/// engine (never an external runtime). For each corpus passage: `reset_kv_cache`, then per position
310/// embed → `dispatch_transformer_forward` → `apply_output_norm_inplace` → `dispatch_output_logits_into`
311/// → NLL of the true next token; PPL = `exp(ΣNLL / Σtokens)`. `max_tok` = 0 scores the whole passage,
312/// >0 caps it (to bound the slow F16-on-CPU path for big models). Returns `(perplexity, tokens_scored)`.
313/// Runs on a dedicated thread with a current-thread tokio runtime (mirrors the decode path) so the
314/// engine's GPU readback works. Handles both GGUF and `.q42` containers.
315#[cfg(not(target_arch = "wasm32"))]
316pub fn perplexity_eval_blocking(model_path: &str, max_tok: usize) -> Result<(f64, usize), String> {
317    use crate::gguf_bridge::QTensorEngine;
318    use crate::gguf_sharder::{GgufTensorIndex, GgufTokenizer};
319
320    let corpus = crate::llm_eval::load_corpus().map_err(|e| format!("corpus load: {e}"))?;
321    if corpus.is_empty() {
322        return Err("eval corpus is empty".into());
323    }
324    let model_path = model_path.to_string();
325
326    std::thread::spawn(move || -> Result<(f64, usize), String> {
327        let rt = tokio::runtime::Builder::new_current_thread()
328            .enable_all()
329            .build()
330            .map_err(|e| e.to_string())?;
331        let _g = rt.enter();
332
333        let mut engine = QTensorEngine::new();
334        let mut magic = [0u8; 4];
335        let is_q42 = {
336            use std::io::Read;
337            std::fs::File::open(&model_path)
338                .and_then(|mut f| f.read_exact(&mut magic))
339                .map(|_| &magic == b"p64\0")
340                .unwrap_or(false)
341        };
342        if is_q42 {
343            let f = std::fs::File::open(&model_path).map_err(|e| e.to_string())?;
344            let mmap = unsafe { memmap2::Mmap::map(&f) }.map_err(|e| e.to_string())?;
345            engine
346                .adopt_resident_p64_mmap(std::sync::Arc::new(mmap))
347                .map_err(|e| format!("q42 adopt: {e}"))?;
348        } else {
349            engine.load_gguf(&model_path);
350        }
351
352        let mmap = engine
353            .clone_model_bytes()
354            .ok_or_else(|| "model did not memory-map (load failed)".to_string())?;
355        let is_q42_mmap = mmap.len() >= 4 && mmap[0..4] == *b"p64\0";
356        let tok = if is_q42_mmap {
357            crate::p64_weight::P64TensorIndex::from_p64(&mmap)
358                .ok()
359                .and_then(|qi| GgufTokenizer::from_p64_section(qi.tokenizer_bytes(&mmap)))
360                .unwrap_or_default()
361        } else {
362            GgufTokenizer::from_gguf(&mmap)
363        };
364        let tensor_idx = if is_q42_mmap {
365            crate::p64_weight::P64TensorIndex::from_p64(&mmap)
366                .map(|qi| qi.to_gguf_index())
367                .map_err(|e| format!("q42 index: {e}"))?
368        } else {
369            GgufTensorIndex::from_gguf(&mmap)
370        };
371
372        let emb_dim = tensor_idx.emb_dim();
373        if emb_dim == 0 {
374            return Err("embedding dimension is 0 (tensor index parse failed)".into());
375        }
376        let vocab = tok.vocab_len().max(1) as usize;
377
378        let mut emb_buf = vec![0f32; emb_dim.max(8192)];
379        let mut scratch_a = vec![0f32; 16384];
380        let mut scratch_b = vec![0f32; 16384];
381        let mut logits = vec![0f32; vocab];
382        let mmap_bytes: &[u8] = &mmap;
383
384        let mut total_nll = 0.0f64;
385        let mut total_tok = 0usize;
386        for passage in &corpus {
387            let toks = tok.encode(passage);
388            if toks.len() < 2 {
389                continue;
390            }
391            let limit = if max_tok > 0 {
392                (max_tok + 1).min(toks.len())
393            } else {
394                toks.len()
395            };
396            engine.reset_kv_cache();
397            for i in 0..limit - 1 {
398                let n_emb = tensor_idx.dequantize_token_embedding_into(
399                    mmap_bytes,
400                    toks[i],
401                    &mut emb_buf[..emb_dim],
402                );
403                if n_emb == 0 {
404                    return Err(format!("embedding lookup failed for token {}", toks[i]));
405                }
406                // AWQ calibration: reset the per-forward layer cursor so the FFN hook tags layers
407                // 0..n_layer-1 correctly (no-op when AWQ capture is off).
408                crate::llm_awq::begin_forward();
409                let _ = engine.dispatch_transformer_forward(
410                    &tensor_idx,
411                    &mut emb_buf[..emb_dim],
412                    emb_dim,
413                    &mut scratch_a,
414                    &mut scratch_b,
415                    i as u32,
416                    0, // 0 = all layers (full model depth)
417                );
418                let _ =
419                    engine.apply_output_norm_inplace(&tensor_idx, &mut emb_buf[..emb_dim], emb_dim);
420                let n = engine.dispatch_output_logits_into(
421                    &tensor_idx,
422                    &emb_buf[..emb_dim],
423                    emb_dim,
424                    &mut logits,
425                );
426                if n == 0 {
427                    return Err("output projection produced no logits".into());
428                }
429                let nll = crate::llm_eval::token_nll(&logits[..n], toks[i + 1] as usize);
430                if nll.is_finite() {
431                    total_nll += nll;
432                    total_tok += 1;
433                }
434            }
435        }
436        if total_tok == 0 {
437            return Err("no tokens scored".into());
438        }
439        Ok((crate::llm_eval::perplexity(total_nll, total_tok), total_tok))
440    })
441    .join()
442    .map_err(|_| "perplexity eval thread panicked".to_string())?
443}
444
445/// GPU-readback KV capture for the W5b sparse-dictionary go/no-go — the independent route to the
446/// CPU-reference hook. Loads the model with an **f32** KV cache, runs the REAL fast GPU decode forward
447/// over the eval corpus, and after each passage reads the KV arena back from VRAM
448/// (`QTensorEngine::capture_kv_f32()`), accumulating up to `max_per_layer` K and V vectors per layer.
449/// Because it samples the actual decode-path vectors (not the CPU reference), it cross-checks the hook
450/// capture: if both agree, the measured KV geometry is trustworthy. Stops early once every layer's cap
451/// is hit. Needs a GPU.
452#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
453pub fn capture_kv_gpu_readback(
454    model_path: &str,
455    max_tok: usize,
456    max_per_layer: usize,
457) -> Result<crate::kv_capture::KvCapture, String> {
458    use crate::gguf_bridge::QTensorEngine;
459    use crate::gguf_sharder::{GgufTensorIndex, GgufTokenizer};
460
461    let corpus = crate::llm_eval::load_corpus().map_err(|e| format!("corpus load: {e}"))?;
462    if corpus.is_empty() {
463        return Err("eval corpus is empty".into());
464    }
465    let model_path = model_path.to_string();
466
467    std::thread::spawn(move || -> Result<crate::kv_capture::KvCapture, String> {
468        let rt = tokio::runtime::Builder::new_current_thread()
469            .enable_all()
470            .build()
471            .map_err(|e| e.to_string())?;
472        let _g = rt.enter();
473
474        // Force an f32 KV layout so the readback decodes via k_index/v_index (int8 packs differently).
475        let prev_int8 = kv_int8_enabled();
476        set_kv_int8(false);
477
478        let mut engine = QTensorEngine::new();
479        let mut magic = [0u8; 4];
480        let is_q42 = {
481            use std::io::Read;
482            std::fs::File::open(&model_path)
483                .and_then(|mut f| f.read_exact(&mut magic))
484                .map(|_| &magic == b"p64\0")
485                .unwrap_or(false)
486        };
487        if is_q42 {
488            let f = std::fs::File::open(&model_path).map_err(|e| e.to_string())?;
489            let mmap = unsafe { memmap2::Mmap::map(&f) }.map_err(|e| e.to_string())?;
490            engine
491                .adopt_resident_p64_mmap(std::sync::Arc::new(mmap))
492                .map_err(|e| format!("q42 adopt: {e}"))?;
493        } else {
494            engine.load_gguf(&model_path);
495        }
496
497        let mmap = engine
498            .clone_model_bytes()
499            .ok_or_else(|| "model did not memory-map (load failed)".to_string())?;
500        let is_q42_mmap = mmap.len() >= 4 && mmap[0..4] == *b"p64\0";
501        let tok = if is_q42_mmap {
502            crate::p64_weight::P64TensorIndex::from_p64(&mmap)
503                .ok()
504                .and_then(|qi| GgufTokenizer::from_p64_section(qi.tokenizer_bytes(&mmap)))
505                .unwrap_or_default()
506        } else {
507            GgufTokenizer::from_gguf(&mmap)
508        };
509        let tensor_idx = if is_q42_mmap {
510            crate::p64_weight::P64TensorIndex::from_p64(&mmap)
511                .map(|qi| qi.to_gguf_index())
512                .map_err(|e| format!("q42 index: {e}"))?
513        } else {
514            GgufTensorIndex::from_gguf(&mmap)
515        };
516
517        let emb_dim = tensor_idx.emb_dim();
518        if emb_dim == 0 {
519            return Err("embedding dimension is 0".into());
520        }
521        let mut emb_buf = vec![0f32; emb_dim.max(8192)];
522        let mut scratch_a = vec![0f32; 16384];
523        let mut scratch_b = vec![0f32; 16384];
524        let mmap_bytes: &[u8] = &mmap;
525
526        let mut acc_k: Vec<Vec<Vec<f32>>> = Vec::new();
527        let mut acc_v: Vec<Vec<Vec<f32>>> = Vec::new();
528        let mut head_dim = 0usize;
529
530        'corpus: for passage in &corpus {
531            let toks = tok.encode(passage);
532            if toks.len() < 2 {
533                continue;
534            }
535            let limit = if max_tok > 0 {
536                (max_tok + 1).min(toks.len())
537            } else {
538                toks.len()
539            };
540            engine.reset_kv_cache();
541            for i in 0..limit - 1 {
542                let n_emb = tensor_idx.dequantize_token_embedding_into(
543                    mmap_bytes,
544                    toks[i],
545                    &mut emb_buf[..emb_dim],
546                );
547                if n_emb == 0 {
548                    return Err(format!("embedding lookup failed for token {}", toks[i]));
549                }
550                let _ = engine.dispatch_transformer_forward(
551                    &tensor_idx,
552                    &mut emb_buf[..emb_dim],
553                    emb_dim,
554                    &mut scratch_a,
555                    &mut scratch_b,
556                    i as u32,
557                    0,
558                );
559            }
560            // Read this passage's KV back from VRAM and merge into the accumulator.
561            if let Some(cap) = engine.capture_kv_f32((limit - 1) as u32, max_per_layer) {
562                head_dim = cap.head_dim;
563                if acc_k.is_empty() {
564                    acc_k = vec![Vec::new(); cap.k.len()];
565                    acc_v = vec![Vec::new(); cap.v.len()];
566                }
567                let mut all_full = true;
568                for l in 0..cap.k.len().min(acc_k.len()) {
569                    for vk in &cap.k[l] {
570                        if acc_k[l].len() < max_per_layer {
571                            acc_k[l].push(vk.clone());
572                        }
573                    }
574                    for vv in &cap.v[l] {
575                        if acc_v[l].len() < max_per_layer {
576                            acc_v[l].push(vv.clone());
577                        }
578                    }
579                    if acc_k[l].len() < max_per_layer {
580                        all_full = false;
581                    }
582                }
583                if all_full {
584                    break 'corpus; // caps hit — stop early, don't burn the rest of the corpus
585                }
586            }
587        }
588
589        set_kv_int8(prev_int8);
590        if head_dim == 0 {
591            return Err("no KV captured via GPU readback (int8 layout, or empty forward)".into());
592        }
593        Ok(crate::kv_capture::KvCapture {
594            head_dim,
595            k: acc_k,
596            v: acc_v,
597        })
598    })
599    .join()
600    .map_err(|_| "kv capture thread panicked".to_string())?
601}
602
603#[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
604/// `?::capture_kv_gpu_readback`.
605pub fn capture_kv_gpu_readback(
606    model_path: &str,
607    max_tok: usize,
608    max_per_layer: usize,
609) -> Result<crate::kv_capture::KvCapture, String> {
610    let _ = (model_path, max_tok, max_per_layer);
611    Err("GPU KV readback requires the `gpu-runtime` feature; refusing to stub GPU success".into())
612}
613
614/// AWQ α-sweep on the ternary FFN (AWQ steps 1–3 end to end): capture activation salience from the Q8
615/// reference at `gguf_path`, then for each α compile an AWQ-scaled ternary `.q42`
616/// (`compile_gguf_to_q42_ternary_ffn_awq`), evaluate its perplexity + unique-word coherence, and return
617/// `(reference_ppl, [(alpha, ppl, uniq)])`. α=0.0 is plain ternary (the baseline). `max_tok` caps
618/// tokens/passage to bound the sweep. Honest: this measures whether AWQ rescues ternary — it does not
619/// assume it does. Needs a GPU.
620#[cfg(not(target_arch = "wasm32"))]
621pub fn awq_sweep_blocking(
622    gguf_path: &str,
623    alphas: &[f32],
624    max_tok: usize,
625    quant: crate::p64_weight::FfnQuant,
626) -> Result<(f64, Vec<(f32, f64, f64)>), String> {
627    use crate::p64_weight::compile_gguf_to_q42_ffn_quant_awq;
628
629    let bytes = std::fs::read(gguf_path).map_err(|e| format!("read gguf: {e}"))?;
630    let idx = crate::gguf_sharder::GgufTensorIndex::from_gguf(&bytes);
631    let n_layer = idx.hyperparams.n_layer;
632    let n_embd = idx.hyperparams.n_embd;
633    if n_layer == 0 || n_embd == 0 {
634        return Err("gguf parse failed (n_layer/n_embd = 0)".into());
635    }
636
637    // 1. Capture per-channel salience + the Q8 reference PPL in one calibration forward.
638    set_ternary_ffn(false);
639    crate::llm_awq::enable(n_layer, n_embd)?;
640    let (ref_ppl, _) = perplexity_eval_blocking(gguf_path, max_tok)?;
641    let stats = crate::llm_awq::snapshot();
642    crate::llm_awq::disable();
643    if stats.is_empty() {
644        return Err("AWQ: no activation stats captured".into());
645    }
646
647    // 2. Sweep: AWQ-scaled .q42 per α → eval PPL + coherence. Ternary needs the resident 2-bit path;
648    //    Q4_0 runs through the standard quantized GEMM (no ternary toggle).
649    set_ternary_ffn(matches!(quant, crate::p64_weight::FfnQuant::Ternary));
650    let tmp = std::env::temp_dir();
651    let mut results = Vec::with_capacity(alphas.len());
652    for &alpha in alphas {
653        let scales = if alpha == 0.0 {
654            None
655        } else {
656            Some(stats.as_slice())
657        };
658        let q42 = compile_gguf_to_q42_ffn_quant_awq(&bytes, 14, scales, alpha, quant)
659            .map_err(|e| format!("AWQ compile (alpha={alpha}): {e}"))?;
660        let path = tmp.join(format!("awq_sweep_a{:.2}.q42", alpha));
661        std::fs::write(&path, &q42).map_err(|e| format!("write q42: {e}"))?;
662        let ps = path.to_string_lossy().to_string();
663        let (ppl, _) = perplexity_eval_blocking(&ps, max_tok)?;
664        let (text, _) = decode_with_metrics_blocking(&ps, "Once upon a time, there was a", 24)?;
665        let uniq = crate::llm_eval::unique_word_ratio(&text);
666        let _ = std::fs::remove_file(&path);
667        results.push((alpha, ppl, uniq));
668    }
669    set_ternary_ffn(false);
670    Ok((ref_ppl, results))
671}
672
673/// `decode_with_metrics` inside a fresh multi-thread runtime (residency mount needs `block_in_place`).
674#[cfg(not(target_arch = "wasm32"))]
675pub fn decode_with_metrics_blocking(
676    model_path: &str,
677    prompt: &str,
678    decode_tokens: u32,
679) -> Result<(String, f64), String> {
680    let rt = tokio::runtime::Builder::new_multi_thread()
681        .worker_threads(2)
682        .enable_all()
683        .build()
684        .map_err(|e| e.to_string())?;
685    rt.block_on(async { decode_with_metrics(model_path, prompt, decode_tokens) })
686}
687
688/// W2: decode with the exact CPU sampler installed for the duration of the call. Returns
689/// `(text, tok/s)`. Restores greedy (`None`) afterwards so it never leaks into other tests.
690#[cfg(not(target_arch = "wasm32"))]
691pub fn decode_sampled_blocking(
692    model_path: &str,
693    prompt: &str,
694    decode_tokens: u32,
695    cfg: crate::sampler::SamplerConfig,
696) -> Result<(String, f64), String> {
697    set_sampler_config(Some(cfg));
698    let out = decode_with_metrics_blocking(model_path, prompt, decode_tokens);
699    set_sampler_config(None);
700    out
701}
702
703/// W6a — batched verify-primitive correctness probe. Prefills `prompt` (positions `[0, p)`,
704/// `p = prompt_len-1`), snapshots the KV cache, then:
705///   * **reference** — sequentially forwards `b` steps from the last prompt token via the exact
706///     per-token path (`dispatch_transformer_forward` → output norm → full-logit argmax), collecting
707///     the greedy continuation `r0..r_{b-1}` and the inputs `[cur, r0..r_{b-2}]`;
708///   * restores the post-prefill KV, then runs the **batched** `verify_draft_batch(inputs, p)`.
709/// Returns `(reference, verify)`; they must be equal — the batched forward writes byte-identical KV
710/// and both sides take a full-logit CPU argmax (no top-k tie-break gap). Runs on a dedicated thread
711/// with a current-thread runtime (mirrors the decode/perplexity paths so GPU readback works).
712#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
713pub fn spec_verify_probe_blocking(
714    model_path: &str,
715    prompt: &str,
716    b: usize,
717) -> Result<(Vec<u32>, Vec<u32>, Vec<u32>), String> {
718    use crate::gguf_bridge::QTensorEngine;
719    use crate::gguf_sharder::{GgufTensorIndex, GgufTokenizer};
720    if !std::path::Path::new(model_path).exists() {
721        return Err(format!("model not found: {model_path}"));
722    }
723    let model_path = model_path.to_string();
724    let prompt = prompt.to_string();
725
726    std::thread::spawn(move || -> Result<(Vec<u32>, Vec<u32>, Vec<u32>), String> {
727        let rt = tokio::runtime::Builder::new_current_thread()
728            .enable_all()
729            .build()
730            .map_err(|e| e.to_string())?;
731        let _g = rt.enter();
732
733        let mut engine = QTensorEngine::new();
734        engine.load_gguf(&model_path);
735        let mmap = engine
736            .clone_model_bytes()
737            .ok_or_else(|| "model did not memory-map".to_string())?;
738        let tok = GgufTokenizer::from_gguf(&mmap);
739        let idx = GgufTensorIndex::from_gguf(&mmap);
740        // The batched verify tail binds the RESIDENT logits projection (like resident_decode). Plain
741        // `load_gguf` does not upload it (only the residency-mount path does), so do it here.
742        if !engine.mc8_upload_resident_logits(&idx) {
743            return Err("resident logits upload failed (verify tail needs it)".into());
744        }
745        let emb_dim = idx.emb_dim();
746        if emb_dim == 0 {
747            return Err("embedding dimension is 0".into());
748        }
749        let vocab = tok.vocab_len().max(1) as usize;
750        let toks = tok.encode(&prompt);
751        if toks.len() < b + 2 {
752            return Err(format!(
753                "prompt too short: {} tokens, need >= {}",
754                toks.len(),
755                b + 2
756            ));
757        }
758
759        let mut emb = vec![0f32; emb_dim.max(8192)];
760        let mut sa = vec![0f32; 16384];
761        let mut sb = vec![0f32; 16384];
762        let mut logits = vec![0f32; vocab];
763        let mmap_b: &[u8] = &mmap;
764
765        let argmax = |v: &[f32]| -> u32 {
766            let mut best_i = 0u32;
767            let mut best_v = f32::NEG_INFINITY;
768            for (i, &x) in v.iter().enumerate() {
769                if x > best_v {
770                    best_v = x;
771                    best_i = i as u32;
772                }
773            }
774            best_i
775        };
776
777        // Prefill positions [0, p) so the prefix KV matches what decode would have.
778        let p = toks.len() - 1;
779        engine.reset_kv_cache();
780        for i in 0..p {
781            let n = idx.dequantize_token_embedding_into(mmap_b, toks[i], &mut emb[..emb_dim]);
782            if n == 0 {
783                return Err(format!("embedding lookup failed for token {}", toks[i]));
784            }
785            let _ = engine.dispatch_transformer_forward(
786                &idx,
787                &mut emb[..emb_dim],
788                emb_dim,
789                &mut sa,
790                &mut sb,
791                i as u32,
792                0,
793            );
794        }
795        // No KV snapshot needed: the reference decode below writes only positions [p, p+b), never the
796        // prefix [0, p) — so the prefix KV stays valid — and `verify_draft_batch` overwrites [p, p+b)
797        // with the same inputs. (Note: `get_kv_cache_cpu` returns the CPU mirror, which is NOT synced
798        // from the GPU KV writes, so it cannot be used to snapshot the GPU cache here.)
799
800        // Reference: exact sequential greedy decode of `b` steps from the last prompt token.
801        let mut inputs: Vec<u32> = Vec::with_capacity(b);
802        let mut reference: Vec<u32> = Vec::with_capacity(b);
803        let mut input = toks[p];
804        let mut pos = p as u32;
805        for _ in 0..b {
806            inputs.push(input);
807            let n = idx.dequantize_token_embedding_into(mmap_b, input, &mut emb[..emb_dim]);
808            if n == 0 {
809                return Err("reference embedding lookup failed".into());
810            }
811            let _ = engine.dispatch_transformer_forward(
812                &idx,
813                &mut emb[..emb_dim],
814                emb_dim,
815                &mut sa,
816                &mut sb,
817                pos,
818                0,
819            );
820            let _ = engine.apply_output_norm_inplace(&idx, &mut emb[..emb_dim], emb_dim);
821            let nl =
822                engine.dispatch_output_logits_into(&idx, &emb[..emb_dim], emb_dim, &mut logits);
823            if nl == 0 {
824                return Err("reference output projection produced no logits".into());
825            }
826            let out = argmax(&logits[..nl]);
827            reference.push(out);
828            input = out;
829            pos += 1;
830        }
831
832        // Run the batched verify over the same inputs (prefix KV [0, p) is still valid).
833        let mut verify_out: Vec<u32> = Vec::new();
834        let mut verify_logit: Vec<f32> = Vec::new();
835        engine
836            .verify_draft_batch(&idx, &inputs, p as u32, &mut verify_out, &mut verify_logit)
837            .ok_or_else(|| "verify_draft_batch ineligible/failed".to_string())?;
838
839        // Resident-path reference: re-prefill the prefix, then forward each input through the DEFAULT
840        // single-fence resident path (GPU top-1). Isolates verify(batched, CPU argmax) vs
841        // resident(single, GPU top-1) — the crux of the transparency question. u32::MAX marks a
842        // resident-ineligible position.
843        engine.reset_kv_cache();
844        for i in 0..p {
845            let n = idx.dequantize_token_embedding_into(mmap_b, toks[i], &mut emb[..emb_dim]);
846            if n == 0 {
847                return Err("resident-ref prefill embedding failed".into());
848            }
849            let _ = engine.dispatch_transformer_forward(
850                &idx,
851                &mut emb[..emb_dim],
852                emb_dim,
853                &mut sa,
854                &mut sb,
855                i as u32,
856                0,
857            );
858        }
859        let mut resident_out: Vec<u32> = Vec::with_capacity(b);
860        for (j, &t) in inputs.iter().enumerate() {
861            let n = idx.dequantize_token_embedding_into(mmap_b, t, &mut emb[..emb_dim]);
862            if n == 0 {
863                return Err("resident-ref embedding failed".into());
864            }
865            match engine.dispatch_token_forward_resident(&idx, &emb[..emb_dim], (p + j) as u32) {
866                Some(r) => resident_out.push(r.best_token_id),
867                None => resident_out.push(u32::MAX),
868            }
869        }
870
871        Ok((reference, verify_out, resident_out))
872    })
873    .join()
874    .map_err(|_| "spec-verify probe thread panicked".to_string())?
875}
876
877#[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
878/// `?::spec_verify_probe_blocking`.
879pub fn spec_verify_probe_blocking(
880    model_path: &str,
881    prompt: &str,
882    b: usize,
883) -> Result<(Vec<u32>, Vec<u32>, Vec<u32>), String> {
884    let _ = (model_path, prompt, b);
885    Err(
886        "speculative-verify probe requires the `gpu-runtime` feature; refusing to stub GPU success"
887            .into(),
888    )
889}