Skip to main content

qualia_core_db/gguf_bridge/
mod.rs

1//! Model-inference runtime (honest module name: `crate::inference_runtime`).
2//!
3//! This is the **runtime** that runs model inference — not an "AI engine". It does two ordinary
4//! systems jobs: (1) **reads the GGUF weight file format** (tensors memory-mapped via `memmap2`,
5//! zero heap copy) and (2) **dispatches the tensor program to a GPU backend** (DirectML 1.15 on
6//! Windows x64; wgpu/WGSL — Vulkan/Metal/WebGPU — elsewhere).
7//!
8//! The *mathematics* of inference is not here and is not proprietary: it lives in `crate::solvers`
9//! as named STEM — GEMM (`linear_algebra::gemm`), activations/softmax/normalization
10//! (`activation`), attention (`attention`), RoPE (`rope`), the SwiGLU FFN (`feed_forward`) — and
11//! each kernel in this crate is proven equal to that library definition (the `*_stem_parity_tests`
12//! and `substrate_parity_tests`). What remains here is plumbing: GPU command encoding, the KV
13//! cache, weight loading, the autoregressive loop, and the GGUF/dequant codec.
14
15#![cfg_attr(not(feature = "gpu-runtime"), allow(dead_code))]
16#![allow(unused_imports, dead_code)]
17
18// pub(crate) so the concern submodules (gemm / ffn / attention / output / embedding / mc8_wasm …)
19// inherit these via `use super::*` — keeps the impl split a pure structural move.
20pub(crate) use crate::NQuin;
21pub(crate) use crate::gguf_sharder::GgufTensorInfo;
22use log;
23#[cfg(not(target_arch = "wasm32"))]
24pub(crate) use memmap2::MmapOptions;
25pub(crate) use std::sync::Arc;
26
27pub use crate::ggml_quants::{ExecutionError, fetch_token_embedding};
28
29/// Dequantize a mmap embedding row into caller-supplied `out` (no heap allocation).
30pub fn dequantize_token_embedding_into(
31    raw: &[u8],
32    tensor: &GgufTensorInfo,
33    out: &mut [f32],
34) -> Result<usize, ExecutionError> {
35    let n_embd = tensor.dims[0] as usize;
36    if out.len() < n_embd {
37        return Err(ExecutionError::MmapBounds);
38    }
39    crate::ggml_quants::dequantize_row_into(raw, tensor.ggml_type, n_embd, out).map_err(|e| match e
40    {
41        crate::ggml_quants::GgmlDequantError::UnsupportedType => ExecutionError::UnsupportedType,
42        crate::ggml_quants::GgmlDequantError::BufferTooSmall
43        | crate::ggml_quants::GgmlDequantError::TruncatedInput => ExecutionError::MmapBounds,
44    })
45}
46
47/// Represents a Q4_K Quantized or standard float Tensor mapped from a monolithic GGUF file.
48#[derive(Debug, Clone)]
49pub struct QTensor {
50    pub shape: Vec<usize>,
51    pub byte_offset: u64,
52    pub is_quantized_q4_k: bool,
53}
54
55impl QTensor {
56    /// Creates a tensor descriptor from shape, byte offset into the GGUF, and a Q4_K quantization flag.
57    pub fn new(shape: Vec<usize>, byte_offset: u64, is_quantized_q4_k: bool) -> Self {
58        Self {
59            shape,
60            byte_offset,
61            is_quantized_q4_k,
62        }
63    }
64
65    /// Maps the tensor descriptor from a Quin pointer and a parsed `GgufTensorIndex`.
66    /// Resolves the genuine shape, quantization type, and payload offset by looking up
67    /// the tensor by its subject name hash or relative byte offset.
68    pub fn map_from_pointer_with_index(
69        quin: &NQuin,
70        index: &crate::gguf_sharder::GgufTensorIndex,
71    ) -> Option<Self> {
72        use crate::QuinPointerExt;
73
74        let flag = quin.extract_modality_flag();
75        if flag != crate::MODALITY_FLAG_LLM_TENSOR && flag != crate::MODALITY_FLAG_VISION_TENSOR {
76            return None;
77        }
78
79        let offset = quin.extract_byte_offset();
80        let subject_hash = quin.subject;
81
82        for (name_hash, info) in &index.entries {
83            if *name_hash == subject_hash || info.byte_offset == offset {
84                let n_dims = (info.n_dims as usize).min(4);
85                let shape: Vec<usize> = info.dims[..n_dims].iter().map(|&d| d as usize).collect();
86                let is_quantized_q4_k = info.ggml_type == 12 || info.ggml_type == 112;
87                let abs_offset = index.tensor_data_start.saturating_add(info.byte_offset);
88                return Some(Self::new(shape, abs_offset, is_quantized_q4_k));
89            }
90        }
91        None
92    }
93
94    /// Maps the tensor's byte offset from the GGUF via the 60-bit pointer.
95    /// Honestly resolves against the currently mounted resident model if available,
96    /// or fails closed (`None`) if no resident model is loaded or the tensor is not found.
97    pub fn map_from_pointer(quin: &NQuin) -> Option<Self> {
98        use crate::QuinPointerExt;
99
100        let flag = quin.extract_modality_flag();
101        if flag != crate::MODALITY_FLAG_LLM_TENSOR && flag != crate::MODALITY_FLAG_VISION_TENSOR {
102            return None;
103        }
104
105        #[cfg(not(target_arch = "wasm32"))]
106        {
107            if let Some(mmap) = crate::resident_model::current_resident_mmap() {
108                let index = crate::gguf_sharder::GgufTensorIndex::from_gguf(&mmap);
109                return Self::map_from_pointer_with_index(quin, &index);
110            }
111        }
112
113        None
114    }
115}
116
117// ── gguf_bridge library submodules (extracted from the former 9k-line monolith) ──
118// GPU uniform-buffer param structs (EmbeddingGpuParams / GemmGpuParams / AttentionGpuParams /
119// ElemGpuParams + ELEM_OP_* codes) and the quant-support gates now live in dedicated files.
120mod gpu_params;
121mod quant_support;
122#[cfg(feature = "gpu-runtime")]
123mod submission;
124pub(crate) use gpu_params::*;
125pub(crate) use quant_support::*;
126
127/// KV attention bitmask words uploaded to `fused_attention.wgsl` binding 5.
128pub const KV_ATTENTION_MASK_WORDS: usize = crate::compute_universe::KV_ATTENTION_MASK_WORDS;
129
130// ElemGpuParams + ELEM_OP_* codes moved to `gpu_params` (see submodule declarations above).
131
132/// FNV-1a hash for bind group cache keys.
133#[cfg(target_arch = "wasm32")]
134#[inline]
135pub(crate) fn mc8_bg_hash(parts: &[u64]) -> u64 {
136    etalon_inference::browser_prefill::encoding::bg_hash(parts)
137}
138
139/// MC8 Part 3s: WebGPU dynamic uniform offsets must be multiples of 256 bytes.
140#[cfg(target_arch = "wasm32")]
141pub(crate) const MC8_UNIFORM_ALIGN: usize = 256;
142#[cfg(target_arch = "wasm32")]
143pub(crate) const MC8_MAX_GEMM_UNIFORM_SLOTS: usize = 8;
144#[cfg(target_arch = "wasm32")]
145pub(crate) const MC8_MAX_ELEM_UNIFORM_SLOTS: usize = 8;
146#[cfg(target_arch = "wasm32")]
147pub(crate) const MC8_MAX_ATTN_UNIFORM_SLOTS: usize = 8;
148#[cfg(target_arch = "wasm32")]
149pub(crate) const MC8_MAX_ELEM_UNIFORM_LAYER_SLOTS: usize = MC8_MAX_ELEM_UNIFORM_SLOTS;
150/// MC8 Part 3v / Phase 5.4: layers encoded into one submit batch. Sizes the per-layer uniform
151/// buffers (slots_per_layer × this) and is the decode forward's chunk size — 64 → the whole
152/// ≤64-layer forward is a single submit. Per-chunk flush + reset handles deeper models.
153#[cfg(target_arch = "wasm32")]
154pub(crate) const MC8_LAYERS_PER_ENCODER: u32 = 64;
155/// Uniform slots reserved per layer within a chunk (must cover K/V/Q + tail).
156#[cfg(target_arch = "wasm32")]
157pub(crate) const MC8_ATTN_SLOTS_PER_LAYER: usize = 4;
158#[cfg(target_arch = "wasm32")]
159pub(crate) const MC8_ELEM_SLOTS_PER_LAYER: usize = 6;
160#[cfg(target_arch = "wasm32")]
161pub(crate) const MC8_GEMM_SLOTS_PER_LAYER: usize = 8; // o, gate, up, down + Phase 5.5 Q/K/V projection
162#[cfg(target_arch = "wasm32")]
163pub(crate) const MC8_MAX_ATTN_UNIFORM_CHUNK_SLOTS: usize =
164    MC8_ATTN_SLOTS_PER_LAYER * MC8_LAYERS_PER_ENCODER as usize;
165#[cfg(target_arch = "wasm32")]
166pub(crate) const MC8_MAX_ELEM_UNIFORM_CHUNK_SLOTS: usize =
167    MC8_ELEM_SLOTS_PER_LAYER * MC8_LAYERS_PER_ENCODER as usize;
168#[cfg(target_arch = "wasm32")]
169pub(crate) const MC8_MAX_GEMM_UNIFORM_CHUNK_SLOTS: usize =
170    MC8_GEMM_SLOTS_PER_LAYER * MC8_LAYERS_PER_ENCODER as usize;
171
172/// Part 3v: absolute uniform slot cursors within one encoder chunk.
173#[cfg(target_arch = "wasm32")]
174pub(crate) use etalon_inference::browser_prefill::types::Mc8ChunkUniformCursors;
175
176/// MC8 Part 3t: disjoint weight staging — eliminates `write_buffer` races within one layer submit.
177#[cfg(target_arch = "wasm32")]
178pub(crate) use etalon_inference::browser_prefill::types::Mc8WeightRole;
179
180/// One buffer per GEMM role so mid-layer weight uploads never clobber in-flight dispatches.
181#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
182pub(crate) struct Mc8WeightArenaBufs {
183    qkv_k: etalon::Buffer,
184    qkv_v: etalon::Buffer,
185    qkv_q: etalon::Buffer,
186    o_proj: etalon::Buffer,
187    gate: etalon::Buffer,
188    up: etalon::Buffer,
189    down: etalon::Buffer,
190}
191
192/// Part 3u: dynamic offsets for one full prefill layer (staged before encoder dispatches).
193#[cfg(target_arch = "wasm32")]
194pub(crate) use etalon_inference::browser_prefill::types::Mc8PrefillLayerUniforms;
195
196/// Strided work-buffer geometry shared by layer dispatches.
197#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
198pub(crate) use etalon_inference::browser_prefill::types::Mc8PrefillLayerGeom;
199
200/// Stack arena for batched uniform uploads (one `write_buffer` per layer section).
201#[cfg(target_arch = "wasm32")]
202pub(crate) use etalon_inference::browser_prefill::types::Mc8UniformArena;
203
204/// Part 3u: larger elem arena for full-layer super-staging.
205#[cfg(target_arch = "wasm32")]
206pub(crate) use etalon_inference::browser_prefill::types::Mc8ElemUniformArena;
207
208#[cfg(target_arch = "wasm32")]
209pub(crate) use etalon_inference::browser_prefill::types::Mc8AttnUniformArena;
210
211/// MC8: accumulates compute passes; submit + map_async only at pipeline boundary.
212#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
213pub struct WasmGpuPipeline {
214    encoder: etalon::CommandEncoder,
215}
216
217// ggml_gpu_quant_supported / ggml_gpu_attention_shader_supported / ggml_gpu_gemm_supported moved to
218// the `quant_support` submodule (declared above; re-exported via `pub(crate) use quant_support::*`).
219
220/// Await `map_async` without `poll(Wait)` — yields to the browser event loop (MC6).
221#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
222pub(crate) async fn await_wgpu_map(slice: etalon::BufferSlice<'_>) -> bool {
223    etalon_wasm::readback::await_map(slice, etalon_wasm::readback::browser_worker()).await
224}
225
226#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
227impl WasmGpuPipeline {
228    /// Starts a WASM compute-pipeline encoding session for `engine`.
229    pub(crate) fn begin(engine: &QTensorEngine) -> Self {
230        Self {
231            encoder: engine
232                .device()
233                .create_command_encoder(&etalon::CommandEncoderDescriptor {
234                    label: Some("MC8FusedEncoder"),
235                }),
236        }
237    }
238
239    /// Finishes encoding and returns the command buffer.
240    pub(crate) fn finish(self) -> etalon::CommandBuffer {
241        self.encoder.finish()
242    }
243}
244
245// WASM-only MC8 GPU engine methods (resident weight arena, fused-encoder prefill/decode, async
246// readback) carved into the `mc8_wasm` submodule. cfg-gated so native never compiles it.
247#[cfg(target_arch = "wasm32")]
248mod mc8_wasm;
249
250/// Hard context ceiling — sized to keep KV arena under the 512MB RAM floor (Gemma 42L).
251pub const MAX_CONTEXT_WINDOW: u32 = 1024;
252/// Maximum bytes for the static KV arena (load-time allocation only).
253pub const KV_CACHE_MAX_BYTES: usize = 768 * 1024 * 1024;
254
255/// Static ring-buffer KV layout: `[layer][slot][K | V]` in f32, OR (W5a int8 mode) packed int8 +
256/// per-(slot,kv_head) f32 scale in the same 4-byte-element buffer. `total_f32_elems` counts 4-byte
257/// slots either way (u32/f32 share the size), so the allocation math is unchanged.
258#[derive(Debug, Clone, Copy, PartialEq, Eq)]
259pub struct KvCacheLayout {
260    pub max_context: u32,
261    pub n_layer: u32,
262    pub n_kv_head: u32,
263    pub head_dim: u32,
264    pub slot_kv_elems: u32,
265    pub layer_stride: u32,
266    pub total_f32_elems: usize,
267    /// W5a: true ⇒ the arena is int8-quantized (K/V stored as packed i8 lanes + one f32 scale per
268    /// (slot, kv_head), K then V). `layer_stride` is then the int8 slot layout, not the f32 one.
269    pub int8: bool,
270    /// W5b Phase 4b: `> 0` ⇒ the arena stores k-sparse **dictionary codes** (`dict_k` code-words per
271    /// K/V vector — each word packs `u16 atom-index | f16 coefficient`), reconstructed in the attention
272    /// shader. Mutually exclusive with `int8` (dict mode wins). `0` ⇒ f32/int8 as above.
273    pub dict_k: u32,
274    /// W5b Phase 4b: dictionary size (atoms) — the atoms live in each layer's arena AFTER the code
275    /// region (`[K atoms n_atoms×head_dim][V atoms …]`), so the per-layer binding covers them.
276    pub dict_n_atoms: u32,
277}
278
279impl KvCacheLayout {
280    /// Derives the KV-cache ring layout from GGUF hyperparameters; `None` when fields needed are absent/zero.
281    pub fn from_hyperparams(h: &crate::gguf_sharder::GgufHyperparams) -> Option<Self> {
282        // W5b Phase 4b: sparse-dictionary KV takes precedence over int8/f32 — but only when its toggle is
283        // on AND a certified dictionary is installed whose head_dim matches this model. Otherwise it
284        // transparently falls back (dict_k = 0).
285        #[cfg(not(target_arch = "wasm32"))]
286        let (dict_k, dict_n_atoms) = if crate::llm_bench::kv_dict_enabled() {
287            crate::kv_dict_runtime::installed_meta()
288                .filter(|&(_, hd, _)| hd == h.head_dim() as usize)
289                .map(|(k, _, na)| (k as u32, na as u32))
290                .unwrap_or((0, 0))
291        } else {
292            (0, 0)
293        };
294        #[cfg(target_arch = "wasm32")]
295        let (dict_k, dict_n_atoms) = (0u32, 0u32);
296
297        // W5a int8 KV is a native decode-path optimization, gated behind its own toggle and only when
298        // head_dim packs cleanly into u32 lanes. WASM always uses the f32 layout.
299        #[cfg(not(target_arch = "wasm32"))]
300        let want_int8 = dict_k == 0
301            && crate::llm_bench::kv_int8_enabled()
302            && (h.head_dim() % 4 == 0)
303            && h.head_dim() > 0;
304        #[cfg(target_arch = "wasm32")]
305        let want_int8 = false;
306        Self::from_hyperparams_mode(h, want_int8, dict_k, dict_n_atoms)
307    }
308
309    fn from_hyperparams_mode(
310        h: &crate::gguf_sharder::GgufHyperparams,
311        int8: bool,
312        dict_k: u32,
313        dict_n_atoms: u32,
314    ) -> Option<Self> {
315        let n_layer = h.n_layer;
316        let n_kv_head = h.effective_n_kv_head();
317        let head_dim = h.head_dim();
318        if n_layer == 0 || n_kv_head == 0 || head_dim == 0 {
319            return None;
320        }
321        // dict mode wins over int8.
322        let int8 = int8 && dict_k == 0;
323        let slot_kv_elems = n_kv_head * head_dim;
324        // f32:  per slot = 2·n_kv_head·head_dim 4-byte elems (K then V).
325        // int8: per slot = 2·n_kv_head·(1 scale + head_dim/4 packed words), K then V — ~3.8× smaller.
326        // dict: per slot = 2·n_kv_head·dict_k code-words (K then V) — each word = u16 index | f16 coeff.
327        let layer_stride = if dict_k > 0 {
328            // codes ([max_context][2 streams][n_kv_head][dict_k]) + the layer's dictionary atoms
329            // ([2 streams][dict_n_atoms][head_dim]) resident in the same slice for the shader.
330            MAX_CONTEXT_WINDOW * 2 * n_kv_head * dict_k + 2 * dict_n_atoms * head_dim
331        } else if int8 {
332            MAX_CONTEXT_WINDOW * 2 * n_kv_head * (1 + head_dim / 4)
333        } else {
334            MAX_CONTEXT_WINDOW * slot_kv_elems * 2
335        };
336        let total = (n_layer as usize).checked_mul(layer_stride as usize)?;
337        let bytes = total.checked_mul(std::mem::size_of::<f32>())?;
338        if bytes > KV_CACHE_MAX_BYTES {
339            return None;
340        }
341        Some(Self {
342            max_context: MAX_CONTEXT_WINDOW,
343            n_layer,
344            n_kv_head,
345            head_dim,
346            slot_kv_elems,
347            layer_stride,
348            total_f32_elems: total,
349            int8,
350            dict_k,
351            dict_n_atoms,
352        })
353    }
354
355    /// Derive the dense f32 device layout used by native CUDA kernels without changing the host
356    /// cache representation. Host int8/dictionary compression and the CUDA execution arena are
357    /// independent storage decisions.
358    #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
359    pub(crate) fn dense_device_layout(self) -> Option<Self> {
360        let slot_kv_elems = self.n_kv_head.checked_mul(self.head_dim)?;
361        let layer_stride = self
362            .max_context
363            .checked_mul(slot_kv_elems)?
364            .checked_mul(2)?;
365        let total_f32_elems = (self.n_layer as usize).checked_mul(layer_stride as usize)?;
366        total_f32_elems
367            .checked_mul(std::mem::size_of::<f32>())
368            .filter(|bytes| *bytes <= KV_CACHE_MAX_BYTES)?;
369        Some(Self {
370            slot_kv_elems,
371            layer_stride,
372            total_f32_elems,
373            int8: false,
374            dict_k: 0,
375            dict_n_atoms: 0,
376            ..self
377        })
378    }
379
380    #[inline]
381    /// Ring-buffer slot for `token_idx` modulo capacity.
382    pub fn ring_slot(&self, token_idx: u32) -> u32 {
383        token_idx % self.max_context
384    }
385
386    #[inline]
387    /// Flat K-cache index for `(layer, slot, kv_head, dim)`.
388    pub fn k_index(&self, layer: u32, slot: u32, kv_head: u32, dim: u32) -> usize {
389        let base = layer as usize * self.layer_stride as usize
390            + slot as usize * self.slot_kv_elems as usize * 2;
391        base + kv_head as usize * self.head_dim as usize + dim as usize
392    }
393
394    #[inline]
395    /// Flat V-cache index for `(layer, slot, kv_head, dim)`.
396    pub fn v_index(&self, layer: u32, slot: u32, kv_head: u32, dim: u32) -> usize {
397        let k_base = layer as usize * self.layer_stride as usize
398            + slot as usize * self.slot_kv_elems as usize * 2;
399        let v_off = self.n_kv_head as usize * self.head_dim as usize;
400        k_base + v_off + kv_head as usize * self.head_dim as usize + dim as usize
401    }
402
403    /// W5b Phase 4b (dict mode): word offset of the `i`-th code word (`0..dict_k`) for the K
404    /// (`k_not_v = true`) or V vector of `(layer, slot, kv_head)`. Each code word packs
405    /// `u16 atom-index (high 16) | f16 coefficient (low 16)`. Slot layout mirrors f32 (K region then V
406    /// region), but each vector is `dict_k` words instead of `head_dim` floats.
407    #[inline]
408    pub fn code_index(&self, layer: u32, slot: u32, kv_head: u32, k_not_v: bool, i: u32) -> usize {
409        let dk = self.dict_k as usize;
410        let per_slot = 2 * self.n_kv_head as usize * dk;
411        let base = layer as usize * self.layer_stride as usize + slot as usize * per_slot;
412        let stream_off = if k_not_v {
413            0
414        } else {
415            self.n_kv_head as usize * dk
416        };
417        base + stream_off + kv_head as usize * dk + i as usize
418    }
419}
420
421/// Max GEMM row/column for stack buffers and reusable GPU staging (Gemma 4 E4B double-wide FFN = 8×2560 = 20480).
422const MAX_STACK_GEMM_DIM: usize = 20480;
423const MAX_STACK_GEMM_OUT: usize = MAX_STACK_GEMM_DIM;
424const MAX_STACK_GEMM_IN: usize = MAX_STACK_GEMM_DIM;
425/// Stack scratch for pre-norm hidden (SmolLM2 n_embd=960; cap supports Gemma-class models).
426const MAX_HIDDEN_DIM: usize = 4096;
427/// RMSNorm epsilon when GGUF KV does not expose `rms_norm_eps` (Llama/SmolLM default).
428pub(crate) const RMS_NORM_EPS: f32 = 1e-5;
429/// Prompt tokens per prefill GPU batch (stack + staging footprint = `emb_dim ×` this).
430pub const PREFILL_CHUNK_SIZE: usize = 64;
431/// Per-token KV masks uploaded for batched Q-SDPA (`PREFILL_CHUNK_SIZE × mask words`).
432const MAX_ATTN_MASK_UPLOAD_WORDS: usize = PREFILL_CHUNK_SIZE * KV_ATTENTION_MASK_WORDS;
433/// Max stacked embedding floats in a prefill chunk (`10240 × 64`).
434pub const MAX_PREFILL_BATCH_FLOATS: usize = 10240 * PREFILL_CHUNK_SIZE;
435/// `llm_agent` stack chunk buffer (Gemma 2560 × 64).
436pub const PREFILL_CHUNK_STACK_FLOATS: usize = 2560 * PREFILL_CHUNK_SIZE;
437/// wgpu default max buffer size on many drivers (256 MiB).
438const MAX_WGPU_WEIGHT_STAGING: usize = 64 * 1024 * 1024;
439/// Vocabulary projection rows per chunked logits sweep.
440/// 10240 is the native GEMM output-buffer ceiling and a 256-row multiple, so
441/// resident logits chunk offsets stay storage-binding aligned while the current
442/// 49k-vocab model drops from six output chunks/token to five.
443pub const VOCAB_CHUNK_ROWS: usize = 10240;
444
445/// Streaming argmax result across chunked vocabulary projection.
446pub use etalon_inference::projections::StreamingArgmaxResult;
447
448#[derive(Debug, Clone, Copy, PartialEq, Eq)]
449pub struct GgufLoadReport {
450    pub mapped_bytes: u64,
451    pub tensor_data_offset: u64,
452    pub n_layer: u32,
453    pub n_head: u32,
454    pub n_kv_head: u32,
455    pub max_tensor_bytes: usize,
456    pub kv_cache_bytes: u64,
457    pub directml_enabled: bool,
458}
459
460// CPU numeric kernels + pre-norm helpers (bytes_to_gib / scrub_f32_volatile / update_streaming_argmax
461// [+sieved] / relu / silu / add_residual / rms_norm / dequant_norm_row_into / prepare_pre_norm_input)
462// moved to the `cpu_ops` submodule (declared below; re-exported via `pub(crate) use cpu_ops::*`).
463mod cpu_ops;
464pub(crate) use cpu_ops::*;
465#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
466mod pipeline_cache;
467/// Prepared CPU execution floor for browser WASM. This backend owns no wgpu
468/// objects and remains available when the browser exposes no WebGPU adapter.
469pub mod wasm_cpu;
470#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
471pub(crate) use pipeline_cache::*;
472
473// Concern submodules — each holds an `impl QTensorEngine` block for one hot-path area. Methods are
474// pub(crate) so they call across modules freely; types/imports arrive via each file's `use super::*`.
475mod async_dispatch;
476mod attention;
477mod attention_adapter;
478#[cfg(target_arch = "wasm32")]
479mod browser;
480#[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
481mod cuda_decode_plan;
482#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
483mod native_attention_adapter;
484#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
485mod native_resident_adapter;
486#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
487mod native_resident_planning_adapter;
488mod projection_adapter;
489#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
490mod projection_gpu_adapter;
491/// Hard cap on the KV context window a decode plan may request. Declared here rather than in the
492/// `cuda`-gated plan module because the raw-decode harness and the mega-pass guard validate
493/// against it on every target, CUDA or not.
494#[cfg(not(target_arch = "wasm32"))]
495pub(crate) const MAX_CUDA_CONTEXT_WINDOW: u32 = 4096;
496mod embedding;
497mod ffn;
498mod forward;
499#[cfg(not(target_arch = "wasm32"))]
500mod gemma4;
501mod gemma4_qwen35;
502mod moe_ffn;
503mod moe_gguf;
504mod ssm_forward;
505pub use etalon_inference::ordinary::ssm_layout::SsmStateError;
506/// Conservative legacy default; callers may admit larger external model state.
507pub const DEFAULT_SSM_STATE_BYTE_BUDGET: usize = 128 * 1024 * 1024;
508#[cfg(not(target_arch = "wasm32"))]
509pub(crate) use forward::PrefillDispatchFailure;
510mod gemm;
511mod init;
512mod load;
513mod output;
514#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
515mod prefill_arena;
516mod prefill_async;
517#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
518mod resident_decode;
519#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
520mod verify_arena;
521/// Cooperative browser yields + init-status for WASM LLM boot (phones).
522#[cfg(target_arch = "wasm32")]
523pub(crate) mod wasm_yield;
524
525/// MC8 pt3e: max abs error over the first `n` elements.
526#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
527fn probe_max_abs_diff(a: &[f32], b: &[f32], n: usize) -> f32 {
528    let n = n.min(a.len()).min(b.len());
529    let mut m = 0.0f32;
530    for i in 0..n {
531        m = m.max((a[i] - b[i]).abs());
532    }
533    m
534}
535
536#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
537fn probe_log_diff(phase: &str, cpu: &[f32], gpu: &[f32], n: usize) {
538    let n = n.min(8).min(cpu.len()).min(gpu.len());
539    if n == 0 {
540        return;
541    }
542    let err = probe_max_abs_diff(cpu, gpu, n);
543    wlog(&format!(
544        "[MC8 L0 diff] {phase}: cpu[0]={:.6} gpu[0]={:.6} max_abs_err={:.6}",
545        cpu[0], gpu[0], err
546    ));
547}
548
549#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
550fn probe_log_mid_diff(phase: &str, cpu: &[f32], gpu: &[f32], n: usize) {
551    let n = n.min(8).min(cpu.len()).min(gpu.len());
552    if n == 0 {
553        return;
554    }
555    let err = probe_max_abs_diff(cpu, gpu, n);
556    wlog(&format!(
557        "[MC8 L0 mid] {phase}: cpu[0]={:.6} gpu[0]={:.6} max_abs_err={:.6}",
558        cpu[0], gpu[0], err
559    ));
560}
561
562#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
563fn probe_log_ffn_diff(phase: &str, cpu: &[f32], gpu: &[f32], n: usize) {
564    let n = n.min(8).min(cpu.len()).min(gpu.len());
565    if n == 0 {
566        return;
567    }
568    let err = probe_max_abs_diff(cpu, gpu, n);
569    wlog(&format!(
570        "[MC8 L0 ffn] {phase}: cpu[0]={:.6} gpu[0]={:.6} max_abs_err={:.6}",
571        cpu[0], gpu[0], err
572    ));
573}
574
575/// MC8 pt3g: CPU SwiGLU stages from post-attn hidden @ L0.
576#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
577async fn mc8_cpu_l0_ffn_stages(
578    engine: &QTensorEngine,
579    index: &crate::gguf_sharder::GgufTensorIndex,
580    mmap: &[u8],
581    layout: &KvCacheLayout,
582    hidden_cpu: &[f32],
583    n_embd: usize,
584    token_idx: u32,
585    post_attn: &mut [f32],
586    ffn_input: &mut [f32],
587    gate: &mut [f32],
588    up: &mut [f32],
589    swiglu: &mut [f32],
590    down: &mut [f32],
591) -> Option<usize> {
592    let tensors = index.get_layer_tensors(0);
593    let mut attn_out = [0f32; MAX_HIDDEN_DIM];
594    let q_dim = mc8_cpu_l0_attn_out(
595        engine,
596        index,
597        mmap,
598        layout,
599        hidden_cpu,
600        n_embd,
601        token_idx,
602        &mut attn_out,
603    )
604    .await?;
605    let out_info = tensors.attn_output.as_ref()?;
606    let o_raw =
607        crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, out_info).ok()?;
608    let (o_in, _) = QTensorEngine::matmul_dims(out_info);
609    if o_in > q_dim {
610        return None;
611    }
612    let mut o_proj = [0f32; MAX_HIDDEN_DIM];
613    if !stack_gemm_quant(
614        o_raw,
615        out_info,
616        &attn_out[..q_dim],
617        &mut o_proj[..n_embd],
618        o_in,
619        n_embd,
620    ) {
621        return None;
622    }
623    for i in 0..n_embd {
624        post_attn[i] = hidden_cpu[i] + o_proj[i];
625    }
626    let mut norm_w = [0f32; MAX_HIDDEN_DIM];
627    let mut ffn_norm_scratch = [0f32; MAX_HIDDEN_DIM];
628    let normed = prepare_pre_norm_input(
629        &post_attn[..n_embd],
630        n_embd,
631        tensors.ffn_norm.as_ref(),
632        Some(mmap),
633        index.tensor_data_start,
634        &mut ffn_norm_scratch,
635        &mut norm_w,
636    );
637    ffn_input[..n_embd].copy_from_slice(normed);
638    let gate_info = tensors.ffn_gate.as_ref()?;
639    let up_info = tensors.ffn_up.as_ref()?;
640    let down_info = tensors.ffn_down.as_ref()?;
641    let gate_raw =
642        crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, gate_info).ok()?;
643    let up_raw =
644        crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, up_info).ok()?;
645    let down_raw =
646        crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, down_info).ok()?;
647    let (gate_in, n_ffn) = QTensorEngine::matmul_dims(gate_info);
648    let (up_in, up_out) = QTensorEngine::matmul_dims(up_info);
649    let (dn_in, dn_out) = QTensorEngine::matmul_dims(down_info);
650    if gate_in > n_embd
651        || up_in != gate_in
652        || up_out != n_ffn
653        || dn_in != n_ffn
654        || n_ffn > gate.len()
655        || dn_out < n_embd
656    {
657        return None;
658    }
659    if !stack_gemm_quant(
660        gate_raw,
661        gate_info,
662        normed,
663        &mut gate[..n_ffn],
664        gate_in,
665        n_ffn,
666    ) {
667        return None;
668    }
669    if !stack_gemm_quant(up_raw, up_info, normed, &mut up[..n_ffn], up_in, n_ffn) {
670        return None;
671    }
672    for i in 0..n_ffn {
673        let g = gate[i];
674        let silu = g / (1.0 + (-g).exp());
675        swiglu[i] = silu * up[i];
676    }
677    if !stack_gemm_quant(
678        down_raw,
679        down_info,
680        &swiglu[..dn_in],
681        &mut down[..n_embd],
682        dn_in,
683        n_embd,
684    ) {
685        return None;
686    }
687    Some(n_ffn)
688}
689
690/// Read one KV head from the CPU mirror arena.
691#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
692fn read_kv_cpu_head(
693    layout: &KvCacheLayout,
694    kv: &[f32],
695    layer: u32,
696    token_pos: u32,
697    kv_h: u32,
698    head_dim: usize,
699    k_not_v: bool,
700    out: &mut [f32],
701) -> bool {
702    if head_dim == 0 || head_dim > out.len() {
703        return false;
704    }
705    let slot = layout.ring_slot(token_pos);
706    for d in 0..head_dim {
707        let idx = if k_not_v {
708            layout.k_index(layer, slot, kv_h, d as u32)
709        } else {
710            layout.v_index(layer, slot, kv_h, d as u32)
711        };
712        if idx >= kv.len() {
713            return false;
714        }
715        out[d] = kv[idx];
716    }
717    true
718}
719
720#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
721fn probe_log_prefill_diff(phase: &str, cpu: &[f32], gpu: &[f32], n: usize) {
722    let n = n.min(8).min(cpu.len()).min(gpu.len());
723    if n == 0 {
724        return;
725    }
726    let err = probe_max_abs_diff(cpu, gpu, n);
727    wlog(&format!(
728        "[MC8 prefill] {phase}: cpu[0]={:.6} gpu[0]={:.6} max_abs_err={:.6}",
729        cpu[0], gpu[0], err
730    ));
731}
732
733/// MC8 pt3f: CPU SDPA @ L0 → full `q_dim` Attn_Out (async KV readback).
734#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
735async fn mc8_cpu_l0_attn_out(
736    engine: &QTensorEngine,
737    index: &crate::gguf_sharder::GgufTensorIndex,
738    mmap: &[u8],
739    layout: &KvCacheLayout,
740    hidden_cpu: &[f32],
741    n_embd: usize,
742    token_idx: u32,
743    out: &mut [f32],
744) -> Option<usize> {
745    let h = &index.hyperparams;
746    let tensors = index.get_layer_tensors(0);
747    let head_dim = h.head_dim() as usize;
748    let n_head = h.n_head as usize;
749    let q_heads_per_kv = h.q_heads_per_kv() as usize;
750    if head_dim == 0 || n_head == 0 || q_heads_per_kv == 0 {
751        return None;
752    }
753    let q_dim = n_head * head_dim;
754    if q_dim > out.len() {
755        return None;
756    }
757    let mut norm_w = [0f32; MAX_HIDDEN_DIM];
758    let mut norm_cpu = [0f32; MAX_HIDDEN_DIM];
759    let mut proj = [0f32; MAX_STACK_GEMM_OUT];
760    let normed = prepare_pre_norm_input(
761        &hidden_cpu[..n_embd],
762        n_embd,
763        tensors.attn_norm.as_ref(),
764        Some(mmap),
765        index.tensor_data_start,
766        &mut norm_cpu,
767        &mut norm_w,
768    );
769    let q_info = tensors.attn_q.as_ref()?;
770    let q_raw =
771        crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, q_info).ok()?;
772    let (q_in, q_out) = QTensorEngine::matmul_dims(q_info);
773    if q_out != q_dim || !stack_gemm_quant(q_raw, q_info, normed, &mut proj[..q_out], q_in, q_out) {
774        return None;
775    }
776    rope_inplace(
777        &mut proj[..q_out],
778        n_head,
779        head_dim,
780        token_idx,
781        h.effective_rope_freq_base(),
782        h.effective_rope_scale(),
783    );
784    let scale = 1.0f32 / (head_dim as f32).sqrt();
785    let mut k_slot = [0f32; 128];
786    let mut v_slot = [0f32; 128];
787    if head_dim > k_slot.len() {
788        return None;
789    }
790    for qh in 0..n_head {
791        let kv_h = qh / q_heads_per_kv;
792        let q_off = qh * head_dim;
793        let mut att_scores = [0f32; MAX_CONTEXT_WINDOW as usize];
794        let mut max_score = f32::NEG_INFINITY;
795        for past_pos in 0..=token_idx {
796            let past_slot = layout.ring_slot(past_pos);
797            if !engine
798                .pipeline_read_kv_head(
799                    layout,
800                    0,
801                    past_slot,
802                    kv_h as u32,
803                    head_dim,
804                    true,
805                    &mut k_slot,
806                )
807                .await
808            {
809                return None;
810            }
811            let mut dot = 0.0f32;
812            for d in 0..head_dim {
813                dot += proj[q_off + d] * k_slot[d];
814            }
815            let score = dot * scale;
816            att_scores[past_pos as usize] = score;
817            max_score = max_score.max(score);
818        }
819        let mut sum_exp = 0.0f32;
820        for past_pos in 0..=token_idx {
821            let exp_val = (att_scores[past_pos as usize] - max_score).exp();
822            att_scores[past_pos as usize] = exp_val;
823            sum_exp += exp_val;
824        }
825        if sum_exp > 0.0 {
826            for past_pos in 0..=token_idx {
827                let prob = att_scores[past_pos as usize] / sum_exp;
828                let past_slot = layout.ring_slot(past_pos);
829                if !engine
830                    .pipeline_read_kv_head(
831                        layout,
832                        0,
833                        past_slot,
834                        kv_h as u32,
835                        head_dim,
836                        false,
837                        &mut v_slot,
838                    )
839                    .await
840                {
841                    return None;
842                }
843                for d in 0..head_dim {
844                    out[q_off + d] += v_slot[d] * prob;
845                }
846            }
847        }
848    }
849    Some(q_dim)
850}
851
852// --- PHASE 1 WASM OOB DIAGNOSTIC INSTRUMENTATION (remove once the trap is fixed) ---
853#[cfg(target_arch = "wasm32")]
854#[inline]
855/// Emits a log line to the WASM console (or no-op off-wasm).
856pub(crate) fn wlog(s: &str) {
857    web_sys::console::log_1(&wasm_bindgen::JsValue::from_str(s));
858}
859#[cfg(not(target_arch = "wasm32"))]
860#[inline]
861/// Emits a log line to the WASM console (or no-op off-wasm).
862pub(crate) fn wlog(s: &str) {
863    log::warn!("{s}");
864}
865
866/// Decode-profiler: count of GPU `submit → poll(Maintain::Wait)` round-trips. Incremented by
867/// `QTensorEngine::poll_wait` (every native blocking sync point routes through it); read/reset by
868/// the bench to derive per-token synchronization overhead.
869pub static GPU_WAIT_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
870
871/// Total native GPU blocking-wait round-trips since the last reset.
872#[inline]
873pub fn gpu_wait_count() -> u64 {
874    GPU_WAIT_COUNT.load(std::sync::atomic::Ordering::Relaxed)
875}
876
877/// Reset the GPU blocking-wait counter before a measured run.
878#[inline]
879pub fn reset_gpu_wait_count() {
880    GPU_WAIT_COUNT.store(0, std::sync::atomic::Ordering::Relaxed);
881}
882
883/// In-place NEOX-style RoPE over `n_heads` consecutive `head_dim` blocks of `vec`.
884/// Rotates split-half pairs `(i, i + head_dim/2)` — required for Llama/SmolLM2 GGUF weights.
885/// (`fused_attention.wgsl` mirrors this NEOX split-half layout since MC8 Part 2.)
886use etalon_inference::ordinary::rope::rope_inplace;
887
888/// STEM-grounding proof: the LLM's `rope_inplace` is the 2-D rotation defined in
889/// `solvers::rope` — RoPE is trigonometry (a rotation per dimension pair), not a proprietary
890/// operation. The inline `f32` kernel is checked against the `f64` STEM definition.
891#[cfg(test)]
892mod rope_stem_parity_tests {
893    #[test]
894    fn rope_kernel_is_the_stem_rotation() {
895        let n_heads = 2usize;
896        let head_dim = 8usize;
897        let (pos, base, scale) = (7u32, 10000.0f32, 1.0f32);
898        let xs: Vec<f32> = (0..n_heads * head_dim)
899            .map(|i| (i as f32 - 8.0) * 0.25)
900            .collect();
901
902        let mut got = xs.clone();
903        super::rope_inplace(&mut got, n_heads, head_dim, pos, base, scale);
904
905        let mut want: Vec<f64> = xs.iter().map(|&v| v as f64).collect();
906        crate::solvers::rope::rope_interleaved(
907            &mut want,
908            n_heads,
909            head_dim,
910            pos as f64,
911            base as f64,
912            scale as f64,
913        );
914
915        for i in 0..xs.len() {
916            assert!(
917                (got[i] as f64 - want[i]).abs() < 1e-4,
918                "RoPE kernel diverges from solvers::rope at {i}: {} vs {}",
919                got[i],
920                want[i]
921            );
922        }
923    }
924}
925
926/// Zero-heap CPU GEMM: `out[i] = dot(weight_row(i), input)` with per-row dequant.
927/// `pub(crate)` so toolkit/parity probes can exercise the same kernel as the hot path.
928pub(crate) fn stack_gemm_quant(
929    raw: &[u8],
930    info: &GgufTensorInfo,
931    input: &[f32],
932    out: &mut [f32],
933    n_in: usize,
934    n_out: usize,
935) -> bool {
936    if n_in > input.len() || n_out > out.len() || n_in > MAX_STACK_GEMM_IN {
937        wlog(&format!(
938            "[stack_gemm] GUARD tripped n_in={n_in} n_out={n_out} input={} out={} MAX_IN={MAX_STACK_GEMM_IN}",
939            input.len(),
940            out.len()
941        ));
942        return false;
943    }
944    #[cfg(target_arch = "wasm32")]
945    if info.ggml_type == crate::ggml_quants::GGML_TYPE_Q8_0 {
946        return wasm_cpu::q8_0_gemv_into(raw, input, out, n_in, n_out);
947    }
948    let mut row = [0f32; MAX_STACK_GEMM_IN];
949    for i in 0..n_out {
950        if crate::ggml_quants::dequant_matrix_row_into(raw, info, i, &mut row[..n_in]).unwrap_or(0)
951            < n_in
952        {
953            return false;
954        }
955        out[i] = row[..n_in]
956            .iter()
957            .zip(&input[..n_in])
958            .map(|(w, x)| w * x)
959            .sum();
960    }
961    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
962    etalon_agent::provider_scope::record_route(1);
963    true
964}
965
966mod state;
967pub use state::QTensorEngine;
968#[cfg(not(target_arch = "wasm32"))]
969mod provider;
970#[cfg(not(target_arch = "wasm32"))]
971pub use provider::{NativeProviderScope, NativeProviderSnapshot};
972mod model_bytes;
973#[cfg(not(target_arch = "wasm32"))]
974pub mod qwen35;
975#[cfg(not(target_arch = "wasm32"))]
976pub(crate) use model_bytes::ModelByteHandle;
977#[cfg(not(target_arch = "wasm32"))]
978pub(crate) use model_bytes::NativeModelSources;
979pub(crate) mod execution_env;
980
981#[cfg(target_arch = "wasm32")]
982thread_local! {
983    pub static WASM_ENGINE_INSTANCE: std::cell::RefCell<Option<QTensorEngine>> = std::cell::RefCell::new(None);
984}
985
986/// Boot the resident WASM WebGPU engine from GGUF or P64 bytes.
987///
988/// Uses the **original sync adopt path** (full weight + logits + norm upload) so
989/// decode stays coherent. Yields only between major phases so the UI can paint
990/// status; weight upload itself is intentionally one blocking stretch with a
991/// clear status line first (phones will freeze briefly — that is correct).
992#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
993pub async fn initialize_webgpu_engine(model_data: std::sync::Arc<[u8]>) -> Result<(), String> {
994    let stamp = etalon_wasm::readback::begin_browser_model()
995        .map_err(|e| format!("Browser model context: {e:?}"))?;
996    initialize_webgpu_engine_with_stamp(model_data, stamp).await
997}
998
999#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
1000pub(crate) async fn initialize_webgpu_engine_with_stamp(
1001    model_data: std::sync::Arc<[u8]>,
1002    stamp: etalon_wasm::worker_steward::CompletionStamp,
1003) -> Result<(), String> {
1004    use wasm_yield::{clear_init_status, phase, set_init_status};
1005
1006    {
1007        let worker = etalon_wasm::readback::browser_worker();
1008        let worker = worker.lock().unwrap();
1009        if worker.completion_stamp() != stamp || !worker.is_ready() {
1010            return Err("Browser model initialization superseded or cancelled".into());
1011        }
1012        WASM_ENGINE_INSTANCE.with(|slot| *slot.borrow_mut() = None);
1013    }
1014
1015    // Use `try_new()` (not `new_async()`) so a missing/incompatible WebGPU adapter
1016    // surfaces as a rejected promise the JS layer can display, rather than an
1017    // `.expect()` panic that aborts the wasm module and leaves the init promise
1018    // pending forever (the "stuck on Initialising…" hang).
1019    phase("Requesting WebGPU adapter + device…").await;
1020    let mut engine = QTensorEngine::try_new().await?;
1021    phase("WebGPU device + pipelines ready — loading model (weights may take 1–3 min)…").await;
1022
1023    // Dual-format boot gate. P64 owns the canonical lowercase four-byte
1024    // `p64\0` magic. Sync adopt is the proven coherent path (do not defer uploads).
1025    if crate::p64_weight::has_p64_magic(&model_data) {
1026        set_init_status(format!(
1027            "Loading P64 + uploading GPU weights ({:.0} MB)…",
1028            model_data.len() as f64 / (1024.0 * 1024.0)
1029        ));
1030        // One yield so the status line paints before the long sync upload freezes the tab.
1031        wasm_yield::yield_to_browser().await;
1032        engine.adopt_resident_p64(model_data)?;
1033    } else {
1034        set_init_status(format!(
1035            "Loading GGUF + uploading GPU weights ({:.0} MB)…",
1036            model_data.len() as f64 / (1024.0 * 1024.0)
1037        ));
1038        wasm_yield::yield_to_browser().await;
1039        engine.adopt_resident_mmap(model_data)?;
1040    }
1041    phase("Engine resident — ready").await;
1042    let worker = etalon_wasm::readback::browser_worker();
1043    let current = worker.lock().unwrap();
1044    if current.completion_stamp() != stamp || !current.is_ready() {
1045        return Err("Browser model initialization superseded or cancelled".into());
1046    }
1047    WASM_ENGINE_INSTANCE.with(|g| *g.borrow_mut() = Some(engine));
1048    clear_init_status();
1049    Ok(())
1050}
1051
1052#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
1053impl QTensorEngine {
1054    /// W3 kernel-parity probe (test/diagnostic): run the GPU GEMM (`dispatch_gemm_raw_into`) and the
1055    /// CPU reference (`stack_gemm_quant`) on the SAME quantized weights + input, writing each into a
1056    /// caller-provided buffer. Ensures the GEMM buffers exist first, so a fresh engine (no model
1057    /// loaded) can be probed directly. Returns `true` only if both ran; the caller compares the two
1058    /// outputs with [`crate::llm_kernel_parity`]. Enable [`crate::llm_gpu_profiler`] around the call
1059    /// to witness that the GPU path actually executed rather than silently falling back to the CPU.
1060    pub fn gemm_parity_probe(
1061        &mut self,
1062        info: &GgufTensorInfo,
1063        raw: &[u8],
1064        input: &[f32],
1065        gpu_out: &mut [f32],
1066        cpu_out: &mut [f32],
1067        n_in: usize,
1068        n_out: usize,
1069    ) -> bool {
1070        self.ensure_gemm_buffers(raw.len().max(1), n_out as u32);
1071        let gpu_ok = self.dispatch_gemm_raw_into(info, raw, input, gpu_out, n_in, n_out);
1072        let cpu_ok = stack_gemm_quant(raw, info, input, cpu_out, n_in, n_out);
1073        gpu_ok && cpu_ok
1074    }
1075}
1076
1077mod cpu_attention_adapter;
1078
1079#[cfg(target_arch = "wasm32")]
1080mod async_attention_adapter;
1081
1082#[cfg(target_arch = "wasm32")]
1083mod prefill_adapter;
1084
1085#[cfg(target_arch = "wasm32")]
1086mod browser_projection_adapter;