Skip to main content

qualia_core_db/gguf_bridge/
gemm.rs

1//! GEMM: weight upload + raw/quantized dispatch + ternary-FFN dispatch
2//! Split from gguf_bridge/mod.rs (structural refactor; no behaviour change).
3use super::*;
4
5/// XOR into the resident-weight key so f16-promoted FFN blobs never alias quant blobs.
6#[cfg(not(target_arch = "wasm32"))]
7const F16_PROMOTE_KEY_TAG: u64 = 0xF16E_F16E_0000_00F1;
8
9/// Host policy and retained immutable source. Ownership equality preserves leased-model warmth
10/// and prevents recycled mmap addresses from aliasing a previous model's weights.
11#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
12pub(super) struct ResidentProjectionCache {
13    source: Option<super::model_bytes::ModelByteHandle>,
14    package: Option<Arc<crate::inference::moe::ftw_loader::FtwModelPackage>>,
15    buffers: etalon_inference::projections::weight_resources::ResidentWeightCache,
16    byte_budget: u64,
17}
18#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
19impl Default for ResidentProjectionCache {
20    fn default() -> Self {
21        Self {
22            source: None,
23            package: None,
24            buffers: Default::default(),
25            // Local retained-weight limit derives from the existing native host policy.
26            // This ledger does not reserve or subtract usage in the global VRAM ledger.
27            byte_budget: crate::gpu_context::global_vram_ledger().budget(),
28        }
29    }
30}
31
32impl QTensorEngine {
33    /// Dequant full 2-D weight to dense f32 (row-major n_out × n_in) for CUDA TC path.
34    /// Parallel over rows (rayon) — Q4 densify is cold-path once, then cached.
35    #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
36    pub(crate) fn dequant_weight_dense_f32(
37        info: &GgufTensorInfo,
38        raw: &[u8],
39        n_in: usize,
40        n_out: usize,
41    ) -> Option<Vec<f32>> {
42        etalon_inference::projections::weights::dense_f32(
43            info,
44            raw,
45            n_in,
46            n_out,
47            crate::inference::cuda_lane::MAX_DENSE_ELEMS * 4,
48        )
49    }
50
51    /// Pre-densify and cache a weight for mode=cuda / TC GEMM (no thrash on first token).
52    ///
53    /// Used by the CUDA_DECODE plan path to fill the host dense-weight cache so
54    /// `try_cuda_batch_gemv` can hit without a cold dequant on first use. Skips
55    /// when already cached, dims too small for WMMA pad-16, or densify fails.
56    #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
57    pub(crate) fn prewarm_cuda_weight(
58        info: &GgufTensorInfo,
59        raw: &[u8],
60        n_in: usize,
61        n_out: usize,
62    ) -> bool {
63        if !crate::prefer_tensor_core_gemm() {
64            return false;
65        }
66        if n_in < 16 || n_out < 16 {
67            return false;
68        }
69        // Cap densify cost: one matrix ≤ ~64 MiB f32 (fits 3B FFN rows on A2000 headroom).
70        if n_in.saturating_mul(n_out).saturating_mul(4) > 64 * 1024 * 1024 {
71            return false;
72        }
73        let key = crate::weight_fingerprint(raw, n_in, n_out);
74        if crate::dense_weight_cached(key) {
75            return true;
76        }
77        // Dequantize directly into the 2 MiB-aligned huge-page buffer — skips
78        // the intermediate Vec<f32> allocation + copy that cache_dense_weight
79        // would require. Rayon parallelizes over rows.
80        crate::inference::cuda_lane::cache_dense_weight_direct(key, n_in, n_out, |buf| {
81            etalon_inference::projections::weights::dequant_dense_into(info, raw, n_in, buf)
82        })
83    }
84
85    /// Promote a 2-D quant weight (Q4_K / SoA / Q6_K / Q8_0) to a resident **f16** buffer
86    /// for the fast coop GEMV path. Returns `(buffer, ggml_type=F16, byte_len, row_elems)`.
87    /// `None` when disabled, unsupported type, or OOM/dequant failure — caller keeps quant.
88    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
89    pub(crate) fn promote_matrix_to_f16_resident(
90        &self,
91        info: &GgufTensorInfo,
92        raw: &[u8],
93    ) -> Option<(etalon::Buffer, u32, u32, u32)> {
94        if !crate::llm_bench::ffn_f16_enabled() {
95            return None;
96        }
97        let f16_bytes =
98            etalon_inference::projections::weights::promote_f16(info, raw, 512 * 1024 * 1024)?;
99        let nbytes = f16_bytes.len();
100        let n0 = info.dims[0] as usize;
101        let key = (raw.as_ptr() as u64) ^ F16_PROMOTE_KEY_TAG;
102        let buf = self.resident_weight_buffer(key, &f16_bytes)?;
103        Some((
104            buf,
105            crate::ggml_quants::GGML_TYPE_F16,
106            nbytes as u32,
107            n0 as u32,
108        ))
109    }
110
111    #[cfg(feature = "gpu-runtime")]
112    /// Writes `weight-words`.
113    pub(crate) fn write_weight_words(&self, raw: &[u8], max_bytes: usize) {
114        etalon_inference::projections::weight_resources::write_weight_words(
115            self.gpu_queue(),
116            self.gemm_weight_buf.as_ref().expect("gemm weight buf"),
117            raw,
118            max_bytes,
119        )
120    }
121
122    /// Phase 2: get-or-create the resident VRAM buffer for a weight byte-region, keyed by `key`
123    /// (the region's absolute mmap address — unique per distinct weight, stable across tokens). The
124    /// bytes (from the immutable mmap) are uploaded **once** on first use, then this returns a clone
125    /// of the resident buffer handle (wgpu buffers are Arc-backed) to bind in place of the shared
126    /// per-token `gemm_weight_buf` — eliminating the per-token weight re-upload. Buffer size is
127    /// 256-aligned ≥ `raw.len()`; the shader only reads `weight_byte_len`.
128    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
129    pub(crate) fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<etalon::Buffer> {
130        let source = self.clone_model_bytes()?;
131        let mut cache = self.gemm_resident_weights.lock().ok()?;
132        let same_package = match (cache.package.as_ref(), self.ftw_package.as_ref()) {
133            (Some(a), Some(b)) => Arc::ptr_eq(a, b),
134            (None, None) => true,
135            _ => false,
136        };
137        if !same_package
138            || !cache
139                .source
140                .as_ref()
141                .is_some_and(|old| old.same_owner(&source))
142        {
143            cache.buffers.clear();
144            cache.source = Some(source);
145            cache.package = self.ftw_package.clone();
146        }
147        let mut budget =
148            etalon_inference::gpu_resources::ResourceBudget::for_device(self.gpu_device());
149        budget.max_buffer_bytes = cache.byte_budget;
150        etalon_inference::projections::weight_resources::resident_weight_buffer(
151            self.gpu_device(),
152            self.gpu_queue(),
153            &mut cache.buffers,
154            key,
155            raw,
156            budget,
157        )
158    }
159
160    /// Set the admitted cold native-projection/readback payload budget. This legacy Vec-returning
161    /// entrypoint reports native admission exhaustion by panic instead of choosing another layout.
162    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
163    pub fn set_projection_cold_byte_budget(&mut self, bytes: usize) {
164        self.projection_cold_byte_budget = bytes;
165    }
166
167    /// Set aggregate projection-weight allowance. Admission misses retain uncached GPU/CPU fallback.
168    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
169    pub fn set_resident_projection_byte_budget(&mut self, bytes: u64) {
170        let cache = self
171            .gemm_resident_weights
172            .get_mut()
173            .unwrap_or_else(|e| e.into_inner());
174        cache.byte_budget = bytes;
175        if cache.buffers.retained_bytes() > bytes {
176            cache.buffers.clear();
177        }
178    }
179
180    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
181    pub(super) fn invalidate_resident_projection_weights(&mut self) {
182        let cache = self
183            .gemm_resident_weights
184            .get_mut()
185            .unwrap_or_else(|e| e.into_inner());
186        cache.buffers.clear();
187        cache.source = None;
188        cache.package = None;
189    }
190
191    /// Quantized GEMM from a pre-sliced weight byte range (chunk-local row indices).
192    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
193    pub(crate) fn dispatch_gemm_raw_into(
194        &self,
195        info: &GgufTensorInfo,
196        raw: &[u8],
197        input: &[f32],
198        out: &mut [f32],
199        n_in: usize,
200        n_out: usize,
201    ) -> bool {
202        etalon_inference::projections::ProjectionGpuEngine::new(self)
203            .dispatch_gemm_raw_into(info, raw, input, out, n_in, n_out)
204    }
205
206    /// CPU GEMM when wgpu is not compiled in (qdnf / no gpu-runtime).
207    #[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
208    pub(crate) fn dispatch_gemm_raw_into(
209        &self,
210        info: &GgufTensorInfo,
211        raw: &[u8],
212        input: &[f32],
213        out: &mut [f32],
214        n_in: usize,
215        n_out: usize,
216    ) -> bool {
217        if n_in > input.len() || n_out > out.len() {
218            return false;
219        }
220        stack_gemm_quant(raw, info, input, out, n_in, n_out)
221    }
222
223    /// Browser compatibility path for synchronous callers. WebGPU mapping must
224    /// be awaited, so the public browser inference route uses the async GEMM
225    /// dispatcher and this fallback executes the deterministic CPU kernel.
226    #[cfg(target_arch = "wasm32")]
227    pub(crate) fn dispatch_gemm_raw_into(
228        &self,
229        info: &GgufTensorInfo,
230        raw: &[u8],
231        input: &[f32],
232        out: &mut [f32],
233        n_in: usize,
234        n_out: usize,
235    ) -> bool {
236        if n_in > input.len() || n_out > out.len() {
237            return false;
238        }
239        stack_gemm_quant(raw, info, input, out, n_in, n_out)
240    }
241
242    /// A1b: dispatch one **ternary** FFN GEMM (`GGML_TYPE_TERNARY_158`). The resident 2-bit GPU
243    /// kernel when the toggle is on AND the tensor is resident; otherwise the CPU base-3 oracle on
244    /// the blob fetched from the mmap. Fail-closed (returns false, never garbage) on any mismatch.
245    pub(crate) fn dispatch_ternary_ffn(
246        &self,
247        info: &GgufTensorInfo,
248        input: &[f32],
249        out: &mut [f32],
250        n_in: usize,
251        n_out: usize,
252    ) -> bool {
253        if n_in > input.len() || n_out > out.len() {
254            return false;
255        }
256        // GPU resident path (toggle on): keyed by the P64 blob offset (== info.byte_offset).
257        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
258        if crate::llm_bench::ternary_ffn_enabled() {
259            if let Some(res) = self.ternary_ffn.as_ref() {
260                if res.gemv(
261                    self.gpu_device(),
262                    self.gpu_queue(),
263                    info.byte_offset,
264                    input,
265                    out,
266                    n_in,
267                    n_out,
268                ) {
269                    return true;
270                }
271            }
272        }
273        // CPU oracle fallback (toggle off, or GPU unavailable): the SAME ternary weights via the
274        // base-3 CPU GEMM — correct, slower; this is the toggle's OFF baseline.
275        let mmap = match self.model_bytes() {
276            Some(m) => m,
277            None => return false,
278        };
279        let raw = match crate::ggml_quants::fetch_tensor_bytes(mmap, self.tensor_data_offset, info)
280        {
281            Ok(s) => s,
282            Err(_) => return false,
283        };
284        if raw.len() < 4 {
285            return false;
286        }
287        let scale = f32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]);
288        crate::ternary::ternary_gemm_cpu(
289            &input[..n_in],
290            &raw[4..],
291            scale,
292            n_in,
293            n_out,
294            1,
295            0,
296            0,
297            &mut out[..n_out],
298        );
299        true
300    }
301
302    /// Quantized GEMM into caller `out` using reused GPU buffers (Q6_K) or CPU dequant fallback.
303    pub fn dispatch_gemm_into(
304        &self,
305        index: &crate::gguf_sharder::GgufTensorIndex,
306        info: &GgufTensorInfo,
307        input: &[f32],
308        out: &mut [f32],
309        n_in: usize,
310        n_out: usize,
311    ) -> bool {
312        if n_in > input.len() || n_out > out.len() {
313            wlog(&format!(
314                "[gemm_into] GUARD n_in={n_in} n_out={n_out} input={} out={}",
315                input.len(),
316                out.len()
317            ));
318            return false;
319        }
320        // A1b: ternary FFN tensors are not row-block quantized — route them to the dedicated ternary
321        // dispatch (resident 2-bit GPU kernel / CPU oracle) before the standard fetch+GEMM path.
322        if info.ggml_type == crate::ternary::GGML_TYPE_TERNARY_158 {
323            return self.dispatch_ternary_ffn(info, input, out, n_in, n_out);
324        }
325        #[cfg(not(target_arch = "wasm32"))]
326        let raw = match self.fetch_tensor_raw_bytes(info) {
327            Some(s) => s,
328            None => {
329                let mmap = match self.model_bytes() {
330                    Some(m) => m,
331                    None => return false,
332                };
333                match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, info) {
334                    Ok(s) => s,
335                    Err(_) => return false,
336                }
337            }
338        };
339        #[cfg(target_arch = "wasm32")]
340        let raw = {
341            let mmap = match self.model_bytes() {
342                Some(m) => m,
343                None => return false,
344            };
345            match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, info) {
346                Ok(s) => s,
347                Err(_) => return false,
348            }
349        };
350        self.dispatch_gemm_raw_into(info, raw, input, out, n_in, n_out)
351    }
352}
353
354/// Substrate-parity proof (CPU, no GPU): the LLM's quantized GEMV is *the same linear
355/// operation* as the engine's dense `solvers::linear_algebra::gemm`.
356///
357/// The LLM forward path is not a bespoke "AI inference" kernel — its weight×activation
358/// step is matrix–vector multiplication `out[i] = Σ_j W[i][j]·x[j]`, with `W` dequantized
359/// on the fly. This proves it: dequantize the quantized weights to a dense matrix, run the
360/// engine's `matvec` on them, and show the LLM kernel (`stack_gemm_quant`) agrees to f32
361/// rounding. Together with the existing GPU↔CPU probe (`gemm_parity_probe`), this closes
362/// the chain  substrate GEMM ≡ LLM CPU GEMV ≡ LLM GPU GEMV.
363#[cfg(test)]
364mod substrate_parity_tests {
365    use crate::gguf_sharder::GgufTensorInfo;
366    use crate::solvers::linear_algebra::gemm::{matvec, Transpose};
367
368    /// Returns `(exact_err, quant_err)`:
369    /// - `exact_err` = max|LLM_kernel(Q8(W)) − substrate(dequant(Q8(W)))| — should be ~f32 ε,
370    ///   proving the two compute the *same* operation;
371    /// - `quant_err` = max|LLM_kernel(Q8(W)) − substrate(W_original)| — the Q8 quantization cost.
372    fn run(n_in: usize, n_out: usize, seed: u64) -> (f32, f32) {
373        // Deterministic LCG → values in [-1, 1) (mirrors gemm_parity_probe_blocking).
374        let mut s = seed | 1;
375        let mut rng = move || -> f32 {
376            s = s
377                .wrapping_mul(6364136223846793005)
378                .wrapping_add(1442695040888963407);
379            ((s >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0
380        };
381
382        let row_bytes = crate::llm_kernel_parity::q8_0_bytes(n_in);
383        let mut raw = vec![0u8; row_bytes * n_out];
384        let mut w_orig = vec![0f32; n_in * n_out]; // dense, pre-quantization (row-major n_out×n_in)
385        let mut row_f32 = vec![0f32; n_in];
386        for r in 0..n_out {
387            for x in row_f32.iter_mut() {
388                *x = rng();
389            }
390            w_orig[r * n_in..(r + 1) * n_in].copy_from_slice(&row_f32);
391            assert!(crate::llm_kernel_parity::quantize_q8_0_from_f32(
392                &row_f32,
393                &mut raw[r * row_bytes..(r + 1) * row_bytes],
394            ));
395        }
396        let input: Vec<f32> = (0..n_in).map(|_| rng()).collect();
397
398        let info = GgufTensorInfo {
399            dims: [n_in as u64, n_out as u64, 1, 1],
400            n_dims: 2,
401            ggml_type: crate::ggml_quants::GGML_TYPE_Q8_0,
402            byte_offset: 0,
403        };
404
405        // (1) The actual LLM CPU kernel.
406        let mut out_llm = vec![0f32; n_out];
407        assert!(crate::gguf_bridge::stack_gemm_quant(
408            &raw,
409            &info,
410            &input,
411            &mut out_llm,
412            n_in,
413            n_out
414        ));
415
416        // (2) Dequantize the same quantized weights to a dense matrix, then run the
417        //     engine's GEMM (matvec) on it. Same operands ⇒ must match the LLM kernel.
418        let mut w_deq = vec![0f64; n_in * n_out];
419        let mut deq_row = vec![0f32; n_in];
420        for i in 0..n_out {
421            let got = crate::ggml_quants::dequant_matrix_row_into(&raw, &info, i, &mut deq_row)
422                .unwrap_or(0);
423            assert_eq!(got, n_in, "dequant row {i}");
424            for j in 0..n_in {
425                w_deq[i * n_in + j] = deq_row[j] as f64;
426            }
427        }
428        let x_f64: Vec<f64> = input.iter().map(|&v| v as f64).collect();
429        let mut out_sub_deq = vec![0f64; n_out];
430        matvec(Transpose::No, n_out, n_in, &w_deq, &x_f64, &mut out_sub_deq).unwrap();
431
432        // (3) The engine GEMM on the ORIGINAL (pre-quant) weights — the Q8 cost reference.
433        let w_orig_f64: Vec<f64> = w_orig.iter().map(|&v| v as f64).collect();
434        let mut out_sub_orig = vec![0f64; n_out];
435        matvec(
436            Transpose::No,
437            n_out,
438            n_in,
439            &w_orig_f64,
440            &x_f64,
441            &mut out_sub_orig,
442        )
443        .unwrap();
444
445        let exact_err = (0..n_out)
446            .map(|i| (out_llm[i] as f64 - out_sub_deq[i]).abs() as f32)
447            .fold(0.0f32, f32::max);
448        let quant_err = (0..n_out)
449            .map(|i| (out_llm[i] as f64 - out_sub_orig[i]).abs() as f32)
450            .fold(0.0f32, f32::max);
451        (exact_err, quant_err)
452    }
453
454    #[test]
455    fn llm_quant_gemv_is_the_substrate_gemm() {
456        // Several shapes/seeds; n_in a multiple of 32 (Q8_0 block size).
457        for &(n_in, n_out, seed) in &[
458            (64usize, 32usize, 0xC0FFEEu64),
459            (128, 96, 7),
460            (256, 64, 0xBEEF),
461        ] {
462            let (exact_err, quant_err) = run(n_in, n_out, seed);
463            // Same operation: LLM kernel == engine GEMM on identical (dequantized) weights,
464            // to f32 accumulation rounding only.
465            assert!(
466                exact_err < 1e-4,
467                "LLM GEMV diverges from substrate GEMM on identical weights: exact_err={exact_err} (n_in={n_in}, n_out={n_out})"
468            );
469            // Quantization is the *only* extra divergence from exact math, and it is bounded.
470            assert!(
471                quant_err < 0.5,
472                "Q8 quantization error unexpectedly large: quant_err={quant_err}"
473            );
474        }
475    }
476}