Skip to main content

qualia_core_db/solvers/linear_algebra/
gemm.rs

1//! Dynamic-size general matrix multiply (GEMM) — the canonical dense-linear-algebra core.
2//!
3//! This is the **one** dynamic GEMM for the engine. Before this, three private
4//! re-implementations competed (`specialized_libs/linear_algebra` heap GEMM,
5//! `gguf_bridge` GPU `coop_gemv`, and the fixed-size [`super::Matrix4x4`]). The
6//! specialized libs route their dynamic matmul here; the GPU `coop_gemv` kernel is
7//! the *same contract* executed on `wgpu` and is checked against this code as its
8//! CPU parity reference (`gemm_parity_probe`).
9//!
10//! Idiom (matches `super::cholesky`): **zero allocation**, caller-owned **row-major**
11//! slices with explicit dimensions, fail-closed on a dimension mismatch
12//! (`SolversError::InvalidDimension`). No `DMatrix`, no heap, no dependency.
13
14use crate::solvers::SolversError;
15
16/// Whether a GEMM operand is used as stored (`No`) or transposed (`Yes`).
17///
18/// `op(X)` below means `X` when `No` and `Xᵀ` when `Yes`. Transpose is expressed
19/// by index arithmetic only — no operand is ever materialised.
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
21pub enum Transpose {
22    /// Use the operand as stored.
23    No,
24    /// Use the transpose of the operand.
25    Yes,
26}
27
28/// General matrix multiply (BLAS-3 `gemm` shape), row-major, zero-heap:
29///
30/// ```text
31/// C := alpha · op(A) · op(B) + beta · C
32/// ```
33///
34/// where `op(A)` is `m×k`, `op(B)` is `k×n`, and `C` is `m×n`. Operands are
35/// caller-owned row-major slices:
36/// - `a` holds `op==No ? m×k : k×m` → always `m*k` elements.
37/// - `b` holds `op==No ? k×n : n×k` → always `k*n` elements.
38/// - `c` is `m*n` elements, read (when `beta != 0`) and overwritten in place.
39///
40/// `beta == 0.0` is honoured as a hard zero (the existing contents of `c`, which
41/// may be uninitialised garbage, are *not* read) — matching BLAS semantics so a
42/// fresh output buffer need not be zeroed first.
43///
44/// Returns `SolversError::InvalidDimension` if any slice length disagrees with
45/// `m`, `n`, `k`.
46pub fn gemm(
47    transa: Transpose,
48    transb: Transpose,
49    m: usize,
50    n: usize,
51    k: usize,
52    alpha: f64,
53    a: &[f64],
54    b: &[f64],
55    beta: f64,
56    c: &mut [f64],
57) -> Result<(), SolversError> {
58    if a.len() != m * k || b.len() != k * n || c.len() != m * n {
59        return Err(SolversError::InvalidDimension);
60    }
61
62    // ── Best-path-with-CPU-floor offload (additive; behaviour-preserving) ──────────
63    //
64    // The capability-aware forge dispatcher (`wgsl_forge::dispatch::gemm_f64`) picks
65    // the best compute path *actually present on this machine* for the raw product
66    // `P = op(A)·op(B)`, while always keeping this CPU triple-loop as the floor. We hand
67    // off the product whenever:
68    //   1. an accelerator actually exists (`caps().cuda || caps().wgpu`) — so a no-GPU
69    //      machine takes the EXACT existing CPU path below with zero behaviour change;
70    //   2. the problem is large enough (`m·n·k >= GEMM_GPU_THRESHOLD`) to amortise
71    //      dispatch/transfer overhead — small GEMMs stay on the CPU.
72    //
73    // `gemm_f64` computes only the *plain* row-major product (no alpha/beta/transpose),
74    // so the full BLAS shape is reconstructed around it:
75    //   • a transpose flag is honoured by *materialising* that operand once into a
76    //     row-major scratch buffer (`op(A)` as `m×k`, `op(B)` as `k×n`) — an O(m·k)/O(k·n)
77    //     copy dominated by the O(m·n·k) GEMM being offloaded. This unlocks the hot
78    //     covariance `XᵀX` (PCA/ridge/linear) and attention `Q·Kᵀ`.
79    //   • `alpha`/`beta` are applied in the O(m·n) CPU combine afterward, exactly as the
80    //     loop below would (`beta == 0` is a hard zero, so `c` is not read — BLAS rule).
81    // The materialised operand equals exactly what `a_at`/`b_at` would read, and the
82    // dispatcher's CPU floor uses the same increasing-index accumulation as the loop
83    // below, so the answers agree to f64 summation precision.
84    //
85    // f64 on the GPU is **CUDA-only**: WGSL has no `f64`, so for a non-NVIDIA GPU the
86    // dispatcher itself falls back to its own f64 CPU floor (the `caps().wgpu` term just
87    // means "an accelerator is present"; the actual f64 GPU kernel is native CUDA-f64).
88    // Crucially, a forge error is NEVER propagated out of the solver: on `Err(_)` we fall
89    // through to the CPU path below, which is always correct. Sub-threshold or
90    // off-accelerator, the unchanged CPU code runs — byte-identical to before.
91    // GPU best-path GEMM via the forge — only when it's compiled in (native +
92    // wgsl-forge). On wasm32 the forge module doesn't exist; the CPU floor below runs.
93    #[cfg(all(not(target_arch = "wasm32"), feature = "wgsl-forge"))]
94    {
95        use crate::wgsl_forge::dispatch::{caps, GEMM_GPU_THRESHOLD};
96        use std::borrow::Cow;
97        let work = m.saturating_mul(n).saturating_mul(k);
98        // Skip caps() on small work — the CUDA probe can panic if libcuda is missing
99        // and we do not need it below the GPU threshold.
100        if work >= GEMM_GPU_THRESHOLD {
101            let caps = caps();
102            if caps.cuda || caps.wgpu {
103                // op(A): row-major m×k — stored m×k already (No) or k×m (Yes → transpose).
104                let a_eff: Cow<[f64]> = match transa {
105                    Transpose::No => Cow::Borrowed(a),
106                    Transpose::Yes => {
107                        let mut t = vec![0.0_f64; m * k];
108                        for i in 0..m {
109                            for l in 0..k {
110                                t[i * k + l] = a[l * m + i];
111                            }
112                        }
113                        Cow::Owned(t)
114                    }
115                };
116                // op(B): row-major k×n — stored k×n already (No) or n×k (Yes → transpose).
117                let b_eff: Cow<[f64]> = match transb {
118                    Transpose::No => Cow::Borrowed(b),
119                    Transpose::Yes => {
120                        let mut t = vec![0.0_f64; k * n];
121                        for l in 0..k {
122                            for j in 0..n {
123                                t[l * n + j] = b[j * k + l];
124                            }
125                        }
126                        Cow::Owned(t)
127                    }
128                };
129                // BLAS gemm dims are (m, n, k); the dispatcher takes (m, k, n): op(A) is m×k,
130                // op(B) is k×n, C is m×n, so the mapping is gemm(m,n,k) → dispatch(m,k,n).
131                if let Ok(product) = crate::wgsl_forge::dispatch::gemm_f64(m, k, n, &a_eff, &b_eff)
132                {
133                    // Apply alpha/beta exactly as the CPU loop would (beta==0 ⇒ c not read).
134                    if beta == 0.0 {
135                        for (ci, &p) in c.iter_mut().zip(product.iter()) {
136                            *ci = alpha * p;
137                        }
138                    } else {
139                        for (ci, &p) in c.iter_mut().zip(product.iter()) {
140                            *ci = alpha * p + beta * *ci;
141                        }
142                    }
143                    return Ok(());
144                }
145                // Forge path was eligible but errored — fall through to the CPU floor.
146            }
147        }
148    }
149
150    // op(A)[i][l] — A stored m×k (No) or k×m (Yes).
151    let a_at = |i: usize, l: usize| -> f64 {
152        match transa {
153            Transpose::No => a[i * k + l],
154            Transpose::Yes => a[l * m + i],
155        }
156    };
157    // op(B)[l][j] — B stored k×n (No) or n×k (Yes).
158    let b_at = |l: usize, j: usize| -> f64 {
159        match transb {
160            Transpose::No => b[l * n + j],
161            Transpose::Yes => b[j * k + l],
162        }
163    };
164    for i in 0..m {
165        for j in 0..n {
166            let mut s = 0.0;
167            for l in 0..k {
168                s += a_at(i, l) * b_at(l, j);
169            }
170            let idx = i * n + j;
171            // beta==0 ⇒ hard zero, so c may be uninitialised on entry (BLAS rule).
172            c[idx] = if beta == 0.0 {
173                alpha * s
174            } else {
175                alpha * s + beta * c[idx]
176            };
177        }
178    }
179    Ok(())
180}
181
182/// Plain product `C := A · B` for `A` (`m×k`), `B` (`k×n`), `C` (`m×n`),
183/// all row-major and caller-owned. Thin wrapper over [`gemm`] with
184/// `alpha = 1`, `beta = 0` and no transposes.
185pub fn matmul(
186    m: usize,
187    k: usize,
188    n: usize,
189    a: &[f64],
190    b: &[f64],
191    c: &mut [f64],
192) -> Result<(), SolversError> {
193    gemm(Transpose::No, Transpose::No, m, n, k, 1.0, a, b, 0.0, c)
194}
195
196/// Matrix–vector product `y := op(A) · x`.
197///
198/// `op(A)` is `m×n`; `a` holds `op==No ? m×n : n×m` (always `m*n` elements),
199/// `x` is length `n`, `y` is length `m` (overwritten). Zero-heap; the dynamic
200/// analogue of [`super::Matrix4x4::multiply_vector`], and the shape the GPU
201/// `coop_gemv` decode kernel computes per output row.
202pub fn matvec(
203    transa: Transpose,
204    m: usize,
205    n: usize,
206    a: &[f64],
207    x: &[f64],
208    y: &mut [f64],
209) -> Result<(), SolversError> {
210    if a.len() != m * n || x.len() != n || y.len() != m {
211        return Err(SolversError::InvalidDimension);
212    }
213
214    // ── Best-path-with-CPU-floor offload (additive; behaviour-preserving) ──────────
215    //
216    // Mirror of the [`gemm`] fast-path above. The forge dispatcher
217    // (`wgsl_forge::dispatch::gemv_f64`) computes the *plain* product `y = A·x` only
218    // (it has no transpose support), so we only hand off when ALL hold:
219    //   1. it IS the plain product — `transa == Transpose::No` — because `gemv_f64`
220    //      cannot express `Aᵀ·x`;
221    //   2. an accelerator actually exists (`caps().cuda || caps().wgpu`) — so a no-GPU
222    //      machine takes the EXACT existing CPU loop below with zero behaviour change;
223    //   3. the problem is large enough (`m·n >= GEMM_GPU_THRESHOLD`) to amortise
224    //      dispatch/transfer overhead — small GEMVs stay on the CPU.
225    //
226    // Dimensions map directly: here `A` is m×n (row-major), `x` is length n, `y` is
227    // length m — exactly `gemv_f64(m, n, a, x)`'s `y[M] = A[M×N]·x[N]` contract, so no
228    // re-mapping is needed (unlike gemm's (m,n,k)→(m,k,n) swap). f64 on the GPU is
229    // CUDA-only (WGSL has no f64); on a non-NVIDIA GPU the dispatcher itself falls back
230    // to its own f64 CPU floor, whose increasing-`j` summation order matches this loop,
231    // so answers agree to f64 summation precision.
232    //
233    // A forge error is NEVER propagated: on `Err(_)` we fall through to the CPU loop
234    // below, which is always correct. Any transpose, sub-threshold size, or
235    // off-accelerator run executes the unchanged CPU code — byte-identical to before.
236    #[cfg(all(not(target_arch = "wasm32"), feature = "wgsl-forge"))]
237    if transa == Transpose::No {
238        use crate::wgsl_forge::dispatch::{caps, GEMM_GPU_THRESHOLD};
239        let work = m.saturating_mul(n);
240        let caps = caps();
241        if (caps.cuda || caps.wgpu) && work >= GEMM_GPU_THRESHOLD {
242            if let Ok(result) = crate::wgsl_forge::dispatch::gemv_f64(m, n, a, x) {
243                y.copy_from_slice(&result);
244                return Ok(());
245            }
246            // Forge path was eligible but errored — fall through to the CPU floor.
247        }
248    }
249
250    let a_at = |i: usize, j: usize| -> f64 {
251        match transa {
252            Transpose::No => a[i * n + j],
253            Transpose::Yes => a[j * m + i],
254        }
255    };
256    for i in 0..m {
257        let mut s = 0.0;
258        for j in 0..n {
259            s += a_at(i, j) * x[j];
260        }
261        y[i] = s;
262    }
263    Ok(())
264}
265
266/// Transpose the `m×n` row-major matrix `a` into the `n×m` row-major buffer `out`.
267/// Caller-owned, zero-heap. `a` is `m*n`, `out` is `n*m`.
268pub fn transpose(m: usize, n: usize, a: &[f64], out: &mut [f64]) -> Result<(), SolversError> {
269    if a.len() != m * n || out.len() != m * n {
270        return Err(SolversError::InvalidDimension);
271    }
272    for i in 0..m {
273        for j in 0..n {
274            out[j * m + i] = a[i * n + j];
275        }
276    }
277    Ok(())
278}
279
280#[cfg(test)]
281mod tests {
282    use super::*;
283
284    fn approx(a: &[f64], b: &[f64]) {
285        assert_eq!(a.len(), b.len());
286        for i in 0..a.len() {
287            assert!((a[i] - b[i]).abs() < 1e-9, "idx {i}: {} != {}", a[i], b[i]);
288        }
289    }
290
291    #[test]
292    fn matmul_rectangular() {
293        // A (2×3) · B (3×2) = C (2×2)
294        let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
295        let b = [7.0, 8.0, 9.0, 10.0, 11.0, 12.0];
296        let mut c = [0.0; 4];
297        matmul(2, 3, 2, &a, &b, &mut c).unwrap();
298        // [[1*7+2*9+3*11, 1*8+2*10+3*12],[4*7+5*9+6*11, 4*8+5*10+6*12]]
299        approx(&c, &[58.0, 64.0, 139.0, 154.0]);
300    }
301
302    #[test]
303    fn matmul_identity_is_noop() {
304        let a = [1.0, 2.0, 3.0, 4.0];
305        let id = [1.0, 0.0, 0.0, 1.0];
306        let mut c = [0.0; 4];
307        matmul(2, 2, 2, &a, &id, &mut c).unwrap();
308        approx(&c, &a);
309    }
310
311    #[test]
312    fn gemm_alpha_beta_accumulate() {
313        // C := 2·A·B + 3·C
314        let a = [1.0, 2.0, 3.0, 4.0];
315        let b = [1.0, 0.0, 0.0, 1.0];
316        let mut c = [1.0, 1.0, 1.0, 1.0];
317        gemm(
318            Transpose::No,
319            Transpose::No,
320            2,
321            2,
322            2,
323            2.0,
324            &a,
325            &b,
326            3.0,
327            &mut c,
328        )
329        .unwrap();
330        // 2·A + 3·C0 = 2·[1,2,3,4] + 3·[1,1,1,1]
331        approx(&c, &[5.0, 7.0, 9.0, 11.0]);
332    }
333
334    #[test]
335    fn gemm_beta_zero_ignores_garbage() {
336        // beta=0 must not read c (here pre-filled with NaN-ish garbage).
337        let a = [1.0, 2.0, 3.0, 4.0];
338        let b = [1.0, 0.0, 0.0, 1.0];
339        let mut c = [f64::NAN, f64::NAN, f64::NAN, f64::NAN];
340        gemm(
341            Transpose::No,
342            Transpose::No,
343            2,
344            2,
345            2,
346            1.0,
347            &a,
348            &b,
349            0.0,
350            &mut c,
351        )
352        .unwrap();
353        approx(&c, &a);
354    }
355
356    #[test]
357    fn gemm_transpose_a_normal_equations() {
358        // AᵀA for A (3×2): expect symmetric 2×2. (op(A)=Aᵀ is 2×3, op(A)=A... )
359        // m=2, n=2, k=3: C = Aᵀ(2×3) · A(3×2).
360        let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 3×2, row-major
361        let mut c = [0.0; 4];
362        gemm(
363            Transpose::Yes,
364            Transpose::No,
365            2,
366            2,
367            3,
368            1.0,
369            &a,
370            &a,
371            0.0,
372            &mut c,
373        )
374        .unwrap();
375        // AᵀA = [[1+9+25, 2+12+30],[2+12+30, 4+16+36]] = [[35,44],[44,56]]
376        approx(&c, &[35.0, 44.0, 44.0, 56.0]);
377    }
378
379    #[test]
380    fn gemm_transpose_b() {
381        // A (2×3) · Bᵀ where B is (2×3) → C (2×2)
382        let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
383        let b = [1.0, 0.0, 1.0, 0.0, 1.0, 0.0]; // 2×3
384        let mut c = [0.0; 4];
385        gemm(
386            Transpose::No,
387            Transpose::Yes,
388            2,
389            2,
390            3,
391            1.0,
392            &a,
393            &b,
394            0.0,
395            &mut c,
396        )
397        .unwrap();
398        // op(B) = Bᵀ (3×2): rows of A dotted with rows of B.
399        // C[0][0]=1*1+2*0+3*1=4 ; C[0][1]=1*0+2*1+3*0=2
400        // C[1][0]=4*1+5*0+6*1=10; C[1][1]=4*0+5*1+6*0=5
401        approx(&c, &[4.0, 2.0, 10.0, 5.0]);
402    }
403
404    #[test]
405    fn matvec_basic() {
406        let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 2×3
407        let x = [1.0, 1.0, 1.0];
408        let mut y = [0.0; 2];
409        matvec(Transpose::No, 2, 3, &a, &x, &mut y).unwrap();
410        approx(&y, &[6.0, 15.0]);
411    }
412
413    #[test]
414    fn matvec_transposed_matches_dense_transpose() {
415        let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 2×3
416        let x = [1.0, 1.0];
417        let mut y = [0.0; 3];
418        // op(A) = Aᵀ (3×2) · x(2)
419        matvec(Transpose::Yes, 3, 2, &a, &x, &mut y).unwrap();
420        // Aᵀ rows: [1,4],[2,5],[3,6] dotted with [1,1] = 5,7,9
421        approx(&y, &[5.0, 7.0, 9.0]);
422    }
423
424    #[test]
425    fn transpose_roundtrip() {
426        let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 2×3
427        let mut t = [0.0; 6];
428        transpose(2, 3, &a, &mut t).unwrap();
429        approx(&t, &[1.0, 4.0, 2.0, 5.0, 3.0, 6.0]); // 3×2
430        let mut back = [0.0; 6];
431        transpose(3, 2, &t, &mut back).unwrap();
432        approx(&back, &a);
433    }
434
435    #[test]
436    fn gemm_via_transpose_equals_matvec() {
437        // A·x as a GEMM with n=1 must equal matvec.
438        let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 2×3
439        let x = [2.0, 0.0, 1.0];
440        let mut y_mv = [0.0; 2];
441        matvec(Transpose::No, 2, 3, &a, &x, &mut y_mv).unwrap();
442        let mut y_gemm = [0.0; 2];
443        gemm(
444            Transpose::No,
445            Transpose::No,
446            2,
447            1,
448            3,
449            1.0,
450            &a,
451            &x,
452            0.0,
453            &mut y_gemm,
454        )
455        .unwrap();
456        approx(&y_mv, &y_gemm);
457    }
458
459    #[test]
460    fn plain_gemm_matches_cpu_above_threshold() {
461        // A large plain product (48×48×48 = 110_592 ≥ GEMM_GPU_THRESHOLD) routed
462        // through `gemm(No, No, .., 1.0, a, b, 0.0, c)`. On an accelerator box this
463        // exercises the forge offload; on CI (no accelerator) it exercises the CPU
464        // floor. EITHER WAY the result must equal a SEPARATE pure-CPU triple-loop
465        // reference computed inline here, so the test is hermetic and GPU-agnostic.
466        const M: usize = 48;
467        const K: usize = 48;
468        const N: usize = 48;
469
470        // Deterministic data — no RNG, no GPU assumption.
471        let mut a = vec![0.0f64; M * K];
472        let mut b = vec![0.0f64; K * N];
473        for i in 0..M {
474            for l in 0..K {
475                a[i * K + l] = ((i * 7 + l * 3) % 11) as f64 * 0.25 - 1.0;
476            }
477        }
478        for l in 0..K {
479            for j in 0..N {
480                b[l * N + j] = ((l * 5 + j * 2) % 13) as f64 * 0.125 - 0.75;
481            }
482        }
483
484        // Reference: independent triple loop, increasing-k accumulation order.
485        let mut reference = vec![0.0f64; M * N];
486        for i in 0..M {
487            for j in 0..N {
488                let mut acc = 0.0f64;
489                for l in 0..K {
490                    acc += a[i * K + l] * b[l * N + j];
491                }
492                reference[i * N + j] = acc;
493            }
494        }
495
496        // Through the wired gemm (plain product → eligible for offload).
497        let mut c = vec![0.0f64; M * N];
498        gemm(
499            Transpose::No,
500            Transpose::No,
501            M,
502            N,
503            K,
504            1.0,
505            &a,
506            &b,
507            0.0,
508            &mut c,
509        )
510        .unwrap();
511
512        approx(&c, &reference);
513    }
514
515    #[test]
516    fn plain_gemv_matches_cpu_above_threshold() {
517        // A large plain matrix–vector product (m=n=256 → work = 65_536 ≥
518        // GEMM_GPU_THRESHOLD=32_768) routed through `matvec(No, ..)`. On an
519        // accelerator box this exercises the forge offload (`dispatch::gemv_f64`);
520        // on CI (no accelerator) it exercises the CPU floor. EITHER WAY the result
521        // must equal a SEPARATE pure-CPU reference computed inline here, so the test
522        // is hermetic and GPU-agnostic.
523        const M: usize = 256;
524        const N: usize = 256;
525
526        // Deterministic data — no RNG, no GPU assumption.
527        let mut a = vec![0.0f64; M * N];
528        for i in 0..M {
529            for j in 0..N {
530                a[i * N + j] = ((i * 7 + j * 3) % 11) as f64 * 0.25 - 1.0;
531            }
532        }
533        let mut x = vec![0.0f64; N];
534        for (j, xj) in x.iter_mut().enumerate() {
535            *xj = ((j * 5) % 13) as f64 * 0.125 - 0.75;
536        }
537
538        // Reference: independent dot-product per row, increasing-j accumulation order.
539        let mut reference = vec![0.0f64; M];
540        for i in 0..M {
541            let row = i * N;
542            let mut acc = 0.0f64;
543            for j in 0..N {
544                acc += a[row + j] * x[j];
545            }
546            reference[i] = acc;
547        }
548
549        // Through the wired matvec (plain product → eligible for offload).
550        let mut y = vec![0.0f64; M];
551        matvec(Transpose::No, M, N, &a, &x, &mut y).unwrap();
552
553        approx(&y, &reference);
554    }
555
556    #[test]
557    fn transposed_covariance_matches_cpu_above_threshold() {
558        // The hot ML op: covariance `Cov = Xᵀ·X` for tall-skinny `X` (n≫p), expressed
559        // as `gemm(Transpose::Yes, Transpose::No, p, p, n, 1.0, x, x, 0.0, cov)`. With
560        // n=4096, p=8 the work is p·p·n = 262_144 ≥ GEMM_GPU_THRESHOLD, so on an
561        // accelerator box this exercises the NEW transpose-materialising offload path;
562        // on CI (no accelerator) it exercises the CPU floor. EITHER WAY the result must
563        // equal a SEPARATE pure-CPU `XᵀX` reference computed inline here — hermetic,
564        // GPU-agnostic, and the regression guard that materialise-then-dispatch equals
565        // the index-arithmetic CPU loop.
566        const NSAMP: usize = 4096; // k (contraction dim)
567        const P: usize = 8; // m == n (feature dim)
568
569        // X is NSAMP×P row-major (stored as op==Yes's k×m). Deterministic, no RNG.
570        let mut x = vec![0.0f64; NSAMP * P];
571        for r in 0..NSAMP {
572            for c in 0..P {
573                x[r * P + c] = ((r * 3 + c * 7) % 17) as f64 * 0.125 - 1.0;
574            }
575        }
576
577        // Reference: Cov[i][j] = Σ_r X[r][i]·X[r][j], increasing-r accumulation.
578        let mut reference = vec![0.0f64; P * P];
579        for i in 0..P {
580            for j in 0..P {
581                let mut acc = 0.0f64;
582                for r in 0..NSAMP {
583                    acc += x[r * P + i] * x[r * P + j];
584                }
585                reference[i * P + j] = acc;
586            }
587        }
588
589        // Through the wired gemm: op(A)=Xᵀ (P×NSAMP), op(B)=X (NSAMP×P) → C (P×P).
590        let mut cov = vec![0.0f64; P * P];
591        gemm(
592            Transpose::Yes,
593            Transpose::No,
594            P,
595            P,
596            NSAMP,
597            1.0,
598            &x,
599            &x,
600            0.0,
601            &mut cov,
602        )
603        .unwrap();
604
605        approx(&cov, &reference);
606        // Covariance must be symmetric.
607        for i in 0..P {
608            for j in 0..P {
609                assert!((cov[i * P + j] - cov[j * P + i]).abs() < 1e-9);
610            }
611        }
612    }
613
614    #[test]
615    fn scaled_accumulate_transposed_matches_cpu_above_threshold() {
616        // Exercises the offload's `alpha`/`beta` combine *together with* a transpose, on
617        // the real PCA covariance shape: `C := alpha·(Xᵀ·X) + beta·C0` with
618        // alpha=1/(NSAMP−1) (exactly what `pca::fit` passes) and a non-zero beta so the
619        // `c`-read branch is hit. Above threshold (P²·NSAMP ≥ GEMM_GPU_THRESHOLD) → on an
620        // accelerator this is the materialise→dispatch→combine path; on CI the CPU floor.
621        // Either way it must equal the inline reference.
622        const NSAMP: usize = 4096;
623        const P: usize = 8;
624        let alpha = 1.0 / (NSAMP as f64 - 1.0);
625        let beta = 0.5;
626
627        let mut x = vec![0.0f64; NSAMP * P];
628        for r in 0..NSAMP {
629            for c in 0..P {
630                x[r * P + c] = ((r * 5 + c * 3) % 13) as f64 * 0.1 - 0.6;
631            }
632        }
633        // Initial C0 — read because beta != 0.
634        let mut c = vec![0.0f64; P * P];
635        for (i, ci) in c.iter_mut().enumerate() {
636            *ci = (i % 5) as f64 * 0.25;
637        }
638        let c0 = c.clone();
639
640        // Reference: alpha·Σ_r X[r][i]·X[r][j] + beta·C0[i][j].
641        let mut reference = vec![0.0f64; P * P];
642        for i in 0..P {
643            for j in 0..P {
644                let mut acc = 0.0f64;
645                for r in 0..NSAMP {
646                    acc += x[r * P + i] * x[r * P + j];
647                }
648                reference[i * P + j] = alpha * acc + beta * c0[i * P + j];
649            }
650        }
651
652        gemm(
653            Transpose::Yes,
654            Transpose::No,
655            P,
656            P,
657            NSAMP,
658            alpha,
659            &x,
660            &x,
661            beta,
662            &mut c,
663        )
664        .unwrap();
665        approx(&c, &reference);
666    }
667
668    #[test]
669    fn rejects_bad_dims() {
670        let a = [1.0, 2.0, 3.0, 4.0];
671        let b = [1.0, 2.0];
672        let mut c = [0.0; 4];
673        assert_eq!(
674            matmul(2, 2, 2, &a, &b, &mut c),
675            Err(SolversError::InvalidDimension)
676        );
677    }
678}