qualia_core_db/solvers/linear_algebra/gemm.rs
1//! Dynamic-size general matrix multiply (GEMM) — the canonical dense-linear-algebra core.
2//!
3//! This is the **one** dynamic GEMM for the engine. Before this, three private
4//! re-implementations competed (`specialized_libs/linear_algebra` heap GEMM,
5//! `gguf_bridge` GPU `coop_gemv`, and the fixed-size [`super::Matrix4x4`]). The
6//! specialized libs route their dynamic matmul here; the GPU `coop_gemv` kernel is
7//! the *same contract* executed on `wgpu` and is checked against this code as its
8//! CPU parity reference (`gemm_parity_probe`).
9//!
10//! Idiom (matches `super::cholesky`): **zero allocation**, caller-owned **row-major**
11//! slices with explicit dimensions, fail-closed on a dimension mismatch
12//! (`SolversError::InvalidDimension`). No `DMatrix`, no heap, no dependency.
13
14use crate::solvers::SolversError;
15
16/// Whether a GEMM operand is used as stored (`No`) or transposed (`Yes`).
17///
18/// `op(X)` below means `X` when `No` and `Xᵀ` when `Yes`. Transpose is expressed
19/// by index arithmetic only — no operand is ever materialised.
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
21pub enum Transpose {
22 /// Use the operand as stored.
23 No,
24 /// Use the transpose of the operand.
25 Yes,
26}
27
28/// General matrix multiply (BLAS-3 `gemm` shape), row-major, zero-heap:
29///
30/// ```text
31/// C := alpha · op(A) · op(B) + beta · C
32/// ```
33///
34/// where `op(A)` is `m×k`, `op(B)` is `k×n`, and `C` is `m×n`. Operands are
35/// caller-owned row-major slices:
36/// - `a` holds `op==No ? m×k : k×m` → always `m*k` elements.
37/// - `b` holds `op==No ? k×n : n×k` → always `k*n` elements.
38/// - `c` is `m*n` elements, read (when `beta != 0`) and overwritten in place.
39///
40/// `beta == 0.0` is honoured as a hard zero (the existing contents of `c`, which
41/// may be uninitialised garbage, are *not* read) — matching BLAS semantics so a
42/// fresh output buffer need not be zeroed first.
43///
44/// Returns `SolversError::InvalidDimension` if any slice length disagrees with
45/// `m`, `n`, `k`.
46pub fn gemm(
47 transa: Transpose,
48 transb: Transpose,
49 m: usize,
50 n: usize,
51 k: usize,
52 alpha: f64,
53 a: &[f64],
54 b: &[f64],
55 beta: f64,
56 c: &mut [f64],
57) -> Result<(), SolversError> {
58 if a.len() != m * k || b.len() != k * n || c.len() != m * n {
59 return Err(SolversError::InvalidDimension);
60 }
61
62 // ── Best-path-with-CPU-floor offload (additive; behaviour-preserving) ──────────
63 //
64 // The capability-aware forge dispatcher (`wgsl_forge::dispatch::gemm_f64`) picks
65 // the best compute path *actually present on this machine* for the raw product
66 // `P = op(A)·op(B)`, while always keeping this CPU triple-loop as the floor. We hand
67 // off the product whenever:
68 // 1. an accelerator actually exists (`caps().cuda || caps().wgpu`) — so a no-GPU
69 // machine takes the EXACT existing CPU path below with zero behaviour change;
70 // 2. the problem is large enough (`m·n·k >= GEMM_GPU_THRESHOLD`) to amortise
71 // dispatch/transfer overhead — small GEMMs stay on the CPU.
72 //
73 // `gemm_f64` computes only the *plain* row-major product (no alpha/beta/transpose),
74 // so the full BLAS shape is reconstructed around it:
75 // • a transpose flag is honoured by *materialising* that operand once into a
76 // row-major scratch buffer (`op(A)` as `m×k`, `op(B)` as `k×n`) — an O(m·k)/O(k·n)
77 // copy dominated by the O(m·n·k) GEMM being offloaded. This unlocks the hot
78 // covariance `XᵀX` (PCA/ridge/linear) and attention `Q·Kᵀ`.
79 // • `alpha`/`beta` are applied in the O(m·n) CPU combine afterward, exactly as the
80 // loop below would (`beta == 0` is a hard zero, so `c` is not read — BLAS rule).
81 // The materialised operand equals exactly what `a_at`/`b_at` would read, and the
82 // dispatcher's CPU floor uses the same increasing-index accumulation as the loop
83 // below, so the answers agree to f64 summation precision.
84 //
85 // f64 on the GPU is **CUDA-only**: WGSL has no `f64`, so for a non-NVIDIA GPU the
86 // dispatcher itself falls back to its own f64 CPU floor (the `caps().wgpu` term just
87 // means "an accelerator is present"; the actual f64 GPU kernel is native CUDA-f64).
88 // Crucially, a forge error is NEVER propagated out of the solver: on `Err(_)` we fall
89 // through to the CPU path below, which is always correct. Sub-threshold or
90 // off-accelerator, the unchanged CPU code runs — byte-identical to before.
91 // GPU best-path GEMM via the forge — only when it's compiled in (native +
92 // wgsl-forge). On wasm32 the forge module doesn't exist; the CPU floor below runs.
93 #[cfg(all(not(target_arch = "wasm32"), feature = "wgsl-forge"))]
94 {
95 use crate::wgsl_forge::dispatch::{caps, GEMM_GPU_THRESHOLD};
96 use std::borrow::Cow;
97 let work = m.saturating_mul(n).saturating_mul(k);
98 // Skip caps() on small work — the CUDA probe can panic if libcuda is missing
99 // and we do not need it below the GPU threshold.
100 if work >= GEMM_GPU_THRESHOLD {
101 let caps = caps();
102 if caps.cuda || caps.wgpu {
103 // op(A): row-major m×k — stored m×k already (No) or k×m (Yes → transpose).
104 let a_eff: Cow<[f64]> = match transa {
105 Transpose::No => Cow::Borrowed(a),
106 Transpose::Yes => {
107 let mut t = vec![0.0_f64; m * k];
108 for i in 0..m {
109 for l in 0..k {
110 t[i * k + l] = a[l * m + i];
111 }
112 }
113 Cow::Owned(t)
114 }
115 };
116 // op(B): row-major k×n — stored k×n already (No) or n×k (Yes → transpose).
117 let b_eff: Cow<[f64]> = match transb {
118 Transpose::No => Cow::Borrowed(b),
119 Transpose::Yes => {
120 let mut t = vec![0.0_f64; k * n];
121 for l in 0..k {
122 for j in 0..n {
123 t[l * n + j] = b[j * k + l];
124 }
125 }
126 Cow::Owned(t)
127 }
128 };
129 // BLAS gemm dims are (m, n, k); the dispatcher takes (m, k, n): op(A) is m×k,
130 // op(B) is k×n, C is m×n, so the mapping is gemm(m,n,k) → dispatch(m,k,n).
131 if let Ok(product) = crate::wgsl_forge::dispatch::gemm_f64(m, k, n, &a_eff, &b_eff)
132 {
133 // Apply alpha/beta exactly as the CPU loop would (beta==0 ⇒ c not read).
134 if beta == 0.0 {
135 for (ci, &p) in c.iter_mut().zip(product.iter()) {
136 *ci = alpha * p;
137 }
138 } else {
139 for (ci, &p) in c.iter_mut().zip(product.iter()) {
140 *ci = alpha * p + beta * *ci;
141 }
142 }
143 return Ok(());
144 }
145 // Forge path was eligible but errored — fall through to the CPU floor.
146 }
147 }
148 }
149
150 // op(A)[i][l] — A stored m×k (No) or k×m (Yes).
151 let a_at = |i: usize, l: usize| -> f64 {
152 match transa {
153 Transpose::No => a[i * k + l],
154 Transpose::Yes => a[l * m + i],
155 }
156 };
157 // op(B)[l][j] — B stored k×n (No) or n×k (Yes).
158 let b_at = |l: usize, j: usize| -> f64 {
159 match transb {
160 Transpose::No => b[l * n + j],
161 Transpose::Yes => b[j * k + l],
162 }
163 };
164 for i in 0..m {
165 for j in 0..n {
166 let mut s = 0.0;
167 for l in 0..k {
168 s += a_at(i, l) * b_at(l, j);
169 }
170 let idx = i * n + j;
171 // beta==0 ⇒ hard zero, so c may be uninitialised on entry (BLAS rule).
172 c[idx] = if beta == 0.0 {
173 alpha * s
174 } else {
175 alpha * s + beta * c[idx]
176 };
177 }
178 }
179 Ok(())
180}
181
182/// Plain product `C := A · B` for `A` (`m×k`), `B` (`k×n`), `C` (`m×n`),
183/// all row-major and caller-owned. Thin wrapper over [`gemm`] with
184/// `alpha = 1`, `beta = 0` and no transposes.
185pub fn matmul(
186 m: usize,
187 k: usize,
188 n: usize,
189 a: &[f64],
190 b: &[f64],
191 c: &mut [f64],
192) -> Result<(), SolversError> {
193 gemm(Transpose::No, Transpose::No, m, n, k, 1.0, a, b, 0.0, c)
194}
195
196/// Matrix–vector product `y := op(A) · x`.
197///
198/// `op(A)` is `m×n`; `a` holds `op==No ? m×n : n×m` (always `m*n` elements),
199/// `x` is length `n`, `y` is length `m` (overwritten). Zero-heap; the dynamic
200/// analogue of [`super::Matrix4x4::multiply_vector`], and the shape the GPU
201/// `coop_gemv` decode kernel computes per output row.
202pub fn matvec(
203 transa: Transpose,
204 m: usize,
205 n: usize,
206 a: &[f64],
207 x: &[f64],
208 y: &mut [f64],
209) -> Result<(), SolversError> {
210 if a.len() != m * n || x.len() != n || y.len() != m {
211 return Err(SolversError::InvalidDimension);
212 }
213
214 // ── Best-path-with-CPU-floor offload (additive; behaviour-preserving) ──────────
215 //
216 // Mirror of the [`gemm`] fast-path above. The forge dispatcher
217 // (`wgsl_forge::dispatch::gemv_f64`) computes the *plain* product `y = A·x` only
218 // (it has no transpose support), so we only hand off when ALL hold:
219 // 1. it IS the plain product — `transa == Transpose::No` — because `gemv_f64`
220 // cannot express `Aᵀ·x`;
221 // 2. an accelerator actually exists (`caps().cuda || caps().wgpu`) — so a no-GPU
222 // machine takes the EXACT existing CPU loop below with zero behaviour change;
223 // 3. the problem is large enough (`m·n >= GEMM_GPU_THRESHOLD`) to amortise
224 // dispatch/transfer overhead — small GEMVs stay on the CPU.
225 //
226 // Dimensions map directly: here `A` is m×n (row-major), `x` is length n, `y` is
227 // length m — exactly `gemv_f64(m, n, a, x)`'s `y[M] = A[M×N]·x[N]` contract, so no
228 // re-mapping is needed (unlike gemm's (m,n,k)→(m,k,n) swap). f64 on the GPU is
229 // CUDA-only (WGSL has no f64); on a non-NVIDIA GPU the dispatcher itself falls back
230 // to its own f64 CPU floor, whose increasing-`j` summation order matches this loop,
231 // so answers agree to f64 summation precision.
232 //
233 // A forge error is NEVER propagated: on `Err(_)` we fall through to the CPU loop
234 // below, which is always correct. Any transpose, sub-threshold size, or
235 // off-accelerator run executes the unchanged CPU code — byte-identical to before.
236 #[cfg(all(not(target_arch = "wasm32"), feature = "wgsl-forge"))]
237 if transa == Transpose::No {
238 use crate::wgsl_forge::dispatch::{caps, GEMM_GPU_THRESHOLD};
239 let work = m.saturating_mul(n);
240 let caps = caps();
241 if (caps.cuda || caps.wgpu) && work >= GEMM_GPU_THRESHOLD {
242 if let Ok(result) = crate::wgsl_forge::dispatch::gemv_f64(m, n, a, x) {
243 y.copy_from_slice(&result);
244 return Ok(());
245 }
246 // Forge path was eligible but errored — fall through to the CPU floor.
247 }
248 }
249
250 let a_at = |i: usize, j: usize| -> f64 {
251 match transa {
252 Transpose::No => a[i * n + j],
253 Transpose::Yes => a[j * m + i],
254 }
255 };
256 for i in 0..m {
257 let mut s = 0.0;
258 for j in 0..n {
259 s += a_at(i, j) * x[j];
260 }
261 y[i] = s;
262 }
263 Ok(())
264}
265
266/// Transpose the `m×n` row-major matrix `a` into the `n×m` row-major buffer `out`.
267/// Caller-owned, zero-heap. `a` is `m*n`, `out` is `n*m`.
268pub fn transpose(m: usize, n: usize, a: &[f64], out: &mut [f64]) -> Result<(), SolversError> {
269 if a.len() != m * n || out.len() != m * n {
270 return Err(SolversError::InvalidDimension);
271 }
272 for i in 0..m {
273 for j in 0..n {
274 out[j * m + i] = a[i * n + j];
275 }
276 }
277 Ok(())
278}
279
280#[cfg(test)]
281mod tests {
282 use super::*;
283
284 fn approx(a: &[f64], b: &[f64]) {
285 assert_eq!(a.len(), b.len());
286 for i in 0..a.len() {
287 assert!((a[i] - b[i]).abs() < 1e-9, "idx {i}: {} != {}", a[i], b[i]);
288 }
289 }
290
291 #[test]
292 fn matmul_rectangular() {
293 // A (2×3) · B (3×2) = C (2×2)
294 let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
295 let b = [7.0, 8.0, 9.0, 10.0, 11.0, 12.0];
296 let mut c = [0.0; 4];
297 matmul(2, 3, 2, &a, &b, &mut c).unwrap();
298 // [[1*7+2*9+3*11, 1*8+2*10+3*12],[4*7+5*9+6*11, 4*8+5*10+6*12]]
299 approx(&c, &[58.0, 64.0, 139.0, 154.0]);
300 }
301
302 #[test]
303 fn matmul_identity_is_noop() {
304 let a = [1.0, 2.0, 3.0, 4.0];
305 let id = [1.0, 0.0, 0.0, 1.0];
306 let mut c = [0.0; 4];
307 matmul(2, 2, 2, &a, &id, &mut c).unwrap();
308 approx(&c, &a);
309 }
310
311 #[test]
312 fn gemm_alpha_beta_accumulate() {
313 // C := 2·A·B + 3·C
314 let a = [1.0, 2.0, 3.0, 4.0];
315 let b = [1.0, 0.0, 0.0, 1.0];
316 let mut c = [1.0, 1.0, 1.0, 1.0];
317 gemm(
318 Transpose::No,
319 Transpose::No,
320 2,
321 2,
322 2,
323 2.0,
324 &a,
325 &b,
326 3.0,
327 &mut c,
328 )
329 .unwrap();
330 // 2·A + 3·C0 = 2·[1,2,3,4] + 3·[1,1,1,1]
331 approx(&c, &[5.0, 7.0, 9.0, 11.0]);
332 }
333
334 #[test]
335 fn gemm_beta_zero_ignores_garbage() {
336 // beta=0 must not read c (here pre-filled with NaN-ish garbage).
337 let a = [1.0, 2.0, 3.0, 4.0];
338 let b = [1.0, 0.0, 0.0, 1.0];
339 let mut c = [f64::NAN, f64::NAN, f64::NAN, f64::NAN];
340 gemm(
341 Transpose::No,
342 Transpose::No,
343 2,
344 2,
345 2,
346 1.0,
347 &a,
348 &b,
349 0.0,
350 &mut c,
351 )
352 .unwrap();
353 approx(&c, &a);
354 }
355
356 #[test]
357 fn gemm_transpose_a_normal_equations() {
358 // AᵀA for A (3×2): expect symmetric 2×2. (op(A)=Aᵀ is 2×3, op(A)=A... )
359 // m=2, n=2, k=3: C = Aᵀ(2×3) · A(3×2).
360 let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 3×2, row-major
361 let mut c = [0.0; 4];
362 gemm(
363 Transpose::Yes,
364 Transpose::No,
365 2,
366 2,
367 3,
368 1.0,
369 &a,
370 &a,
371 0.0,
372 &mut c,
373 )
374 .unwrap();
375 // AᵀA = [[1+9+25, 2+12+30],[2+12+30, 4+16+36]] = [[35,44],[44,56]]
376 approx(&c, &[35.0, 44.0, 44.0, 56.0]);
377 }
378
379 #[test]
380 fn gemm_transpose_b() {
381 // A (2×3) · Bᵀ where B is (2×3) → C (2×2)
382 let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
383 let b = [1.0, 0.0, 1.0, 0.0, 1.0, 0.0]; // 2×3
384 let mut c = [0.0; 4];
385 gemm(
386 Transpose::No,
387 Transpose::Yes,
388 2,
389 2,
390 3,
391 1.0,
392 &a,
393 &b,
394 0.0,
395 &mut c,
396 )
397 .unwrap();
398 // op(B) = Bᵀ (3×2): rows of A dotted with rows of B.
399 // C[0][0]=1*1+2*0+3*1=4 ; C[0][1]=1*0+2*1+3*0=2
400 // C[1][0]=4*1+5*0+6*1=10; C[1][1]=4*0+5*1+6*0=5
401 approx(&c, &[4.0, 2.0, 10.0, 5.0]);
402 }
403
404 #[test]
405 fn matvec_basic() {
406 let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 2×3
407 let x = [1.0, 1.0, 1.0];
408 let mut y = [0.0; 2];
409 matvec(Transpose::No, 2, 3, &a, &x, &mut y).unwrap();
410 approx(&y, &[6.0, 15.0]);
411 }
412
413 #[test]
414 fn matvec_transposed_matches_dense_transpose() {
415 let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 2×3
416 let x = [1.0, 1.0];
417 let mut y = [0.0; 3];
418 // op(A) = Aᵀ (3×2) · x(2)
419 matvec(Transpose::Yes, 3, 2, &a, &x, &mut y).unwrap();
420 // Aᵀ rows: [1,4],[2,5],[3,6] dotted with [1,1] = 5,7,9
421 approx(&y, &[5.0, 7.0, 9.0]);
422 }
423
424 #[test]
425 fn transpose_roundtrip() {
426 let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 2×3
427 let mut t = [0.0; 6];
428 transpose(2, 3, &a, &mut t).unwrap();
429 approx(&t, &[1.0, 4.0, 2.0, 5.0, 3.0, 6.0]); // 3×2
430 let mut back = [0.0; 6];
431 transpose(3, 2, &t, &mut back).unwrap();
432 approx(&back, &a);
433 }
434
435 #[test]
436 fn gemm_via_transpose_equals_matvec() {
437 // A·x as a GEMM with n=1 must equal matvec.
438 let a = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; // 2×3
439 let x = [2.0, 0.0, 1.0];
440 let mut y_mv = [0.0; 2];
441 matvec(Transpose::No, 2, 3, &a, &x, &mut y_mv).unwrap();
442 let mut y_gemm = [0.0; 2];
443 gemm(
444 Transpose::No,
445 Transpose::No,
446 2,
447 1,
448 3,
449 1.0,
450 &a,
451 &x,
452 0.0,
453 &mut y_gemm,
454 )
455 .unwrap();
456 approx(&y_mv, &y_gemm);
457 }
458
459 #[test]
460 fn plain_gemm_matches_cpu_above_threshold() {
461 // A large plain product (48×48×48 = 110_592 ≥ GEMM_GPU_THRESHOLD) routed
462 // through `gemm(No, No, .., 1.0, a, b, 0.0, c)`. On an accelerator box this
463 // exercises the forge offload; on CI (no accelerator) it exercises the CPU
464 // floor. EITHER WAY the result must equal a SEPARATE pure-CPU triple-loop
465 // reference computed inline here, so the test is hermetic and GPU-agnostic.
466 const M: usize = 48;
467 const K: usize = 48;
468 const N: usize = 48;
469
470 // Deterministic data — no RNG, no GPU assumption.
471 let mut a = vec![0.0f64; M * K];
472 let mut b = vec![0.0f64; K * N];
473 for i in 0..M {
474 for l in 0..K {
475 a[i * K + l] = ((i * 7 + l * 3) % 11) as f64 * 0.25 - 1.0;
476 }
477 }
478 for l in 0..K {
479 for j in 0..N {
480 b[l * N + j] = ((l * 5 + j * 2) % 13) as f64 * 0.125 - 0.75;
481 }
482 }
483
484 // Reference: independent triple loop, increasing-k accumulation order.
485 let mut reference = vec![0.0f64; M * N];
486 for i in 0..M {
487 for j in 0..N {
488 let mut acc = 0.0f64;
489 for l in 0..K {
490 acc += a[i * K + l] * b[l * N + j];
491 }
492 reference[i * N + j] = acc;
493 }
494 }
495
496 // Through the wired gemm (plain product → eligible for offload).
497 let mut c = vec![0.0f64; M * N];
498 gemm(
499 Transpose::No,
500 Transpose::No,
501 M,
502 N,
503 K,
504 1.0,
505 &a,
506 &b,
507 0.0,
508 &mut c,
509 )
510 .unwrap();
511
512 approx(&c, &reference);
513 }
514
515 #[test]
516 fn plain_gemv_matches_cpu_above_threshold() {
517 // A large plain matrix–vector product (m=n=256 → work = 65_536 ≥
518 // GEMM_GPU_THRESHOLD=32_768) routed through `matvec(No, ..)`. On an
519 // accelerator box this exercises the forge offload (`dispatch::gemv_f64`);
520 // on CI (no accelerator) it exercises the CPU floor. EITHER WAY the result
521 // must equal a SEPARATE pure-CPU reference computed inline here, so the test
522 // is hermetic and GPU-agnostic.
523 const M: usize = 256;
524 const N: usize = 256;
525
526 // Deterministic data — no RNG, no GPU assumption.
527 let mut a = vec![0.0f64; M * N];
528 for i in 0..M {
529 for j in 0..N {
530 a[i * N + j] = ((i * 7 + j * 3) % 11) as f64 * 0.25 - 1.0;
531 }
532 }
533 let mut x = vec![0.0f64; N];
534 for (j, xj) in x.iter_mut().enumerate() {
535 *xj = ((j * 5) % 13) as f64 * 0.125 - 0.75;
536 }
537
538 // Reference: independent dot-product per row, increasing-j accumulation order.
539 let mut reference = vec![0.0f64; M];
540 for i in 0..M {
541 let row = i * N;
542 let mut acc = 0.0f64;
543 for j in 0..N {
544 acc += a[row + j] * x[j];
545 }
546 reference[i] = acc;
547 }
548
549 // Through the wired matvec (plain product → eligible for offload).
550 let mut y = vec![0.0f64; M];
551 matvec(Transpose::No, M, N, &a, &x, &mut y).unwrap();
552
553 approx(&y, &reference);
554 }
555
556 #[test]
557 fn transposed_covariance_matches_cpu_above_threshold() {
558 // The hot ML op: covariance `Cov = Xᵀ·X` for tall-skinny `X` (n≫p), expressed
559 // as `gemm(Transpose::Yes, Transpose::No, p, p, n, 1.0, x, x, 0.0, cov)`. With
560 // n=4096, p=8 the work is p·p·n = 262_144 ≥ GEMM_GPU_THRESHOLD, so on an
561 // accelerator box this exercises the NEW transpose-materialising offload path;
562 // on CI (no accelerator) it exercises the CPU floor. EITHER WAY the result must
563 // equal a SEPARATE pure-CPU `XᵀX` reference computed inline here — hermetic,
564 // GPU-agnostic, and the regression guard that materialise-then-dispatch equals
565 // the index-arithmetic CPU loop.
566 const NSAMP: usize = 4096; // k (contraction dim)
567 const P: usize = 8; // m == n (feature dim)
568
569 // X is NSAMP×P row-major (stored as op==Yes's k×m). Deterministic, no RNG.
570 let mut x = vec![0.0f64; NSAMP * P];
571 for r in 0..NSAMP {
572 for c in 0..P {
573 x[r * P + c] = ((r * 3 + c * 7) % 17) as f64 * 0.125 - 1.0;
574 }
575 }
576
577 // Reference: Cov[i][j] = Σ_r X[r][i]·X[r][j], increasing-r accumulation.
578 let mut reference = vec![0.0f64; P * P];
579 for i in 0..P {
580 for j in 0..P {
581 let mut acc = 0.0f64;
582 for r in 0..NSAMP {
583 acc += x[r * P + i] * x[r * P + j];
584 }
585 reference[i * P + j] = acc;
586 }
587 }
588
589 // Through the wired gemm: op(A)=Xᵀ (P×NSAMP), op(B)=X (NSAMP×P) → C (P×P).
590 let mut cov = vec![0.0f64; P * P];
591 gemm(
592 Transpose::Yes,
593 Transpose::No,
594 P,
595 P,
596 NSAMP,
597 1.0,
598 &x,
599 &x,
600 0.0,
601 &mut cov,
602 )
603 .unwrap();
604
605 approx(&cov, &reference);
606 // Covariance must be symmetric.
607 for i in 0..P {
608 for j in 0..P {
609 assert!((cov[i * P + j] - cov[j * P + i]).abs() < 1e-9);
610 }
611 }
612 }
613
614 #[test]
615 fn scaled_accumulate_transposed_matches_cpu_above_threshold() {
616 // Exercises the offload's `alpha`/`beta` combine *together with* a transpose, on
617 // the real PCA covariance shape: `C := alpha·(Xᵀ·X) + beta·C0` with
618 // alpha=1/(NSAMP−1) (exactly what `pca::fit` passes) and a non-zero beta so the
619 // `c`-read branch is hit. Above threshold (P²·NSAMP ≥ GEMM_GPU_THRESHOLD) → on an
620 // accelerator this is the materialise→dispatch→combine path; on CI the CPU floor.
621 // Either way it must equal the inline reference.
622 const NSAMP: usize = 4096;
623 const P: usize = 8;
624 let alpha = 1.0 / (NSAMP as f64 - 1.0);
625 let beta = 0.5;
626
627 let mut x = vec![0.0f64; NSAMP * P];
628 for r in 0..NSAMP {
629 for c in 0..P {
630 x[r * P + c] = ((r * 5 + c * 3) % 13) as f64 * 0.1 - 0.6;
631 }
632 }
633 // Initial C0 — read because beta != 0.
634 let mut c = vec![0.0f64; P * P];
635 for (i, ci) in c.iter_mut().enumerate() {
636 *ci = (i % 5) as f64 * 0.25;
637 }
638 let c0 = c.clone();
639
640 // Reference: alpha·Σ_r X[r][i]·X[r][j] + beta·C0[i][j].
641 let mut reference = vec![0.0f64; P * P];
642 for i in 0..P {
643 for j in 0..P {
644 let mut acc = 0.0f64;
645 for r in 0..NSAMP {
646 acc += x[r * P + i] * x[r * P + j];
647 }
648 reference[i * P + j] = alpha * acc + beta * c0[i * P + j];
649 }
650 }
651
652 gemm(
653 Transpose::Yes,
654 Transpose::No,
655 P,
656 P,
657 NSAMP,
658 alpha,
659 &x,
660 &x,
661 beta,
662 &mut c,
663 )
664 .unwrap();
665 approx(&c, &reference);
666 }
667
668 #[test]
669 fn rejects_bad_dims() {
670 let a = [1.0, 2.0, 3.0, 4.0];
671 let b = [1.0, 2.0];
672 let mut c = [0.0; 4];
673 assert_eq!(
674 matmul(2, 2, 2, &a, &b, &mut c),
675 Err(SolversError::InvalidDimension)
676 );
677 }
678}