Skip to main content

qualia_core_db/inference/lab/
micro.rs

1//! Isolated Q4_K SoA GEMV microbench (CPU oracle + optional CUDA).
2
3use std::time::Instant;
4
5use crate::ggml_quants::{
6    dequantize_row_into, q4k_block_to_soa, BLOCK_Q4K_SOA_BYTES, GGML_TYPE_Q4_K_SOA,
7};
8use crate::inference_kernel_parity::{max_abs_err, max_ulp_diff};
9use crate::inference_modes::{set_inference_mode, InferenceMode};
10
11pub use etalon_inference_lab::micro::MicrobenchResult;
12
13fn synth_soa_weight(n_in: usize, n_out: usize) -> (Vec<u8>, Vec<f32>) {
14    let n_in = n_in.max(256) & !255;
15    let mut stock = [0u8; 144];
16    stock[0] = 0x00;
17    stock[1] = 0x3c;
18    stock[2] = 0x00;
19    stock[3] = 0x38;
20    for i in 4..144 {
21        stock[i] = (i as u8).wrapping_mul(17);
22    }
23    let mut soa = [0u8; BLOCK_Q4K_SOA_BYTES];
24    q4k_block_to_soa(&stock, &mut soa).expect("soa");
25    let n_blocks = n_in / 256;
26    let mut weight = Vec::with_capacity(n_out * n_blocks * BLOCK_Q4K_SOA_BYTES);
27    for _ in 0..n_out {
28        for _ in 0..n_blocks {
29            weight.extend_from_slice(&soa);
30        }
31    }
32    let x: Vec<f32> = (0..n_in).map(|i| (i as f32) * 0.01).collect();
33    (weight, x)
34}
35
36fn cpu_gemv(n_in: usize, n_out: usize, weight: &[u8], x: &[f32], out: &mut [f32]) {
37    let row_bytes = (n_in / 256) * BLOCK_Q4K_SOA_BYTES;
38    let mut row = vec![0.0f32; n_in];
39    for r in 0..n_out {
40        dequantize_row_into(
41            &weight[r * row_bytes..(r + 1) * row_bytes],
42            GGML_TYPE_Q4_K_SOA,
43            n_in,
44            &mut row,
45        )
46        .unwrap();
47        out[r] = row.iter().zip(x.iter()).map(|(a, b)| a * b).sum();
48    }
49}
50
51/// Run CPU Q4 SoA GEMV; if CUDA mode available, differential vs GPU.
52pub fn run_q4k_soa_microbench(n_in: usize, n_out: usize) -> MicrobenchResult {
53    let n_in = n_in.max(256) & !255;
54    let n_out = n_out.max(1).min(512);
55    let (weight, x) = synth_soa_weight(n_in, n_out);
56    let mut cpu_out = vec![0.0f32; n_out];
57
58    // Warm (untimed) — no Instant here (avoids unused `t0` if warm is dropped).
59    cpu_gemv(n_in, n_out, &weight, &x, &mut cpu_out);
60    let cpu_start = Instant::now();
61    cpu_gemv(n_in, n_out, &weight, &x, &mut cpu_out);
62    let cpu_ms = cpu_start.elapsed().as_secs_f64() * 1e3;
63    let flops = (n_in as f64) * (n_out as f64) * 2.0;
64    let cpu_gflops = (flops / (cpu_ms / 1e3).max(1e-9)) / 1e9;
65
66    let mut cuda_ms = None;
67    let mut cuda_ok = false;
68    let mut max_abs = 0.0f32;
69    let mut max_ulp = 0u64;
70
71    // CUDA differential when available.
72    let prev = std::env::var("QUALIA_INFERENCE_MODE").ok();
73    set_inference_mode(InferenceMode::CudaTc);
74    std::env::set_var("QUALIA_INFERENCE_MODE", "cuda");
75    #[cfg(feature = "cuda")]
76    let mut gpu_out = vec![0.0f32; n_out];
77    #[cfg(not(feature = "cuda"))]
78    let gpu_out = vec![0.0f32; n_out];
79    #[cfg(feature = "cuda")]
80    let ok = crate::try_q4k_soa_gemv(n_in, n_out, &x, &weight, &mut gpu_out);
81    #[cfg(not(feature = "cuda"))]
82    let ok = false;
83    // Single assignment (not `mut notes` + overwrite) so unused_assignments stays quiet.
84    let notes = if ok {
85        // Warm GPU path untimed; second call times sticky weights.
86        let cuda_start = Instant::now();
87        #[cfg(feature = "cuda")]
88        let ok2 = crate::try_q4k_soa_gemv(n_in, n_out, &x, &weight, &mut gpu_out);
89        #[cfg(not(feature = "cuda"))]
90        let ok2 = false;
91        cuda_ms = Some(cuda_start.elapsed().as_secs_f64() * 1e3);
92        cuda_ok = ok2;
93        max_abs = max_abs_err(&cpu_out, &gpu_out);
94        max_ulp = max_ulp_diff(&cpu_out, &gpu_out);
95        #[cfg(feature = "cuda")]
96        let wcount = crate::q4k_device_weight_count();
97        #[cfg(not(feature = "cuda"))]
98        let wcount = 0usize;
99        format!("cuda differential max_abs={max_abs:.6e} max_ulp={max_ulp} weights={wcount}")
100    } else {
101        "cuda unavailable or prefer_tensor_core_gemm false - cpu only".into()
102    };
103    match prev {
104        Some(v) => std::env::set_var("QUALIA_INFERENCE_MODE", v),
105        None => std::env::remove_var("QUALIA_INFERENCE_MODE"),
106    }
107    set_inference_mode(InferenceMode::Portable);
108
109    MicrobenchResult {
110        n_in,
111        n_out,
112        cpu_ms,
113        cpu_gflops,
114        cuda_ms,
115        cuda_ok,
116        max_abs_err: max_abs,
117        max_ulp,
118        notes,
119    }
120}
121
122#[cfg(test)]
123mod tests {
124    use super::*;
125
126    #[test]
127    #[serial_test::serial]
128    fn micro_cpu_runs() {
129        let r = run_q4k_soa_microbench(256, 4);
130        assert!(r.cpu_ms > 0.0);
131        assert!(r.cpu_gflops > 0.0);
132    }
133}