qualia_core_db/inference/lab/
micro.rs1use std::time::Instant;
4
5use crate::ggml_quants::{
6 dequantize_row_into, q4k_block_to_soa, BLOCK_Q4K_SOA_BYTES, GGML_TYPE_Q4_K_SOA,
7};
8use crate::inference_kernel_parity::{max_abs_err, max_ulp_diff};
9use crate::inference_modes::{set_inference_mode, InferenceMode};
10
11pub use etalon_inference_lab::micro::MicrobenchResult;
12
13fn synth_soa_weight(n_in: usize, n_out: usize) -> (Vec<u8>, Vec<f32>) {
14 let n_in = n_in.max(256) & !255;
15 let mut stock = [0u8; 144];
16 stock[0] = 0x00;
17 stock[1] = 0x3c;
18 stock[2] = 0x00;
19 stock[3] = 0x38;
20 for i in 4..144 {
21 stock[i] = (i as u8).wrapping_mul(17);
22 }
23 let mut soa = [0u8; BLOCK_Q4K_SOA_BYTES];
24 q4k_block_to_soa(&stock, &mut soa).expect("soa");
25 let n_blocks = n_in / 256;
26 let mut weight = Vec::with_capacity(n_out * n_blocks * BLOCK_Q4K_SOA_BYTES);
27 for _ in 0..n_out {
28 for _ in 0..n_blocks {
29 weight.extend_from_slice(&soa);
30 }
31 }
32 let x: Vec<f32> = (0..n_in).map(|i| (i as f32) * 0.01).collect();
33 (weight, x)
34}
35
36fn cpu_gemv(n_in: usize, n_out: usize, weight: &[u8], x: &[f32], out: &mut [f32]) {
37 let row_bytes = (n_in / 256) * BLOCK_Q4K_SOA_BYTES;
38 let mut row = vec![0.0f32; n_in];
39 for r in 0..n_out {
40 dequantize_row_into(
41 &weight[r * row_bytes..(r + 1) * row_bytes],
42 GGML_TYPE_Q4_K_SOA,
43 n_in,
44 &mut row,
45 )
46 .unwrap();
47 out[r] = row.iter().zip(x.iter()).map(|(a, b)| a * b).sum();
48 }
49}
50
51pub fn run_q4k_soa_microbench(n_in: usize, n_out: usize) -> MicrobenchResult {
53 let n_in = n_in.max(256) & !255;
54 let n_out = n_out.max(1).min(512);
55 let (weight, x) = synth_soa_weight(n_in, n_out);
56 let mut cpu_out = vec![0.0f32; n_out];
57
58 cpu_gemv(n_in, n_out, &weight, &x, &mut cpu_out);
60 let cpu_start = Instant::now();
61 cpu_gemv(n_in, n_out, &weight, &x, &mut cpu_out);
62 let cpu_ms = cpu_start.elapsed().as_secs_f64() * 1e3;
63 let flops = (n_in as f64) * (n_out as f64) * 2.0;
64 let cpu_gflops = (flops / (cpu_ms / 1e3).max(1e-9)) / 1e9;
65
66 let mut cuda_ms = None;
67 let mut cuda_ok = false;
68 let mut max_abs = 0.0f32;
69 let mut max_ulp = 0u64;
70
71 let prev = std::env::var("QUALIA_INFERENCE_MODE").ok();
73 set_inference_mode(InferenceMode::CudaTc);
74 std::env::set_var("QUALIA_INFERENCE_MODE", "cuda");
75 #[cfg(feature = "cuda")]
76 let mut gpu_out = vec![0.0f32; n_out];
77 #[cfg(not(feature = "cuda"))]
78 let gpu_out = vec![0.0f32; n_out];
79 #[cfg(feature = "cuda")]
80 let ok = crate::try_q4k_soa_gemv(n_in, n_out, &x, &weight, &mut gpu_out);
81 #[cfg(not(feature = "cuda"))]
82 let ok = false;
83 let notes = if ok {
85 let cuda_start = Instant::now();
87 #[cfg(feature = "cuda")]
88 let ok2 = crate::try_q4k_soa_gemv(n_in, n_out, &x, &weight, &mut gpu_out);
89 #[cfg(not(feature = "cuda"))]
90 let ok2 = false;
91 cuda_ms = Some(cuda_start.elapsed().as_secs_f64() * 1e3);
92 cuda_ok = ok2;
93 max_abs = max_abs_err(&cpu_out, &gpu_out);
94 max_ulp = max_ulp_diff(&cpu_out, &gpu_out);
95 #[cfg(feature = "cuda")]
96 let wcount = crate::q4k_device_weight_count();
97 #[cfg(not(feature = "cuda"))]
98 let wcount = 0usize;
99 format!("cuda differential max_abs={max_abs:.6e} max_ulp={max_ulp} weights={wcount}")
100 } else {
101 "cuda unavailable or prefer_tensor_core_gemm false - cpu only".into()
102 };
103 match prev {
104 Some(v) => std::env::set_var("QUALIA_INFERENCE_MODE", v),
105 None => std::env::remove_var("QUALIA_INFERENCE_MODE"),
106 }
107 set_inference_mode(InferenceMode::Portable);
108
109 MicrobenchResult {
110 n_in,
111 n_out,
112 cpu_ms,
113 cpu_gflops,
114 cuda_ms,
115 cuda_ok,
116 max_abs_err: max_abs,
117 max_ulp,
118 notes,
119 }
120}
121
122#[cfg(test)]
123mod tests {
124 use super::*;
125
126 #[test]
127 #[serial_test::serial]
128 fn micro_cpu_runs() {
129 let r = run_q4k_soa_microbench(256, 4);
130 assert!(r.cpu_ms > 0.0);
131 assert!(r.cpu_gflops > 0.0);
132 }
133}