Skip to main content

qualia_core_db/platform/
device_benchmark.rs

1//! STELLAR §A AH-track H1(a) — **cross-circuit capability benchmark** (decisions D26/D30).
2//!
3//! "Rank by *measured* throughput, not a static device-type hierarchy" (Timothy). This runs an
4//! identical representative GEMV on **every available compute circuit** — each wgpu adapter
5//! (discrete GPU, integrated GPU) plus a native-Rust CPU path — and produces a **capability
6//! matrix** sorted fastest-first. The residency/device planner (H2) consumes this to decide where
7//! work goes; e.g. a weak iGPU can still beat PCIe-streaming for overflow, and a many-core CPU can
8//! beat an old iGPU — only the numbers decide.
9//!
10//! Scope (honest): GPUs/iGPU via wgpu; CPU via a `rayon` GEMV. **NPU is not benchmarked** — NPU
11//! access is a platform API (DirectML / NNAPI / CoreML), not wgpu, and is reported as "not probed".
12//! This is part (a) of H1 (probe + matrix); the human-key *signing* of the passport (part (b)) is
13//! blocked on the identity remediation (`identity-governance-remediation.md`) and lives elsewhere.
14//!
15//! Native-only (wgpu adapters when `gpu-runtime` is on, plus a `rayon` CPU GEMV row).
16#![cfg(not(target_arch = "wasm32"))]
17#![cfg_attr(not(feature = "gpu-runtime"), allow(dead_code))]
18
19use serde::{Deserialize, Serialize};
20#[cfg(feature = "gpu-runtime")]
21use std::process::Command;
22
23#[cfg(feature = "gpu-runtime")]
24use etalon_diagnostics::device::backend_name;
25#[cfg(any(feature = "gpu-runtime", test))]
26use etalon_diagnostics::ipc::encode_response;
27#[cfg(test)]
28use etalon_diagnostics::ipc::decode_response;
29use etalon_diagnostics::measurement::{bench_cpu_gemv, gflops};
30pub use etalon_diagnostics::{
31    CircuitBench, CircuitKind, DeviceBenchmarkRequest, DeviceBenchmarkResponse,
32};
33
34#[cfg(feature = "gpu-runtime")]
35fn backend_rank(b: etalon::Backend) -> u8 {
36    match b {
37        etalon::Backend::Metal => 0,
38        etalon::Backend::Dx12 => 1,
39        etalon::Backend::Vulkan => 2,
40        etalon::Backend::Gl => 3,
41        _ => 4,
42    }
43}
44
45#[cfg(not(test))]
46const WORKER_ENV: &str = "QUALIA_DEVICE_BENCHMARK_WORKER";
47const WORKER_OUTPUT_ENV: &str = "QUALIA_DEVICE_BENCHMARK_OUTPUT";
48
49/// The measured capability matrix — circuits sorted fastest-first. This IS the priority order.
50#[derive(Debug, Clone, Serialize, Deserialize)]
51pub struct CapabilityMatrix {
52    pub circuits: Vec<CircuitBench>,
53    pub gemv_n: usize,
54    /// NPU left unprobed (no portable compute path); recorded for honesty.
55    pub npu_probed: bool,
56}
57
58impl CapabilityMatrix {
59    /// Highest-throughput circuit, if any.
60    pub fn best(&self) -> Option<&CircuitBench> {
61        self.circuits.first()
62    }
63
64    /// Summary.
65    pub fn summary(&self) -> String {
66        let mut s = format!(
67            "CapabilityMatrix (GEMV {n}x{n}, ranked by measured throughput; NPU probed={npu}):\n",
68            n = self.gemv_n,
69            npu = self.npu_probed
70        );
71        for (i, c) in self.circuits.iter().enumerate() {
72            let upload = if c.upload_gbps.is_infinite() {
73                "in-pool".to_string()
74            } else {
75                format!("{:.1} GB/s up", c.upload_gbps)
76            };
77            let decode = match c.decode_proxy_tok_s {
78                Some(t) => format!("{t:.2} tok/s"),
79                None => "—".into(),
80            };
81            s.push_str(&format!(
82                "  {}. {:<28} [{:?}/{}] {:>8.3} ms  {:>7.1} GFLOP/s  {:>12}  decode {:>10}  score {:.3}  timing {:?}\n",
83                i + 1,
84                c.label,
85                c.kind,
86                c.backend,
87                c.ms_per_gemv,
88                c.gflops,
89                upload,
90                decode,
91                c.rel_score,
92                c.timing_source(),
93            ));
94        }
95        s
96    }
97
98    /// Re-rank circuits: prefer higher `decode_proxy_tok_s` when present on any GPU row;
99    /// otherwise keep GEMV ranking. CPU rows without decode stay at the bottom of GPU ranking.
100    pub fn apply_decode_proxy_ranking(&mut self) {
101        let any_decode = self
102            .circuits
103            .iter()
104            .any(|c| c.decode_proxy_tok_s.is_some() && c.kind != CircuitKind::Cpu);
105        if !any_decode {
106            return;
107        }
108        self.circuits.sort_by(|a, b| {
109            let ta = a.decode_proxy_tok_s.unwrap_or(-1.0);
110            let tb = b.decode_proxy_tok_s.unwrap_or(-1.0);
111            // Higher tok/s first; unmeasured (-1) after measured.
112            match tb.partial_cmp(&ta).unwrap_or(std::cmp::Ordering::Equal) {
113                std::cmp::Ordering::Equal => a
114                    .ms_per_gemv
115                    .partial_cmp(&b.ms_per_gemv)
116                    .unwrap_or(std::cmp::Ordering::Equal),
117                o => o,
118            }
119        });
120        if let Some(best_t) = self
121            .circuits
122            .iter()
123            .filter_map(|c| c.decode_proxy_tok_s)
124            .fold(None, |acc: Option<f64>, t| {
125                Some(acc.map(|a| a.max(t)).unwrap_or(t))
126            })
127        {
128            for c in &mut self.circuits {
129                c.rel_score = match c.decode_proxy_tok_s {
130                    Some(t) if best_t > 0.0 => t / best_t,
131                    _ => 0.0,
132                };
133            }
134        }
135    }
136}
137
138/// Worker entry used by the dedicated binary and the unit-test subprocess route.
139#[cfg(feature = "gpu-runtime")]
140pub fn run_worker_from_env() -> Result<(), String> {
141    let request: DeviceBenchmarkRequest = DeviceBenchmarkRequest {
142        backend: std::env::var("QUALIA_DEVICE_BENCHMARK_BACKEND").map_err(|_| "missing backend")?,
143        vendor: std::env::var("QUALIA_DEVICE_BENCHMARK_VENDOR")
144            .map_err(|_| "missing vendor")?
145            .parse()
146            .map_err(|_| "invalid vendor")?,
147        device: std::env::var("QUALIA_DEVICE_BENCHMARK_DEVICE")
148            .map_err(|_| "missing device")?
149            .parse()
150            .map_err(|_| "invalid device")?,
151        gemv_n: std::env::var("QUALIA_DEVICE_BENCHMARK_N")
152            .map_err(|_| "missing n")?
153            .parse()
154            .map_err(|_| "invalid n")?,
155    };
156    let response = match etalon_diagnostics::device::benchmark_one(&request) {
157        Ok(bench) => DeviceBenchmarkResponse {
158            request,
159            bench: Some(bench),
160            error: None,
161        },
162        Err(error) => DeviceBenchmarkResponse {
163            request,
164            bench: None,
165            error: Some(error),
166        },
167    };
168    let output = std::env::var_os(WORKER_OUTPUT_ENV).ok_or("missing worker output path")?;
169    encode_response(std::path::Path::new(&output), &response)
170}
171
172#[cfg(not(test))]
173fn worker_executable() -> Option<std::path::PathBuf> {
174    if let Some(path) = std::env::var_os(WORKER_ENV) {
175        return Some(path.into());
176    }
177    let current = std::env::current_exe().ok()?;
178    let sibling = current.with_file_name(if cfg!(windows) {
179        "qualia-device-benchmark-worker.exe"
180    } else {
181        "qualia-device-benchmark-worker"
182    });
183    if sibling.is_file() {
184        Some(sibling)
185    } else if current
186        .file_stem()
187        .and_then(|s| s.to_str())
188        .is_some_and(|name| name == "qualia-cli" || name == "webizen-desktop")
189    {
190        // Qualia's shipped CLI and desktop hosts expose the same private worker
191        // entry before normal argument/UI initialization, so no sidecar is
192        // required for those packages. Other embedders can set WORKER_ENV.
193        Some(current)
194    } else {
195        None
196    }
197}
198
199#[cfg(feature = "gpu-runtime")]
200fn invoke_worker(
201    request: &DeviceBenchmarkRequest,
202    cancelled: &dyn Fn() -> bool,
203) -> Result<CircuitBench, String> {
204    let artifacts = etalon_diagnostics::ipc::WorkerArtifacts::new()?;
205    let output = artifacts.response_path();
206    let mut command;
207    #[cfg(test)]
208    {
209        command = Command::new(std::env::current_exe().map_err(|e| e.to_string())?);
210        command.args([
211            "--exact",
212            "platform::device_benchmark::tests::device_benchmark_worker_entry",
213            "--nocapture",
214        ]);
215    }
216    #[cfg(not(test))]
217    {
218        command = Command::new(
219            worker_executable().ok_or_else(|| format!("worker not found; set {WORKER_ENV}"))?,
220        );
221    }
222    command
223        .env("QUALIA_DEVICE_BENCHMARK_BACKEND", &request.backend)
224        .env("QUALIA_DEVICE_BENCHMARK_VENDOR", request.vendor.to_string())
225        .env("QUALIA_DEVICE_BENCHMARK_DEVICE", request.device.to_string())
226        .env("QUALIA_DEVICE_BENCHMARK_N", request.gemv_n.to_string())
227        .env(WORKER_OUTPUT_ENV, &output);
228    etalon_diagnostics::process::run_worker(
229        &mut command,
230        &etalon_diagnostics::process::WorkerLimits::default(),
231        cancelled,
232        &|| artifacts.check_budget(),
233    )?;
234    etalon_diagnostics::ipc::decode_matching_response(&output, request)
235}
236
237/// Benchmark every available compute circuit and return the ranked capability matrix.
238///
239/// `n` is the GEMV side length (representative shape; 2048 is a good fast default).
240/// **Each (vendor, device, backend) triple is benchmarked separately** so DX12 vs Vulkan
241/// (same physical GPU) can rank against each other — the whole point of the passport.
242/// The software/WARP "CPU" wgpu adapter is skipped — the native `rayon` path is the honest CPU number.
243pub fn benchmark_devices(n: usize) -> CapabilityMatrix {
244    match benchmark_devices_cancellable(n, &|| false) {
245        Ok(matrix) => matrix,
246        Err(error) => {
247            log::warn!("device_benchmark|failed|{error}");
248            CapabilityMatrix {
249                circuits: Vec::new(),
250                gemv_n: n,
251                npu_probed: false,
252            }
253        }
254    }
255}
256
257/// Cancellation aborts the matrix and kills the currently isolated adapter worker.
258pub fn benchmark_devices_cancellable(
259    n: usize,
260    cancelled: &dyn Fn() -> bool,
261) -> Result<CapabilityMatrix, String> {
262    let mut circuits: Vec<CircuitBench> = Vec::new();
263    if cancelled() {
264        return Err("device benchmark cancelled".into());
265    }
266    etalon_diagnostics::measurement::validate_shape(
267        n,
268        etalon_diagnostics::measurement::DEFAULT_BENCH_BYTES,
269    )?;
270
271    // ── GPUs / iGPU via wgpu — one circuit row per backend that can open the device ──
272    #[cfg(feature = "gpu-runtime")]
273    {
274        let instance = etalon::Instance::default();
275        let mut cand: Vec<(u8, etalon::Adapter, etalon::AdapterInfo)> = Vec::new();
276        for adapter in pollster::block_on(instance.enumerate_adapters(etalon::Backends::all())) {
277            let info = adapter.get_info();
278            if info.device_type == etalon::DeviceType::Cpu || info.device == 0 {
279                continue;
280            }
281            cand.push((backend_rank(info.backend), adapter, info));
282        }
283        cand.sort_by_key(|(r, _, _)| *r);
284        // Dedup exact (vendor, device, backend) only — keep Metal+DX12+Vulkan rows for the same card.
285        let mut seen: std::collections::HashSet<(u32, u32, u32)> = std::collections::HashSet::new();
286        let mut chosen: Vec<DeviceBenchmarkRequest> = Vec::new();
287        for (_, _adapter, info) in cand {
288            let backend_id = info.backend as u32;
289            if seen.insert((info.vendor, info.device, backend_id)) {
290                chosen.push(DeviceBenchmarkRequest {
291                    backend: backend_name(info.backend).to_string(),
292                    vendor: info.vendor,
293                    device: info.device,
294                    gemv_n: n,
295                });
296            }
297        }
298        for request in chosen {
299            if cancelled() {
300                break;
301            }
302            match invoke_worker(&request, cancelled) {
303                Ok(bench) => circuits.push(bench),
304                Err(error) => log::warn!(
305                    "device_benchmark|skip|{:04x}:{:04x}|{}|{}",
306                    request.vendor,
307                    request.device,
308                    request.backend,
309                    error
310                ),
311            }
312        }
313    }
314
315    if cancelled() {
316        return Err("device benchmark cancelled".into());
317    }
318    // ── CPU via native rayon ──
319    let cpu_ms = bench_cpu_gemv(n, etalon_diagnostics::measurement::DEFAULT_BENCH_BYTES)?;
320    if cancelled() {
321        return Err("device benchmark cancelled".into());
322    }
323    circuits.push(CircuitBench {
324        label: format!("CPU native (rayon, {} cores)", num_cpus::get()),
325        kind: CircuitKind::Cpu,
326        backend: "native".to_string(),
327        ms_per_gemv: cpu_ms,
328        gflops: gflops(n, cpu_ms),
329        upload_gbps: f64::INFINITY, // data already in the CPU's pool — no transfer
330        rel_score: 1.0,
331        decode_proxy_tok_s: None,
332    });
333
334    // Rank fastest-first and fill relative scores.
335    circuits.sort_by(|a, b| {
336        a.ms_per_gemv
337            .partial_cmp(&b.ms_per_gemv)
338            .unwrap_or(std::cmp::Ordering::Equal)
339            .then_with(|| a.backend.cmp(&b.backend))
340            .then_with(|| a.label.cmp(&b.label))
341    });
342    if let Some(best_ms) = circuits.first().map(|c| c.ms_per_gemv) {
343        for c in &mut circuits {
344            c.rel_score = if c.ms_per_gemv > 0.0 {
345                best_ms / c.ms_per_gemv
346            } else {
347                0.0
348            };
349        }
350    }
351
352    Ok(CapabilityMatrix {
353        circuits,
354        gemv_n: n,
355        npu_probed: false,
356    })
357}
358
359#[cfg(test)]
360mod tests {
361    use super::*;
362
363    #[test]
364    #[cfg(feature = "gpu-runtime")]
365    fn device_benchmark_worker_entry() {
366        if std::env::var_os(WORKER_OUTPUT_ENV).is_some() {
367            run_worker_from_env().expect("device benchmark worker must write its response");
368        }
369    }
370
371    #[test]
372    fn worker_protocol_round_trips() {
373        let request = DeviceBenchmarkRequest {
374            backend: "vulkan".into(),
375            vendor: 1,
376            device: 2,
377            gemv_n: 32,
378        };
379        let response = DeviceBenchmarkResponse {
380            request: request.clone(),
381            bench: None,
382            error: Some("expected".into()),
383        };
384        let artifacts = etalon_diagnostics::ipc::WorkerArtifacts::new().unwrap();
385        let path = artifacts.response_path();
386        encode_response(&path, &response).unwrap();
387        let decoded = decode_response(&path).unwrap();
388        assert_eq!(decoded.request.backend, request.backend);
389        assert_eq!(decoded.error.as_deref(), Some("expected"));
390    }
391
392    /// Cross-circuit benchmark on whatever silicon is present. Prints the ranked matrix; asserts the
393    /// CPU path always appears and the ranking is consistent. Skips GPU rows cleanly if headless.
394    #[test]
395    #[serial_test::serial(gpu)]
396    fn h1a_capability_matrix() {
397        let matrix = benchmark_devices(2048);
398        eprintln!("{}", matrix.summary());
399
400        assert!(
401            !matrix.circuits.is_empty(),
402            "at least the CPU circuit must be benchmarked"
403        );
404        assert!(
405            matrix.circuits.iter().any(|c| c.kind == CircuitKind::Cpu),
406            "native CPU circuit must always be present"
407        );
408        // Sorted fastest-first → non-decreasing ms, non-increasing rel_score.
409        for w in matrix.circuits.windows(2) {
410            assert!(
411                w[0].ms_per_gemv <= w[1].ms_per_gemv + 1e-9,
412                "matrix must be sorted fastest-first"
413            );
414        }
415        assert!(
416            (matrix.best().unwrap().rel_score - 1.0).abs() < 1e-9,
417            "best score must be 1.0"
418        );
419    }
420}