1#![cfg(not(target_arch = "wasm32"))]
17#![cfg_attr(not(feature = "gpu-runtime"), allow(dead_code))]
18
19use serde::{Deserialize, Serialize};
20#[cfg(feature = "gpu-runtime")]
21use std::process::Command;
22
23#[cfg(feature = "gpu-runtime")]
24use etalon_diagnostics::device::backend_name;
25#[cfg(any(feature = "gpu-runtime", test))]
26use etalon_diagnostics::ipc::encode_response;
27#[cfg(test)]
28use etalon_diagnostics::ipc::decode_response;
29use etalon_diagnostics::measurement::{bench_cpu_gemv, gflops};
30pub use etalon_diagnostics::{
31 CircuitBench, CircuitKind, DeviceBenchmarkRequest, DeviceBenchmarkResponse,
32};
33
34#[cfg(feature = "gpu-runtime")]
35fn backend_rank(b: etalon::Backend) -> u8 {
36 match b {
37 etalon::Backend::Metal => 0,
38 etalon::Backend::Dx12 => 1,
39 etalon::Backend::Vulkan => 2,
40 etalon::Backend::Gl => 3,
41 _ => 4,
42 }
43}
44
45#[cfg(not(test))]
46const WORKER_ENV: &str = "QUALIA_DEVICE_BENCHMARK_WORKER";
47const WORKER_OUTPUT_ENV: &str = "QUALIA_DEVICE_BENCHMARK_OUTPUT";
48
49#[derive(Debug, Clone, Serialize, Deserialize)]
51pub struct CapabilityMatrix {
52 pub circuits: Vec<CircuitBench>,
53 pub gemv_n: usize,
54 pub npu_probed: bool,
56}
57
58impl CapabilityMatrix {
59 pub fn best(&self) -> Option<&CircuitBench> {
61 self.circuits.first()
62 }
63
64 pub fn summary(&self) -> String {
66 let mut s = format!(
67 "CapabilityMatrix (GEMV {n}x{n}, ranked by measured throughput; NPU probed={npu}):\n",
68 n = self.gemv_n,
69 npu = self.npu_probed
70 );
71 for (i, c) in self.circuits.iter().enumerate() {
72 let upload = if c.upload_gbps.is_infinite() {
73 "in-pool".to_string()
74 } else {
75 format!("{:.1} GB/s up", c.upload_gbps)
76 };
77 let decode = match c.decode_proxy_tok_s {
78 Some(t) => format!("{t:.2} tok/s"),
79 None => "—".into(),
80 };
81 s.push_str(&format!(
82 " {}. {:<28} [{:?}/{}] {:>8.3} ms {:>7.1} GFLOP/s {:>12} decode {:>10} score {:.3} timing {:?}\n",
83 i + 1,
84 c.label,
85 c.kind,
86 c.backend,
87 c.ms_per_gemv,
88 c.gflops,
89 upload,
90 decode,
91 c.rel_score,
92 c.timing_source(),
93 ));
94 }
95 s
96 }
97
98 pub fn apply_decode_proxy_ranking(&mut self) {
101 let any_decode = self
102 .circuits
103 .iter()
104 .any(|c| c.decode_proxy_tok_s.is_some() && c.kind != CircuitKind::Cpu);
105 if !any_decode {
106 return;
107 }
108 self.circuits.sort_by(|a, b| {
109 let ta = a.decode_proxy_tok_s.unwrap_or(-1.0);
110 let tb = b.decode_proxy_tok_s.unwrap_or(-1.0);
111 match tb.partial_cmp(&ta).unwrap_or(std::cmp::Ordering::Equal) {
113 std::cmp::Ordering::Equal => a
114 .ms_per_gemv
115 .partial_cmp(&b.ms_per_gemv)
116 .unwrap_or(std::cmp::Ordering::Equal),
117 o => o,
118 }
119 });
120 if let Some(best_t) = self
121 .circuits
122 .iter()
123 .filter_map(|c| c.decode_proxy_tok_s)
124 .fold(None, |acc: Option<f64>, t| {
125 Some(acc.map(|a| a.max(t)).unwrap_or(t))
126 })
127 {
128 for c in &mut self.circuits {
129 c.rel_score = match c.decode_proxy_tok_s {
130 Some(t) if best_t > 0.0 => t / best_t,
131 _ => 0.0,
132 };
133 }
134 }
135 }
136}
137
138#[cfg(feature = "gpu-runtime")]
140pub fn run_worker_from_env() -> Result<(), String> {
141 let request: DeviceBenchmarkRequest = DeviceBenchmarkRequest {
142 backend: std::env::var("QUALIA_DEVICE_BENCHMARK_BACKEND").map_err(|_| "missing backend")?,
143 vendor: std::env::var("QUALIA_DEVICE_BENCHMARK_VENDOR")
144 .map_err(|_| "missing vendor")?
145 .parse()
146 .map_err(|_| "invalid vendor")?,
147 device: std::env::var("QUALIA_DEVICE_BENCHMARK_DEVICE")
148 .map_err(|_| "missing device")?
149 .parse()
150 .map_err(|_| "invalid device")?,
151 gemv_n: std::env::var("QUALIA_DEVICE_BENCHMARK_N")
152 .map_err(|_| "missing n")?
153 .parse()
154 .map_err(|_| "invalid n")?,
155 };
156 let response = match etalon_diagnostics::device::benchmark_one(&request) {
157 Ok(bench) => DeviceBenchmarkResponse {
158 request,
159 bench: Some(bench),
160 error: None,
161 },
162 Err(error) => DeviceBenchmarkResponse {
163 request,
164 bench: None,
165 error: Some(error),
166 },
167 };
168 let output = std::env::var_os(WORKER_OUTPUT_ENV).ok_or("missing worker output path")?;
169 encode_response(std::path::Path::new(&output), &response)
170}
171
172#[cfg(not(test))]
173fn worker_executable() -> Option<std::path::PathBuf> {
174 if let Some(path) = std::env::var_os(WORKER_ENV) {
175 return Some(path.into());
176 }
177 let current = std::env::current_exe().ok()?;
178 let sibling = current.with_file_name(if cfg!(windows) {
179 "qualia-device-benchmark-worker.exe"
180 } else {
181 "qualia-device-benchmark-worker"
182 });
183 if sibling.is_file() {
184 Some(sibling)
185 } else if current
186 .file_stem()
187 .and_then(|s| s.to_str())
188 .is_some_and(|name| name == "qualia-cli" || name == "webizen-desktop")
189 {
190 Some(current)
194 } else {
195 None
196 }
197}
198
199#[cfg(feature = "gpu-runtime")]
200fn invoke_worker(
201 request: &DeviceBenchmarkRequest,
202 cancelled: &dyn Fn() -> bool,
203) -> Result<CircuitBench, String> {
204 let artifacts = etalon_diagnostics::ipc::WorkerArtifacts::new()?;
205 let output = artifacts.response_path();
206 let mut command;
207 #[cfg(test)]
208 {
209 command = Command::new(std::env::current_exe().map_err(|e| e.to_string())?);
210 command.args([
211 "--exact",
212 "platform::device_benchmark::tests::device_benchmark_worker_entry",
213 "--nocapture",
214 ]);
215 }
216 #[cfg(not(test))]
217 {
218 command = Command::new(
219 worker_executable().ok_or_else(|| format!("worker not found; set {WORKER_ENV}"))?,
220 );
221 }
222 command
223 .env("QUALIA_DEVICE_BENCHMARK_BACKEND", &request.backend)
224 .env("QUALIA_DEVICE_BENCHMARK_VENDOR", request.vendor.to_string())
225 .env("QUALIA_DEVICE_BENCHMARK_DEVICE", request.device.to_string())
226 .env("QUALIA_DEVICE_BENCHMARK_N", request.gemv_n.to_string())
227 .env(WORKER_OUTPUT_ENV, &output);
228 etalon_diagnostics::process::run_worker(
229 &mut command,
230 &etalon_diagnostics::process::WorkerLimits::default(),
231 cancelled,
232 &|| artifacts.check_budget(),
233 )?;
234 etalon_diagnostics::ipc::decode_matching_response(&output, request)
235}
236
237pub fn benchmark_devices(n: usize) -> CapabilityMatrix {
244 match benchmark_devices_cancellable(n, &|| false) {
245 Ok(matrix) => matrix,
246 Err(error) => {
247 log::warn!("device_benchmark|failed|{error}");
248 CapabilityMatrix {
249 circuits: Vec::new(),
250 gemv_n: n,
251 npu_probed: false,
252 }
253 }
254 }
255}
256
257pub fn benchmark_devices_cancellable(
259 n: usize,
260 cancelled: &dyn Fn() -> bool,
261) -> Result<CapabilityMatrix, String> {
262 let mut circuits: Vec<CircuitBench> = Vec::new();
263 if cancelled() {
264 return Err("device benchmark cancelled".into());
265 }
266 etalon_diagnostics::measurement::validate_shape(
267 n,
268 etalon_diagnostics::measurement::DEFAULT_BENCH_BYTES,
269 )?;
270
271 #[cfg(feature = "gpu-runtime")]
273 {
274 let instance = etalon::Instance::default();
275 let mut cand: Vec<(u8, etalon::Adapter, etalon::AdapterInfo)> = Vec::new();
276 for adapter in pollster::block_on(instance.enumerate_adapters(etalon::Backends::all())) {
277 let info = adapter.get_info();
278 if info.device_type == etalon::DeviceType::Cpu || info.device == 0 {
279 continue;
280 }
281 cand.push((backend_rank(info.backend), adapter, info));
282 }
283 cand.sort_by_key(|(r, _, _)| *r);
284 let mut seen: std::collections::HashSet<(u32, u32, u32)> = std::collections::HashSet::new();
286 let mut chosen: Vec<DeviceBenchmarkRequest> = Vec::new();
287 for (_, _adapter, info) in cand {
288 let backend_id = info.backend as u32;
289 if seen.insert((info.vendor, info.device, backend_id)) {
290 chosen.push(DeviceBenchmarkRequest {
291 backend: backend_name(info.backend).to_string(),
292 vendor: info.vendor,
293 device: info.device,
294 gemv_n: n,
295 });
296 }
297 }
298 for request in chosen {
299 if cancelled() {
300 break;
301 }
302 match invoke_worker(&request, cancelled) {
303 Ok(bench) => circuits.push(bench),
304 Err(error) => log::warn!(
305 "device_benchmark|skip|{:04x}:{:04x}|{}|{}",
306 request.vendor,
307 request.device,
308 request.backend,
309 error
310 ),
311 }
312 }
313 }
314
315 if cancelled() {
316 return Err("device benchmark cancelled".into());
317 }
318 let cpu_ms = bench_cpu_gemv(n, etalon_diagnostics::measurement::DEFAULT_BENCH_BYTES)?;
320 if cancelled() {
321 return Err("device benchmark cancelled".into());
322 }
323 circuits.push(CircuitBench {
324 label: format!("CPU native (rayon, {} cores)", num_cpus::get()),
325 kind: CircuitKind::Cpu,
326 backend: "native".to_string(),
327 ms_per_gemv: cpu_ms,
328 gflops: gflops(n, cpu_ms),
329 upload_gbps: f64::INFINITY, rel_score: 1.0,
331 decode_proxy_tok_s: None,
332 });
333
334 circuits.sort_by(|a, b| {
336 a.ms_per_gemv
337 .partial_cmp(&b.ms_per_gemv)
338 .unwrap_or(std::cmp::Ordering::Equal)
339 .then_with(|| a.backend.cmp(&b.backend))
340 .then_with(|| a.label.cmp(&b.label))
341 });
342 if let Some(best_ms) = circuits.first().map(|c| c.ms_per_gemv) {
343 for c in &mut circuits {
344 c.rel_score = if c.ms_per_gemv > 0.0 {
345 best_ms / c.ms_per_gemv
346 } else {
347 0.0
348 };
349 }
350 }
351
352 Ok(CapabilityMatrix {
353 circuits,
354 gemv_n: n,
355 npu_probed: false,
356 })
357}
358
359#[cfg(test)]
360mod tests {
361 use super::*;
362
363 #[test]
364 #[cfg(feature = "gpu-runtime")]
365 fn device_benchmark_worker_entry() {
366 if std::env::var_os(WORKER_OUTPUT_ENV).is_some() {
367 run_worker_from_env().expect("device benchmark worker must write its response");
368 }
369 }
370
371 #[test]
372 fn worker_protocol_round_trips() {
373 let request = DeviceBenchmarkRequest {
374 backend: "vulkan".into(),
375 vendor: 1,
376 device: 2,
377 gemv_n: 32,
378 };
379 let response = DeviceBenchmarkResponse {
380 request: request.clone(),
381 bench: None,
382 error: Some("expected".into()),
383 };
384 let artifacts = etalon_diagnostics::ipc::WorkerArtifacts::new().unwrap();
385 let path = artifacts.response_path();
386 encode_response(&path, &response).unwrap();
387 let decoded = decode_response(&path).unwrap();
388 assert_eq!(decoded.request.backend, request.backend);
389 assert_eq!(decoded.error.as_deref(), Some("expected"));
390 }
391
392 #[test]
395 #[serial_test::serial(gpu)]
396 fn h1a_capability_matrix() {
397 let matrix = benchmark_devices(2048);
398 eprintln!("{}", matrix.summary());
399
400 assert!(
401 !matrix.circuits.is_empty(),
402 "at least the CPU circuit must be benchmarked"
403 );
404 assert!(
405 matrix.circuits.iter().any(|c| c.kind == CircuitKind::Cpu),
406 "native CPU circuit must always be present"
407 );
408 for w in matrix.circuits.windows(2) {
410 assert!(
411 w[0].ms_per_gemv <= w[1].ms_per_gemv + 1e-9,
412 "matrix must be sorted fastest-first"
413 );
414 }
415 assert!(
416 (matrix.best().unwrap().rel_score - 1.0).abs() < 1e-9,
417 "best score must be 1.0"
418 );
419 }
420}