qualia_core_db/inference/inference_bench/
runner.rs1use std::sync::{Arc, Mutex};
7use std::time::{Duration, Instant};
8
9use crate::llm_agent::{AgentBackend, LocalLlmAgent};
10
11use super::metrics::{ms, ns_to_ms, tok_per_s};
12use super::*;
13
14struct RunTiming {
17 ttft: Duration,
18 total: Duration,
19 output_tokens: u64,
20}
21
22fn timed_infer(agent: &LocalLlmAgent, prompt: &str) -> RunTiming {
24 let stamps: Arc<Mutex<Vec<Instant>>> = Arc::new(Mutex::new(Vec::with_capacity(64)));
25 let stamps_cb = Arc::clone(&stamps);
26 let start = Instant::now();
27 let cb = move |_delta: String| {
28 if let Ok(mut v) = stamps_cb.lock() {
29 v.push(Instant::now());
30 }
31 };
32 let (_text, _prov, tokens, _quin) = agent.infer_local_model_with_execution(
33 prompt,
34 "",
35 crate::llm_agent::NativeDecodeRequest::Developer,
36 None,
37 Some(cb),
38 );
39 let total = start.elapsed();
40 let v = stamps.lock().map(|g| g.clone()).unwrap_or_default();
41 let ttft = v.first().map(|f| f.duration_since(start)).unwrap_or(total);
42 RunTiming {
43 ttft,
44 total,
45 output_tokens: tokens as u64,
46 }
47}
48
49pub fn run_bench(cfg: &BenchConfig) -> Result<BenchResult, String> {
56 if !std::path::Path::new(&cfg.model_path).exists() {
57 return Err(format!("model not found: {}", cfg.model_path));
58 }
59
60 let agent = LocalLlmAgent::with_local_backend(
61 "did:qualia:bench",
62 AgentBackend::Local {
63 model_path: cfg.model_path.clone(),
64 context_window: 4096,
65 quantization: cfg.quantization.clone(),
66 vision_projector_path: None,
67 modality: "text".into(),
68 architecture: None,
69 },
70 );
71
72 let _benchmark_budget = benchmark_decode_budget_scope(cfg.decode_tokens);
77
78 crate::resident_model::clear_resident_model();
80 reset_phase_metrics();
81 let cold = timed_infer(&agent, &cfg.prompt);
82 if cold.output_tokens == 0 {
83 crate::resident_model::clear_resident_model();
84 return Err(format!(
85 "native inference produced no tokens during cold run for {} — benchmark aborted",
86 cfg.label
87 ));
88 }
89
90 let model_id = crate::q_hash(&cfg.model_path);
92 let mut meta = ModelMeta::default();
93 if let Ok(report) = crate::resident_model::mount_resident_gguf(model_id, &cfg.model_path, false)
94 {
95 meta = ModelMeta {
96 n_layer: report.n_layer,
97 n_head: report.n_head,
98 n_kv_head: report.n_kv_head,
99 mapped_bytes: report.mapped_bytes,
100 kv_cache_bytes: report.kv_cache_bytes,
101 directml_enabled: report.directml_enabled,
102 };
103 }
104
105 let repeats = cfg.warm_repeats.max(1);
107 let mut warm_ttft = Duration::ZERO;
108 let mut warm_total = Duration::ZERO;
109 let mut acc_load = 0.0f64;
110 let mut acc_prefill_ns = 0u64;
111 let mut acc_prefill_tok = 0u64;
112 let mut acc_decode_ns = 0u64;
113 let mut acc_decode_tok = 0u64;
114 let mut last_warm = RunTiming {
115 ttft: Duration::ZERO,
116 total: Duration::ZERO,
117 output_tokens: 0,
118 };
119 for _ in 0..repeats {
120 reset_phase_metrics();
121 let w = timed_infer(&agent, &cfg.prompt);
122 if w.output_tokens == 0 {
123 crate::resident_model::clear_resident_model();
124 return Err(format!(
125 "native inference produced no tokens during warm run for {} — benchmark aborted",
126 cfg.label
127 ));
128 }
129 let snap = phase_snapshot();
130 warm_ttft += w.ttft;
131 warm_total += w.total;
132 acc_load += ns_to_ms(snap.load_ns);
133 acc_prefill_ns += snap.prefill_ns;
134 acc_prefill_tok += snap.prefill_tokens;
135 acc_decode_ns += snap.decode_ns;
136 acc_decode_tok += snap.decode_tokens;
137 last_warm = w;
138 }
139 let n = repeats as u32;
140
141 crate::resident_model::clear_resident_model();
142 let prompt_tokens = if repeats > 0 {
143 acc_prefill_tok / repeats as u64 + 1 } else {
145 0
146 };
147
148 #[cfg(feature = "gpu-runtime")]
149 let shared_gpu = crate::gpu_context::shared_gpu();
150 #[cfg(feature = "gpu-runtime")]
151 let gpu = BenchGpuMeta::from_shared_context(shared_gpu);
152 #[cfg(feature = "gpu-runtime")]
153 let gpu_timestamp_supported = shared_gpu.timestamps_supported;
154 #[cfg(not(feature = "gpu-runtime"))]
155 let gpu = BenchGpuMeta::unavailable();
156 #[cfg(not(feature = "gpu-runtime"))]
157 let gpu_timestamp_supported = false;
158
159 Ok(BenchResult {
160 label: cfg.label.clone(),
161 model_path: cfg.model_path.clone(),
162 quantization: cfg.quantization.clone(),
163 model: meta,
164 gpu,
165 prompt_tokens,
166 output_tokens: last_warm.output_tokens,
167 cold_ttft_ms: ms(cold.ttft),
168 cold_total_ms: ms(cold.total),
169 warm_ttft_ms: ms(warm_ttft) / n as f64,
170 warm_total_ms: ms(warm_total) / n as f64,
171 load_ms: acc_load / n as f64,
172 prefill_ms: ns_to_ms(acc_prefill_ns) / n as f64,
173 prefill_tok_s: tok_per_s(acc_prefill_tok, acc_prefill_ns),
174 decode_ms: ns_to_ms(acc_decode_ns) / n as f64,
175 decode_tok_s: tok_per_s(acc_decode_tok, acc_decode_ns),
176 gpu_timestamp_supported,
180 note: String::new(),
181 })
182}
183
184pub fn run_suite(cfgs: &[BenchConfig]) -> Vec<BenchResult> {
186 let mut out = Vec::new();
187 for c in cfgs {
188 match run_bench(c) {
189 Ok(r) => out.push(r),
190 Err(e) => log::warn!("llm_bench|skip|{}|{}", c.label, e),
191 }
192 }
193 out
194}
195
196pub fn run_suite_blocking(cfgs: &[BenchConfig]) -> Vec<BenchResult> {
202 let rt = tokio::runtime::Builder::new_multi_thread()
203 .worker_threads(2)
204 .enable_all()
205 .build()
206 .expect("tokio multi-thread runtime for llm_bench");
207 rt.block_on(async { run_suite(cfgs) })
208}