Skip to main content

qualia_core_db/inference/inference_bench/
runner.rs

1//! The public benchmark entry points — drive one case (cold then warm) through
2//! the real inference path, average warm repeats, and assemble a [`BenchResult`];
3//! plus the suite runners and their blocking (own-runtime) wrappers. Includes the
4//! internal one-shot timing (`RunTiming` / `timed_infer`). Pure code motion.
5
6use std::sync::{Arc, Mutex};
7use std::time::{Duration, Instant};
8
9use crate::llm_agent::{AgentBackend, LocalLlmAgent};
10
11use super::metrics::{ms, ns_to_ms, tok_per_s};
12use super::*;
13
14// ── Internal one-shot timing ──────────────────────────────────────────────────
15
16struct RunTiming {
17    ttft: Duration,
18    total: Duration,
19    output_tokens: u64,
20}
21
22/// Drive one real inference, timestamping the first/last streamed token.
23fn timed_infer(agent: &LocalLlmAgent, prompt: &str) -> RunTiming {
24    let stamps: Arc<Mutex<Vec<Instant>>> = Arc::new(Mutex::new(Vec::with_capacity(64)));
25    let stamps_cb = Arc::clone(&stamps);
26    let start = Instant::now();
27    let cb = move |_delta: String| {
28        if let Ok(mut v) = stamps_cb.lock() {
29            v.push(Instant::now());
30        }
31    };
32    let (_text, _prov, tokens, _quin) = agent.infer_local_model_with_execution(
33        prompt,
34        "",
35        crate::llm_agent::NativeDecodeRequest::Developer,
36        None,
37        Some(cb),
38    );
39    let total = start.elapsed();
40    let v = stamps.lock().map(|g| g.clone()).unwrap_or_default();
41    let ttft = v.first().map(|f| f.duration_since(start)).unwrap_or(total);
42    RunTiming {
43        ttft,
44        total,
45        output_tokens: tokens as u64,
46    }
47}
48
49// ── Public entry ──────────────────────────────────────────────────────────────
50
51/// Run one benchmark case end-to-end (cold then warm) against the real path.
52///
53/// Must be called from within a multi-thread Tokio runtime context (the residency
54/// mount uses `block_in_place`). Returns `Err` if the model file is absent.
55pub fn run_bench(cfg: &BenchConfig) -> Result<BenchResult, String> {
56    if !std::path::Path::new(&cfg.model_path).exists() {
57        return Err(format!("model not found: {}", cfg.model_path));
58    }
59
60    let agent = LocalLlmAgent::with_local_backend(
61        "did:qualia:bench",
62        AgentBackend::Local {
63            model_path: cfg.model_path.clone(),
64            context_window: 4096,
65            quantization: cfg.quantization.clone(),
66            vision_projector_path: None,
67            modality: "text".into(),
68            architecture: None,
69        },
70    );
71
72    // Bound decode for a stable, comparable measurement.  This scoped override
73    // has precedence over application-profile bootstrap in the inference
74    // thread; the profile may reset its own normal UI budget without corrupting
75    // this run.
76    let _benchmark_budget = benchmark_decode_budget_scope(cfg.decode_tokens);
77
78    // ── COLD: ensure the model is NOT resident, then measure a fresh run. ──
79    crate::resident_model::clear_resident_model();
80    reset_phase_metrics();
81    let cold = timed_infer(&agent, &cfg.prompt);
82    if cold.output_tokens == 0 {
83        crate::resident_model::clear_resident_model();
84        return Err(format!(
85            "native inference produced no tokens during cold run for {} — benchmark aborted",
86            cfg.label
87        ));
88    }
89
90    // ── Make resident so warm runs adopt the mmap (skip disk load). ──
91    let model_id = crate::q_hash(&cfg.model_path);
92    let mut meta = ModelMeta::default();
93    if let Ok(report) = crate::resident_model::mount_resident_gguf(model_id, &cfg.model_path, false)
94    {
95        meta = ModelMeta {
96            n_layer: report.n_layer,
97            n_head: report.n_head,
98            n_kv_head: report.n_kv_head,
99            mapped_bytes: report.mapped_bytes,
100            kv_cache_bytes: report.kv_cache_bytes,
101            directml_enabled: report.directml_enabled,
102        };
103    }
104
105    // ── WARM: average over repeats. ──
106    let repeats = cfg.warm_repeats.max(1);
107    let mut warm_ttft = Duration::ZERO;
108    let mut warm_total = Duration::ZERO;
109    let mut acc_load = 0.0f64;
110    let mut acc_prefill_ns = 0u64;
111    let mut acc_prefill_tok = 0u64;
112    let mut acc_decode_ns = 0u64;
113    let mut acc_decode_tok = 0u64;
114    let mut last_warm = RunTiming {
115        ttft: Duration::ZERO,
116        total: Duration::ZERO,
117        output_tokens: 0,
118    };
119    for _ in 0..repeats {
120        reset_phase_metrics();
121        let w = timed_infer(&agent, &cfg.prompt);
122        if w.output_tokens == 0 {
123            crate::resident_model::clear_resident_model();
124            return Err(format!(
125                "native inference produced no tokens during warm run for {} — benchmark aborted",
126                cfg.label
127            ));
128        }
129        let snap = phase_snapshot();
130        warm_ttft += w.ttft;
131        warm_total += w.total;
132        acc_load += ns_to_ms(snap.load_ns);
133        acc_prefill_ns += snap.prefill_ns;
134        acc_prefill_tok += snap.prefill_tokens;
135        acc_decode_ns += snap.decode_ns;
136        acc_decode_tok += snap.decode_tokens;
137        last_warm = w;
138    }
139    let n = repeats as u32;
140
141    crate::resident_model::clear_resident_model();
142    let prompt_tokens = if repeats > 0 {
143        acc_prefill_tok / repeats as u64 + 1 // prefill covers prompt_len-1
144    } else {
145        0
146    };
147
148    #[cfg(feature = "gpu-runtime")]
149    let shared_gpu = crate::gpu_context::shared_gpu();
150    #[cfg(feature = "gpu-runtime")]
151    let gpu = BenchGpuMeta::from_shared_context(shared_gpu);
152    #[cfg(feature = "gpu-runtime")]
153    let gpu_timestamp_supported = shared_gpu.timestamps_supported;
154    #[cfg(not(feature = "gpu-runtime"))]
155    let gpu = BenchGpuMeta::unavailable();
156    #[cfg(not(feature = "gpu-runtime"))]
157    let gpu_timestamp_supported = false;
158
159    Ok(BenchResult {
160        label: cfg.label.clone(),
161        model_path: cfg.model_path.clone(),
162        quantization: cfg.quantization.clone(),
163        model: meta,
164        gpu,
165        prompt_tokens,
166        output_tokens: last_warm.output_tokens,
167        cold_ttft_ms: ms(cold.ttft),
168        cold_total_ms: ms(cold.total),
169        warm_ttft_ms: ms(warm_ttft) / n as f64,
170        warm_total_ms: ms(warm_total) / n as f64,
171        load_ms: acc_load / n as f64,
172        prefill_ms: ns_to_ms(acc_prefill_ns) / n as f64,
173        prefill_tok_s: tok_per_s(acc_prefill_tok, acc_prefill_ns),
174        decode_ms: ns_to_ms(acc_decode_ns) / n as f64,
175        decode_tok_s: tok_per_s(acc_decode_tok, acc_decode_ns),
176        // W2/D17: report the real device capability (TIMESTAMP_QUERY negotiation), not a hardcoded
177        // false. Per-kernel µs come from the dedicated `w2_gpu_phase_profile` test (a profiled run
178        // perturbs the headline tok/s, so the baseline run is left unprofiled).
179        gpu_timestamp_supported,
180        note: String::new(),
181    })
182}
183
184/// Run a suite of cases, skipping any whose model file is absent.
185pub fn run_suite(cfgs: &[BenchConfig]) -> Vec<BenchResult> {
186    let mut out = Vec::new();
187    for c in cfgs {
188        match run_bench(c) {
189            Ok(r) => out.push(r),
190            Err(e) => log::warn!("llm_bench|skip|{}|{}", c.label, e),
191        }
192    }
193    out
194}
195
196/// Run a suite inside a fresh multi-thread Tokio runtime.
197///
198/// `mount_resident_gguf` uses `block_in_place`, which requires a multi-thread
199/// runtime context — this wrapper provides one so callers (tests, CLI) don't have
200/// to. Safe to call from a plain (non-async) thread.
201pub fn run_suite_blocking(cfgs: &[BenchConfig]) -> Vec<BenchResult> {
202    let rt = tokio::runtime::Builder::new_multi_thread()
203        .worker_threads(2)
204        .enable_all()
205        .build()
206        .expect("tokio multi-thread runtime for llm_bench");
207    rt.block_on(async { run_suite(cfgs) })
208}