qualia_core_db/inference/lab/
auto_improve.rs1use crate::inference_modes::{set_inference_mode, InferenceMode};
2use crate::llm_bench::{
3 set_coop_gemv, set_ffn_fusion, set_kv_int8, set_resident_decode, set_resident_prefill,
4 set_resident_weights,
5};
6pub use engine::{format_lockin_summary, AutoImproveConfig};
7use etalon_inference_lab::auto_improve::{self as engine, AutoImproveHost, InstrumentReports};
8use std::path::Path;
9use std::time::Instant;
10pub type LabConfig = engine::LabConfig<InferenceMode>;
11pub type TrialResult = engine::TrialResult<InferenceMode>;
12pub type LockInPackage = engine::LockInPackage<InferenceMode>;
13impl engine::LabMode for InferenceMode {
14 const PORTABLE: Self = Self::Portable;
15 const FAST_VERIFY: Self = Self::FastVerify;
16 const CUDA_TC: Self = Self::CudaTc;
17 fn as_str(self) -> &'static str {
18 InferenceMode::as_str(self)
19 }
20}
21pub fn default_search_space() -> Vec<LabConfig> {
22 engine::default_search_space()
23}
24impl engine::ConfigApplier for InferenceMode {
25 fn apply_config(config: &LabConfig) {
26 set_resident_decode(config.resident);
27 set_resident_prefill(config.resident);
28 set_resident_weights(true);
29 set_coop_gemv(config.coop);
30 set_ffn_fusion(config.ffn_fusion);
31 set_kv_int8(config.kv_int8);
32 set_inference_mode(config.mode);
33 std::env::set_var("QUALIA_INFERENCE_MODE", config.mode.as_str());
34 if let Some(b) = config.backend {
35 std::env::set_var("QUALIA_WGPU_BACKEND", b);
36 }
37 }
38}
39struct QualiaAutoHost;
40impl AutoImproveHost<InferenceMode> for QualiaAutoHost {
41 fn instruments(&mut self) -> Result<InstrumentReports, String> {
42 let a = super::audit_path::audit_hot_path();
43 let r = super::device_roof::calibrate_device_roof(512);
44 let m = super::micro::run_q4k_soa_microbench(256, 32);
45 Ok(InstrumentReports {
46 audit: a.format_report(),
47 roof: r.format_report(),
48 micro: m.format_report(),
49 best_label: r.best_label,
50 micro_cuda_ok: m.cuda_ok,
51 micro_max_ulp: m.max_ulp,
52 notes: a.notes,
53 ffn_fusion_in_resident_decode: a.ffn_fusion_in_resident_decode,
54 passport_key: crate::hardware_passport::read_passport(
55 &crate::hardware_passport::default_cache_path(),
56 )
57 .map(|p| p.key)
58 .unwrap_or_default(),
59 })
60 }
61 fn measure(&mut self, model: &Path, tokens: u32, config: &LabConfig) -> TrialResult {
62 let _state = super::state::TrialState::capture();
63 config.apply();
64 let started = Instant::now();
65 let tok_s = crate::hardware_passport::measure_decode_proxy_tok_s(model, tokens);
66 TrialResult {
67 config: config.clone(),
68 tok_s,
69 wall_ms: started.elapsed().as_secs_f64() * 1000.0,
70 generation: 0,
71 }
72 }
73 fn external_comparison(&mut self, url: &str, model: &str, tokens: u32) -> Option<f64> {
74 try_ollama_decode_tok_s(url, model, tokens)
75 }
76}
77pub fn run_auto_improve(cfg: &AutoImproveConfig) -> Result<LockInPackage, String> {
78 engine::run_auto_improve(cfg, &mut QualiaAutoHost)
79}
80pub fn try_ollama_decode_tok_s(url: &str, model: &str, tokens: u32) -> Option<f64> {
82 let body = format!(
86 r#"{{"model":"{model}","prompt":"Write a short paragraph about rivers.","stream":false,"options":{{"num_predict":{tokens},"temperature":0}}}}"#
87 );
88 let out = std::process::Command::new("curl")
89 .args([
90 "-sS",
91 "-X",
92 "POST",
93 &format!("{url}/api/generate"),
94 "-H",
95 "Content-Type: application/json",
96 "-d",
97 &body,
98 "--max-time",
99 "120",
100 ])
101 .output()
102 .ok()?;
103 if !out.status.success() {
104 return None;
105 }
106 let text = String::from_utf8_lossy(&out.stdout);
107 let eval_count = parse_json_u64(&text, "eval_count")?;
109 let eval_duration = parse_json_u64(&text, "eval_duration")?;
110 if eval_duration == 0 {
111 return None;
112 }
113 Some((eval_count as f64) / (eval_duration as f64 / 1e9))
114}
115
116fn parse_json_u64(s: &str, key: &str) -> Option<u64> {
117 let pat = format!("\"{key}\":");
118 let i = s.find(&pat)?;
119 let rest = s[i + pat.len()..].trim_start();
120 let num: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
121 num.parse().ok()
122}
123
124#[cfg(test)]
125#[path = "auto_improve_tests.rs"]
126mod tests;