Skip to main content

qualia_core_db/inference/lab/
auto_improve.rs

1use crate::inference_modes::{set_inference_mode, InferenceMode};
2use crate::llm_bench::{
3    set_coop_gemv, set_ffn_fusion, set_kv_int8, set_resident_decode, set_resident_prefill,
4    set_resident_weights,
5};
6pub use engine::{format_lockin_summary, AutoImproveConfig};
7use etalon_inference_lab::auto_improve::{self as engine, AutoImproveHost, InstrumentReports};
8use std::path::Path;
9use std::time::Instant;
10pub type LabConfig = engine::LabConfig<InferenceMode>;
11pub type TrialResult = engine::TrialResult<InferenceMode>;
12pub type LockInPackage = engine::LockInPackage<InferenceMode>;
13impl engine::LabMode for InferenceMode {
14    const PORTABLE: Self = Self::Portable;
15    const FAST_VERIFY: Self = Self::FastVerify;
16    const CUDA_TC: Self = Self::CudaTc;
17    fn as_str(self) -> &'static str {
18        InferenceMode::as_str(self)
19    }
20}
21pub fn default_search_space() -> Vec<LabConfig> {
22    engine::default_search_space()
23}
24impl engine::ConfigApplier for InferenceMode {
25    fn apply_config(config: &LabConfig) {
26        set_resident_decode(config.resident);
27        set_resident_prefill(config.resident);
28        set_resident_weights(true);
29        set_coop_gemv(config.coop);
30        set_ffn_fusion(config.ffn_fusion);
31        set_kv_int8(config.kv_int8);
32        set_inference_mode(config.mode);
33        std::env::set_var("QUALIA_INFERENCE_MODE", config.mode.as_str());
34        if let Some(b) = config.backend {
35            std::env::set_var("QUALIA_WGPU_BACKEND", b);
36        }
37    }
38}
39struct QualiaAutoHost;
40impl AutoImproveHost<InferenceMode> for QualiaAutoHost {
41    fn instruments(&mut self) -> Result<InstrumentReports, String> {
42        let a = super::audit_path::audit_hot_path();
43        let r = super::device_roof::calibrate_device_roof(512);
44        let m = super::micro::run_q4k_soa_microbench(256, 32);
45        Ok(InstrumentReports {
46            audit: a.format_report(),
47            roof: r.format_report(),
48            micro: m.format_report(),
49            best_label: r.best_label,
50            micro_cuda_ok: m.cuda_ok,
51            micro_max_ulp: m.max_ulp,
52            notes: a.notes,
53            ffn_fusion_in_resident_decode: a.ffn_fusion_in_resident_decode,
54            passport_key: crate::hardware_passport::read_passport(
55                &crate::hardware_passport::default_cache_path(),
56            )
57            .map(|p| p.key)
58            .unwrap_or_default(),
59        })
60    }
61    fn measure(&mut self, model: &Path, tokens: u32, config: &LabConfig) -> TrialResult {
62        let _state = super::state::TrialState::capture();
63        config.apply();
64        let started = Instant::now();
65        let tok_s = crate::hardware_passport::measure_decode_proxy_tok_s(model, tokens);
66        TrialResult {
67            config: config.clone(),
68            tok_s,
69            wall_ms: started.elapsed().as_secs_f64() * 1000.0,
70            generation: 0,
71        }
72    }
73    fn external_comparison(&mut self, url: &str, model: &str, tokens: u32) -> Option<f64> {
74        try_ollama_decode_tok_s(url, model, tokens)
75    }
76}
77pub fn run_auto_improve(cfg: &AutoImproveConfig) -> Result<LockInPackage, String> {
78    engine::run_auto_improve(cfg, &mut QualiaAutoHost)
79}
80/// Probe Ollama decode tok/s if server is up (optional; not a product dependency).
81pub fn try_ollama_decode_tok_s(url: &str, model: &str, tokens: u32) -> Option<f64> {
82    // Use std only — no reqwest in core-db hot path. Best-effort via blocking HTTP if available.
83    // Core-db may not have reqwest; use a tiny TCP+manual approach is heavy.
84    // Prefer env-injected pre-measure; try `std::process` curl on Windows.
85    let body = format!(
86        r#"{{"model":"{model}","prompt":"Write a short paragraph about rivers.","stream":false,"options":{{"num_predict":{tokens},"temperature":0}}}}"#
87    );
88    let out = std::process::Command::new("curl")
89        .args([
90            "-sS",
91            "-X",
92            "POST",
93            &format!("{url}/api/generate"),
94            "-H",
95            "Content-Type: application/json",
96            "-d",
97            &body,
98            "--max-time",
99            "120",
100        ])
101        .output()
102        .ok()?;
103    if !out.status.success() {
104        return None;
105    }
106    let text = String::from_utf8_lossy(&out.stdout);
107    // Crude parse: "eval_count":N ... "eval_duration":N
108    let eval_count = parse_json_u64(&text, "eval_count")?;
109    let eval_duration = parse_json_u64(&text, "eval_duration")?;
110    if eval_duration == 0 {
111        return None;
112    }
113    Some((eval_count as f64) / (eval_duration as f64 / 1e9))
114}
115
116fn parse_json_u64(s: &str, key: &str) -> Option<u64> {
117    let pat = format!("\"{key}\":");
118    let i = s.find(&pat)?;
119    let rest = s[i + pat.len()..].trim_start();
120    let num: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
121    num.parse().ok()
122}
123
124#[cfg(test)]
125#[path = "auto_improve_tests.rs"]
126mod tests;