qualia_core_db/inference/lab/
auto_improve.rs

1//! Autonomous multi-hour lab loop: measure → search configs → re-measure → lock-in.
2//!
3//! This is **not** an LLM rewriting kernels. It is a disciplined experimental
4//! program that:
5//! 1. Explores a discrete configuration space (resident, coop, kv, mode, backend)
6//! 2. Optionally re-samples the best configs (self-improvement via evidence)
7//! 3. Tracks plateau / wall-clock budget
8//! 4. Emits a **lock-in package**: best config, full CSV, methodology text, apply script
9//!
10//! Plan: `docs/plans/inference-superiority-lab-and-toolset-plan.md` L4.
11
12use std::fs::{create_dir_all, File};
13use std::io::Write;
14use std::path::{Path, PathBuf};
15use std::time::{Duration, Instant};
16
17use crate::hardware_passport::measure_decode_proxy_tok_s;
18use crate::inference_modes::{set_inference_mode, InferenceMode};
19use crate::lab::audit_path::audit_hot_path;
20use crate::lab::device_roof::calibrate_device_roof;
21use crate::lab::experiment_log::{append_run_csv, ExperimentRun};
22use crate::lab::micro::run_q4k_soa_microbench;
23use crate::llm_bench::{
24    set_coop_gemv, set_ffn_fusion, set_kv_int8, set_resident_decode, set_resident_prefill,
25    set_resident_weights,
26};
27
28/// One point in the searchable config space.
29#[derive(Debug, Clone, PartialEq, Eq, Hash)]
30pub struct LabConfig {
31    pub label: String,
32    pub resident: bool,
33    pub coop: bool,
34    pub ffn_fusion: bool,
35    pub kv_int8: bool,
36    pub mode: InferenceMode,
37    pub backend: Option<&'static str>, // None = leave env alone
38}
39
40impl LabConfig {
41    pub fn toggles_json(&self) -> String {
42        format!(
43            "{{\"resident\":{},\"coop\":{},\"ffn_fusion\":{},\"kv_int8\":{},\"mode\":\"{}\",\"backend\":\"{}\"}}",
44            self.resident,
45            self.coop,
46            self.ffn_fusion,
47            self.kv_int8,
48            self.mode.as_str(),
49            self.backend.unwrap_or("auto")
50        )
51    }
52
53    pub fn apply(&self) {
54        set_resident_decode(self.resident);
55        set_resident_prefill(self.resident);
56        set_resident_weights(true);
57        set_coop_gemv(self.coop);
58        set_ffn_fusion(self.ffn_fusion);
59        set_kv_int8(self.kv_int8);
60        set_inference_mode(self.mode);
61        std::env::set_var("QUALIA_INFERENCE_MODE", self.mode.as_str());
62        match self.backend {
63            Some(b) => std::env::set_var("QUALIA_WGPU_BACKEND", b),
64            None => {
65                // Leave passport/path_select free unless already set by operator
66            }
67        }
68    }
69}
70
71/// Search space for recursive improvement (expandable).
72pub fn default_search_space() -> Vec<LabConfig> {
73    let mut v = Vec::new();
74    let modes = [
75        InferenceMode::Portable,
76        InferenceMode::FastVerify,
77        InferenceMode::CudaTc,
78    ];
79    let backends: [Option<&'static str>; 3] = [None, Some("vulkan"), Some("dx12")];
80    for &resident in &[true, false] {
81        for &coop in &[true, false] {
82            for &kv in &[true, false] {
83                for &mode in &modes {
84                    // Skip nonsense combos early
85                    if !resident && mode == InferenceMode::CudaTc {
86                        continue;
87                    }
88                    for &backend in &backends {
89                        // Only vary backend for portable/fast-verify resident path
90                        if backend.is_some() && (!resident || mode == InferenceMode::CudaTc) {
91                            continue;
92                        }
93                        let label = format!(
94                            "r{}_c{}_k{}_{}_{}",
95                            resident as u8,
96                            coop as u8,
97                            kv as u8,
98                            mode.as_str(),
99                            backend.unwrap_or("auto")
100                        );
101                        v.push(LabConfig {
102                            label,
103                            resident,
104                            coop,
105                            ffn_fusion: true, // flag only; audit says not in resident yet
106                            kv_int8: kv,
107                            mode,
108                            backend,
109                        });
110                    }
111                }
112            }
113        }
114    }
115    v
116}
117
118#[derive(Debug, Clone)]
119pub struct TrialResult {
120    pub config: LabConfig,
121    pub tok_s: Option<f64>,
122    pub wall_ms: f64,
123    pub generation: u32,
124}
125
126#[derive(Debug, Clone)]
127pub struct AutoImproveConfig {
128    pub model: PathBuf,
129    pub tokens: u32,
130    pub max_duration: Duration,
131    pub out_dir: PathBuf,
132    pub ollama_model: Option<String>,
133    pub ollama_url: String,
134    /// Re-sample top-k each generation (self-improve via variance reduction).
135    pub elite_resample: usize,
136    /// Stop if best has not improved by this relative fraction for `plateau_gens` gens.
137    pub plateau_rel: f64,
138    pub plateau_gens: u32,
139    pub max_generations: u32,
140}
141
142impl Default for AutoImproveConfig {
143    fn default() -> Self {
144        Self {
145            model: PathBuf::from("C:\\LLM_Models\\P64\\smollm2-360m-instruct-q8_0.p64"),
146            tokens: 16,
147            max_duration: Duration::from_secs(2 * 3600),
148            out_dir: PathBuf::from("experiments/inference-lab/lockin"),
149            ollama_model: Some("qualia-smol-q8:latest".into()),
150            ollama_url: "http://127.0.0.1:11434".into(),
151            elite_resample: 3,
152            plateau_rel: 0.02,
153            plateau_gens: 2,
154            max_generations: 8,
155        }
156    }
157}
158
159#[derive(Debug, Clone)]
160pub struct LockInPackage {
161    pub best: Option<TrialResult>,
162    pub ollama_tok_s: Option<f64>,
163    pub a_gap: Option<f64>,
164    pub trials: usize,
165    pub generations: u32,
166    pub elapsed_secs: f64,
167    pub out_dir: PathBuf,
168    pub methodology: String,
169}
170
171/// Probe Ollama decode tok/s if server is up (optional; not a product dependency).
172pub fn try_ollama_decode_tok_s(url: &str, model: &str, tokens: u32) -> Option<f64> {
173    // Use std only — no reqwest in core-db hot path. Best-effort via blocking HTTP if available.
174    // Core-db may not have reqwest; use a tiny TCP+manual approach is heavy.
175    // Prefer env-injected pre-measure; try `std::process` curl on Windows.
176    let body = format!(
177        r#"{{"model":"{model}","prompt":"Write a short paragraph about rivers.","stream":false,"options":{{"num_predict":{tokens},"temperature":0}}}}"#
178    );
179    let out = std::process::Command::new("curl")
180        .args([
181            "-sS",
182            "-X",
183            "POST",
184            &format!("{url}/api/generate"),
185            "-H",
186            "Content-Type: application/json",
187            "-d",
188            &body,
189            "--max-time",
190            "120",
191        ])
192        .output()
193        .ok()?;
194    if !out.status.success() {
195        return None;
196    }
197    let text = String::from_utf8_lossy(&out.stdout);
198    // Crude parse: "eval_count":N ... "eval_duration":N
199    let eval_count = parse_json_u64(&text, "eval_count")?;
200    let eval_duration = parse_json_u64(&text, "eval_duration")?;
201    if eval_duration == 0 {
202        return None;
203    }
204    Some((eval_count as f64) / (eval_duration as f64 / 1e9))
205}
206
207fn parse_json_u64(s: &str, key: &str) -> Option<u64> {
208    let pat = format!("\"{key}\":");
209    let i = s.find(&pat)?;
210    let rest = s[i + pat.len()..].trim_start();
211    let num: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
212    num.parse().ok()
213}
214
215fn measure_config(model: &Path, tokens: u32, cfg: &LabConfig) -> TrialResult {
216    cfg.apply();
217    let t0 = Instant::now();
218    let tok_s = measure_decode_proxy_tok_s(model, tokens);
219    let wall_ms = t0.elapsed().as_secs_f64() * 1e3;
220    TrialResult {
221        config: cfg.clone(),
222        tok_s,
223        wall_ms,
224        generation: 0,
225    }
226}
227
228/// Run the recursive lab program; write lock-in package under `cfg.out_dir`.
229pub fn run_auto_improve(cfg: &AutoImproveConfig) -> Result<LockInPackage, String> {
230    if !cfg.model.is_file() {
231        return Err(format!("model not found: {}", cfg.model.display()));
232    }
233    create_dir_all(&cfg.out_dir).map_err(|e| e.to_string())?;
234    let csv_path = cfg.out_dir.join("runs.csv");
235    let log_path = cfg.out_dir.join("auto_improve.log");
236    let mut log = File::create(&log_path).map_err(|e| e.to_string())?;
237
238    macro_rules! logln {
239        ($($t:tt)*) => {{
240            let line = format!($($t)*);
241            let _ = writeln!(log, "{line}");
242            let _ = log.flush();
243            log::info!("lab_auto|{line}");
244            eprintln!("lab_auto|{line}");
245        }};
246    }
247
248    logln!(
249        "start model={} tokens={} budget_s={:.0}",
250        cfg.model.display(),
251        cfg.tokens,
252        cfg.max_duration.as_secs_f64()
253    );
254
255    // Baseline instruments
256    let audit = audit_hot_path();
257    let _ = std::fs::write(cfg.out_dir.join("audit_path.txt"), audit.format_report());
258    let roof = calibrate_device_roof(512);
259    let _ = std::fs::write(cfg.out_dir.join("device_roof.txt"), roof.format_report());
260    let micro = run_q4k_soa_microbench(256, 32);
261    let _ = std::fs::write(cfg.out_dir.join("micro_q4k.txt"), micro.format_report());
262
263    let ollama_tok_s = cfg.ollama_model.as_ref().and_then(|m| {
264        logln!("ollama_probe model={m}");
265        try_ollama_decode_tok_s(&cfg.ollama_url, m, cfg.tokens.max(16))
266    });
267    if let Some(o) = ollama_tok_s {
268        logln!("ollama_decode_tok_s={o:.3}");
269    } else {
270        logln!("ollama_probe skipped or failed");
271    }
272
273    let space = default_search_space();
274    logln!("search_space_size={}", space.len());
275
276    let t_start = Instant::now();
277    let mut all_trials: Vec<TrialResult> = Vec::new();
278    let mut best: Option<TrialResult> = None;
279    let mut gens_without_improve = 0u32;
280    let mut generation = 0u32;
281
282    let model_id = cfg
283        .model
284        .file_name()
285        .and_then(|s| s.to_str())
286        .unwrap_or("model")
287        .to_string();
288    let passport_key =
289        crate::hardware_passport::read_passport(&crate::hardware_passport::default_cache_path())
290            .map(|p| p.key)
291            .unwrap_or_default();
292
293    // Generation 0: full space (or truncated if huge — still fine)
294    let mut queue: Vec<LabConfig> = space;
295    while generation < cfg.max_generations && t_start.elapsed() < cfg.max_duration {
296        generation += 1;
297        logln!(
298            "generation={generation} queue={} elapsed_s={:.0}",
299            queue.len(),
300            t_start.elapsed().as_secs_f64()
301        );
302
303        let gen_best_before = best.as_ref().and_then(|b| b.tok_s).unwrap_or(0.0);
304
305        for cfg_point in queue.drain(..) {
306            if t_start.elapsed() >= cfg.max_duration {
307                logln!("budget_exhausted mid-generation");
308                break;
309            }
310            let mut trial = measure_config(&cfg.model, cfg.tokens, &cfg_point);
311            trial.generation = generation;
312            logln!(
313                "trial gen={generation} label={} tok_s={:?} wall_ms={:.0}",
314                trial.config.label,
315                trial.tok_s,
316                trial.wall_ms
317            );
318
319            // CSV
320            let mut run = ExperimentRun {
321                run_id: ExperimentRun::new_id(),
322                utc: ExperimentRun::utc_now(),
323                git_sha: String::new(),
324                host_passport_key: passport_key.clone(),
325                adapter: roof.best_label.clone(),
326                backend: trial.config.backend.unwrap_or("auto").to_string(),
327                model_id: model_id.clone(),
328                model_hash: String::new(),
329                layout: if model_id.contains("soa") {
330                    "soa".into()
331                } else if model_id.contains("f16") {
332                    "f16".into()
333                } else {
334                    "verbatim".into()
335                },
336                mode: trial.config.mode.as_str().into(),
337                profile: "lab-auto".into(),
338                toggles_json: trial.config.toggles_json(),
339                qualia_decode_tok_s: trial.tok_s,
340                ollama_decode_tok_s: ollama_tok_s,
341                a_gap: None,
342                prefill_tok_s: None,
343                phase_ns_json: "{}".into(),
344                n_ulp_max: if micro.cuda_ok {
345                    Some(micro.max_ulp)
346                } else {
347                    None
348                },
349                c_score: None,
350                notes: format!("gen={generation}"),
351            };
352            run.compute_a_gap();
353            let _ = append_run_csv(&csv_path, &run);
354
355            let improve = match (&best, trial.tok_s) {
356                (None, Some(t)) if t > 0.0 => true,
357                (Some(b), Some(t)) => t > b.tok_s.unwrap_or(0.0) * (1.0 + 1e-6),
358                _ => false,
359            };
360            if improve {
361                logln!(
362                    "NEW_BEST tok_s={:?} label={}",
363                    trial.tok_s,
364                    trial.config.label
365                );
366                best = Some(trial.clone());
367            }
368            all_trials.push(trial);
369        }
370
371        // Self-improvement step: re-sample elites + neighbors
372        let mut ranked: Vec<_> = all_trials
373            .iter()
374            .filter(|t| t.tok_s.unwrap_or(0.0) > 0.0)
375            .cloned()
376            .collect();
377        ranked.sort_by(|a, b| {
378            b.tok_s
379                .unwrap_or(0.0)
380                .partial_cmp(&a.tok_s.unwrap_or(0.0))
381                .unwrap_or(std::cmp::Ordering::Equal)
382        });
383
384        let gen_best = best.as_ref().and_then(|b| b.tok_s).unwrap_or(0.0);
385        if gen_best > gen_best_before * (1.0 + cfg.plateau_rel) {
386            gens_without_improve = 0;
387        } else {
388            gens_without_improve += 1;
389        }
390        logln!(
391            "generation_end best={:.4} plateau_gens={gens_without_improve}",
392            gen_best
393        );
394        if gens_without_improve >= cfg.plateau_gens && generation >= 2 {
395            logln!("plateau_stop");
396            break;
397        }
398
399        // Next queue: re-measure top elites (variance) + flip one bit neighbors of best
400        queue.clear();
401        for elite in ranked.iter().take(cfg.elite_resample) {
402            queue.push(elite.config.clone());
403        }
404        if let Some(b) = best.as_ref() {
405            for neighbor in neighbors(&b.config) {
406                if !queue.iter().any(|c| c == &neighbor) {
407                    queue.push(neighbor);
408                }
409            }
410        }
411        if queue.is_empty() {
412            break;
413        }
414    }
415
416    // Restore safe defaults
417    set_resident_decode(true);
418    set_coop_gemv(true);
419    set_ffn_fusion(true);
420    set_kv_int8(true);
421    set_inference_mode(InferenceMode::Portable);
422
423    let a_gap = match (best.as_ref().and_then(|b| b.tok_s), ollama_tok_s) {
424        (Some(q), Some(o)) if q > 0.0 => Some(o / q),
425        _ => None,
426    };
427
428    let methodology =
429        build_methodology(&best, ollama_tok_s, a_gap, &audit, &all_trials, generation);
430    let _ = std::fs::write(cfg.out_dir.join("METHODOLOGY.md"), &methodology);
431
432    if let Some(ref b) = best {
433        let _ = std::fs::write(
434            cfg.out_dir.join("BEST_CONFIG.json"),
435            format!(
436                "{{\n  \"label\": \"{}\",\n  \"tok_s\": {},\n  \"toggles\": {},\n  \"mode\": \"{}\",\n  \"backend\": \"{}\",\n  \"resident\": {},\n  \"coop\": {},\n  \"kv_int8\": {}\n}}\n",
437                b.config.label,
438                b.tok_s.unwrap_or(0.0),
439                b.config.toggles_json(),
440                b.config.mode.as_str(),
441                b.config.backend.unwrap_or("auto"),
442                b.config.resident,
443                b.config.coop,
444                b.config.kv_int8
445            ),
446        );
447        let backend_line = b
448            .config
449            .backend
450            .map(|be| format!("$env:QUALIA_WGPU_BACKEND='{be}'\n"))
451            .unwrap_or_default();
452        let apply = format!(
453            "# Auto-generated by lab auto-improve — apply winning config\n\
454             $env:QUALIA_INFERENCE_MODE='{}'\n\
455             {backend_line}\
456             $env:QUALIA_LLM_RESIDENT_DECODE='{}'\n\
457             $env:QUALIA_LLM_COOP_GEMV='{}'\n\
458             $env:QUALIA_LLM_KV_INT8='{}'\n\
459             # tok_s measured ≈ {:?}\n",
460            b.config.mode.as_str(),
461            if b.config.resident { "1" } else { "0" },
462            if b.config.coop { "1" } else { "0" },
463            if b.config.kv_int8 { "1" } else { "0" },
464            b.tok_s
465        );
466        let _ = std::fs::write(cfg.out_dir.join("apply-best.ps1"), apply);
467    }
468
469    let pkg = LockInPackage {
470        best,
471        ollama_tok_s,
472        a_gap,
473        trials: all_trials.len(),
474        generations: generation,
475        elapsed_secs: t_start.elapsed().as_secs_f64(),
476        out_dir: cfg.out_dir.clone(),
477        methodology,
478    };
479    let summary = format_lockin_summary(&pkg);
480    let _ = std::fs::write(cfg.out_dir.join("LOCKIN_SUMMARY.txt"), &summary);
481    logln!("done\n{summary}");
482    Ok(pkg)
483}
484
485fn neighbors(c: &LabConfig) -> Vec<LabConfig> {
486    let mut out = Vec::new();
487    let mut flip = |mut x: LabConfig, label_sfx: &str| {
488        x.label = format!("{}_{label_sfx}", c.label);
489        out.push(x);
490    };
491    let mut a = c.clone();
492    a.coop = !a.coop;
493    flip(a, "flip_coop");
494    let mut b = c.clone();
495    b.kv_int8 = !b.kv_int8;
496    flip(b, "flip_kv");
497    let mut d = c.clone();
498    d.resident = !d.resident;
499    flip(d, "flip_res");
500    if c.mode == InferenceMode::Portable {
501        let mut e = c.clone();
502        e.mode = InferenceMode::FastVerify;
503        flip(e, "to_fast_verify");
504    }
505    out
506}
507
508fn build_methodology(
509    best: &Option<TrialResult>,
510    ollama: Option<f64>,
511    a_gap: Option<f64>,
512    audit: &crate::lab::audit_path::HotPathAudit,
513    trials: &[TrialResult],
514    gens: u32,
515) -> String {
516    let mut s = String::from("# Lab auto-improve methodology lock-in\n\n");
517    s.push_str(
518        "Generated by `qualia-cli llm lab auto` — **evidence-based**, not LLM kernel rewrite.\n\n",
519    );
520    s.push_str("## Best config\n\n");
521    if let Some(b) = best {
522        s.push_str(&format!(
523            "- label: `{}`\n- tok_s: {:?}\n- toggles: `{}`\n\n",
524            b.config.label,
525            b.tok_s,
526            b.config.toggles_json()
527        ));
528    } else {
529        s.push_str("- **none** (all trials failed)\n\n");
530    }
531    s.push_str("## Ollama yardstick\n\n");
532    s.push_str(&format!(
533        "- ollama_tok_s: {:?}\n- a_gap: {:?}\n\n",
534        ollama, a_gap
535    ));
536    s.push_str("## Audit notes (unfinished integration)\n\n");
537    for n in &audit.notes {
538        s.push_str(&format!("- {n}\n"));
539    }
540    s.push_str(&format!(
541        "\n## Search stats\n\n- trials: {}\n- generations: {gens}\n- ffn_fusion_in_resident: {}\n\n",
542        trials.len(),
543        audit.ffn_fusion_in_resident_decode
544    ));
545    s.push_str("## Next engineering (not search)\n\n");
546    s.push_str("1. **T-A1** (done wiring) — re-optimise fused_ffn body (coop/shared act) if A-gap still open.\n");
547    s.push_str("2. **T-A2** CUDA hidden-on-device full layer stack.\n");
548    s.push_str("3. Re-run `lab auto` after each; only lock-in when A-gap phase targets improve.\n");
549    s.push_str("\n## Apply\n\n```powershell\n. .\\experiments\\inference-lab\\lockin\\apply-best.ps1\n```\n");
550    s
551}
552
553pub fn format_lockin_summary(pkg: &LockInPackage) -> String {
554    format!(
555        "LOCK-IN SUMMARY\n  out_dir:     {}\n  trials:      {}\n  generations: {}\n  elapsed_s:   {:.1}\n  best_tok_s:  {:?}\n  best_label:  {}\n  ollama_tok_s:{:?}\n  a_gap:       {:?}\n",
556        pkg.out_dir.display(),
557        pkg.trials,
558        pkg.generations,
559        pkg.elapsed_secs,
560        pkg.best.as_ref().and_then(|b| b.tok_s),
561        pkg.best
562            .as_ref()
563            .map(|b| b.config.label.as_str())
564            .unwrap_or("none"),
565        pkg.ollama_tok_s,
566        pkg.a_gap
567    )
568}
569
570#[cfg(test)]
571mod tests {
572    use super::*;
573
574    #[test]
575    fn search_space_nonempty_and_dedup_labels() {
576        let space = default_search_space();
577        assert!(
578            space.len() >= 8,
579            "expected a real search grid, got {}",
580            space.len()
581        );
582        let mut labels: Vec<&str> = space.iter().map(|c| c.label.as_str()).collect();
583        labels.sort_unstable();
584        labels.dedup();
585        assert_eq!(
586            labels.len(),
587            space.len(),
588            "duplicate labels in search space"
589        );
590    }
591
592    #[test]
593    fn neighbors_flip_bits() {
594        let c = LabConfig {
595            label: "base".into(),
596            resident: true,
597            coop: true,
598            ffn_fusion: true,
599            kv_int8: true,
600            mode: InferenceMode::Portable,
601            backend: None,
602        };
603        let n = neighbors(&c);
604        assert!(n.len() >= 3);
605        assert!(n.iter().any(|x| x.coop != c.coop));
606        assert!(n.iter().any(|x| x.kv_int8 != c.kv_int8));
607        assert!(n.iter().any(|x| x.mode == InferenceMode::FastVerify));
608    }
609
610    #[test]
611    fn toggles_json_is_compact() {
612        let c = LabConfig {
613            label: "t".into(),
614            resident: true,
615            coop: false,
616            ffn_fusion: true,
617            kv_int8: true,
618            mode: InferenceMode::FastVerify,
619            backend: Some("vulkan"),
620        };
621        let j = c.toggles_json();
622        assert!(j.contains("\"resident\":true"));
623        assert!(j.contains("\"coop\":false"));
624        assert!(j.contains("fast-verify"));
625        assert!(j.contains("vulkan"));
626    }
627
628    #[test]
629    fn lockin_summary_handles_empty_best() {
630        let pkg = LockInPackage {
631            best: None,
632            ollama_tok_s: None,
633            a_gap: None,
634            trials: 0,
635            generations: 0,
636            elapsed_secs: 0.0,
637            out_dir: PathBuf::from("experiments/inference-lab/lockin"),
638            methodology: String::new(),
639        };
640        let s = format_lockin_summary(&pkg);
641        assert!(s.contains("none"));
642        assert!(s.contains("LOCK-IN SUMMARY"));
643    }
644}
qualia_core_db/inference/lab/auto_improve.rs

qualia_core_db/inference/lab/
auto_improve.rs
