Skip to main content

qualia_core_db/inference/lab/
experiment.rs

1//! Experiment execution engine — wraps the existing `run_bench` harness with
2//! quality verification, thermal sampling, and structured result collection.
3//!
4//! Each experiment produces an `ExperimentResult` containing:
5//! - `BenchResult` (latency, tok/s, phase split)
6//! - `VerifiedTurn` (post-turn quality checks)
7//! - Thermal snapshot (GPU temp/power before + after)
8//! - VRAM usage from the GPU context ledger
9//! - CBOR config snapshot for reproduction
10
11use std::time::{SystemTime, UNIX_EPOCH};
12
13use crate::gpu_context::global_vram_ledger;
14use crate::inference::runtime::receipt::execution::{
15    BackendKind, ExecutionReceipt, COUNTER_COMMITTED_PREFILL_TOKENS, COUNTER_DECODE_STEPS,
16    COUNTER_SUBMITTED_PREFILL_TOKENS, COUNTER_TIME_TO_FIRST_TOKEN_US,
17};
18use crate::inference_bench::{
19    run_bench, set_attention_o_fuse, set_attention_preproject, set_coop_gemv,
20    set_decode_budget_override, set_ffn_fusion, set_gpu_topk, set_kv_dict, set_kv_int8,
21    set_kv_pool_budget_mb, set_prefill_chunk_size_override, set_prefix_cache_enabled,
22    set_prompt_precision_mode, set_resident_decode, set_resident_prefill, set_resident_weights,
23    set_spec_decode, set_ternary_ffn,
24};
25use crate::post_turn_verify::verify_and_heal_turn;
26use crate::thermal_telemetry::sample_gpu_thermal;
27
28use super::experiment_adapter::*;
29
30pub use etalon_inference_lab::experiment::*;
31pub trait ExperimentConfigExt {
32    fn apply_toggles(&self);
33}
34impl ExperimentConfigExt for ExperimentConfig {
35    /// Apply the configuration's toggle values to the runtime.
36    /// This reads known parameter names from the config and sets the corresponding toggles.
37    fn apply_toggles(&self) {
38        if let Some(v) = self.config.get_bool("gpu_topk") {
39            set_gpu_topk(v);
40        }
41        if let Some(v) = self.config.get_bool("ternary_ffn") {
42            set_ternary_ffn(v);
43        }
44        if let Some(v) = self.config.get_bool("kv_dict") {
45            set_kv_dict(v);
46        }
47        if let Some(v) = self.config.get_bool("attention_preproject") {
48            set_attention_preproject(v);
49        }
50        if let Some(v) = self.config.get_bool("attention_o_fuse") {
51            set_attention_o_fuse(v);
52        }
53        if let Some(v) = self.config.get_bool("spec_decode") {
54            set_spec_decode(v);
55        }
56        if let Some(v) = self.config.get_bool("ffn_fusion") {
57            set_ffn_fusion(v);
58        }
59        if let Some(v) = self.config.get_bool("coop_gemv") {
60            set_coop_gemv(v);
61        }
62        if let Some(v) = self.config.get_bool("kv_int8") {
63            set_kv_int8(v);
64        }
65        if let Some(v) = self.config.get_bool("resident_decode") {
66            set_resident_decode(v);
67        }
68        if let Some(v) = self.config.get_bool("resident_prefill") {
69            set_resident_prefill(v);
70        }
71        if let Some(v) = self.config.get_bool("resident_weights") {
72            set_resident_weights(v);
73        }
74        if let Some(v) = self.config.get_bool("prefix_cache_enabled") {
75            set_prefix_cache_enabled(v);
76        }
77        if let Some(v) = self.config.get_int("kv_pool_budget_mb") {
78            set_kv_pool_budget_mb(v as u32);
79        }
80        if let Some(v) = self.config.get_int("prefill_chunk_size") {
81            set_prefill_chunk_size_override(v as u32);
82        }
83        if let Some(s) = self.config.get_string("prompt_precision_mode") {
84            set_prompt_precision_mode(s);
85        }
86    }
87}
88
89/// Run one experiment end-to-end: apply toggles, run benchmark, verify quality,
90/// sample thermal, collect results.
91pub fn run_experiment(cfg: &ExperimentConfig) -> ExperimentResult {
92    let _state = super::state::TrialState::capture();
93    let config_hash = cfg.config.hash();
94    let config_cbor = cfg.config.to_cbor();
95    let timestamp_ms = SystemTime::now()
96        .duration_since(UNIX_EPOCH)
97        .map(|d| d.as_millis() as u64)
98        .unwrap_or(0);
99
100    // Apply toggle configuration
101    cfg.apply_toggles();
102    set_decode_budget_override(cfg.decode_tokens);
103
104    // Thermal sample before
105    let thermal_before = sample_gpu_thermal();
106    let t_start = std::time::Instant::now();
107
108    // Run the benchmark
109    let bench_result = run_bench(&cfg.to_bench_config());
110
111    let elapsed = t_start.elapsed().as_secs_f64();
112
113    // Thermal sample after
114    let thermal_after = sample_gpu_thermal();
115    let thermal = capture_thermal(thermal_before, thermal_after, elapsed);
116    let energy_state = if thermal_before.is_some() && thermal_after.is_some() {
117        etalon_inference_lab::runner::MetricState::Estimated(thermal.energy_j)
118    } else {
119        etalon_inference_lab::runner::MetricState::Unavailable
120    };
121
122    // Collect phase metrics
123    let phase = crate::inference_bench::phase_snapshot();
124    let phase_serde = phase_serde(phase);
125
126    // VRAM usage from the global VRAM ledger
127    let vram_used = global_vram_ledger().used_bytes();
128
129    // Quality check: run post-turn verify on the output
130    let quality = match &bench_result {
131        Ok(br) => {
132            // We don't have the actual generated text from run_bench (it only returns metrics),
133            // so we use a quality proxy from the bench result itself.
134            // In a full implementation, we'd run decode_with_metrics to get the text.
135            // For now, we score based on whether the benchmark succeeded and produced tokens.
136            let has_output = br.output_tokens > 0;
137            QualityScore {
138                pass_rate: if has_output { 1.0 } else { 0.0 },
139                total_checks: 1,
140                passed: if has_output { 1 } else { 0 },
141                repaired: false,
142                text_len: 0,
143            }
144        }
145        Err(_) => QualityScore {
146            pass_rate: 0.0,
147            total_checks: 1,
148            passed: 0,
149            repaired: false,
150            text_len: 0,
151        },
152    };
153
154    // Restore defaults
155    set_decode_budget_override(0);
156
157    let (bench, error) = match bench_result {
158        Ok(r) => (Some(bench_serde(&r)), None),
159        Err(e) => (None, Some(e)),
160    };
161
162    let receipt = if let Some(ref br) = bench {
163        let mut rc = ExecutionReceipt::new(BackendKind::Unknown, BackendKind::Unknown, "", "");
164        rc.counters.decode_steps = br.output_tokens;
165        rc.mark_measured(COUNTER_DECODE_STEPS);
166        rc.counters.submitted_prefill_tokens = br.prompt_tokens;
167        rc.mark_measured(COUNTER_SUBMITTED_PREFILL_TOKENS);
168        rc.counters.committed_prefill_tokens = br.prompt_tokens;
169        rc.mark_measured(COUNTER_COMMITTED_PREFILL_TOKENS);
170        rc.counters.pool_high_water_bytes = vram_used;
171        // A ledger snapshot is not a measured high-water mark.
172
173        let ttft_us = (br.cold_ttft_ms * 1000.0) as u64;
174        let itl_us = if br.output_tokens > 1 {
175            ((br.decode_ms / (br.output_tokens - 1) as f64) * 1000.0) as u64
176        } else {
177            0
178        };
179        rc.latency.time_to_first_token_us = ttft_us;
180        rc.mark_measured(COUNTER_TIME_TO_FIRST_TOKEN_US);
181        rc.latency.inter_token_latency_us = itl_us;
182        // Inter-token latency is derived from aggregate decode duration.
183        Some(rc)
184    } else {
185        None
186    };
187
188    ExperimentResult {
189        config_hash,
190        hypothesis_id: cfg.hypothesis_id.clone(),
191        evaluation_mode: cfg.evaluation_mode,
192        receipt,
193        bench,
194        phase: phase_serde,
195        quality,
196        thermal,
197        vram_used,
198        config_cbor,
199        timestamp_ms,
200        seed: cfg.seed,
201        error,
202        evidence: TrialEvidence {
203            schema_version: 1,
204            requested_token_budget: Some(cfg.decode_tokens),
205            actual_token_budget: Some(cfg.decode_tokens),
206            quality_is_token_proxy: true,
207            energy_j: energy_state,
208            ..Default::default()
209        },
210    }
211}
212
213/// Run an experiment with quality verification (uses `decode_with_metrics` to get
214/// the actual generated text, then runs `verify_and_heal_turn` on it).
215pub fn run_experiment_with_quality(cfg: &ExperimentConfig) -> ExperimentResult {
216    let _state = super::state::TrialState::capture();
217    use crate::inference_bench::{decode_with_metrics, reset_phase_metrics};
218
219    let config_hash = cfg.config.hash();
220    let config_cbor = cfg.config.to_cbor();
221    let timestamp_ms = SystemTime::now()
222        .duration_since(UNIX_EPOCH)
223        .map(|d| d.as_millis() as u64)
224        .unwrap_or(0);
225
226    cfg.apply_toggles();
227
228    let thermal_before = sample_gpu_thermal();
229    let t_start = std::time::Instant::now();
230
231    reset_phase_metrics();
232
233    let decode_tokens = cfg.decode_tokens.max(8).min(128);
234    let result = decode_with_metrics(&cfg.model_path, &cfg.prompt, decode_tokens);
235
236    let elapsed = t_start.elapsed().as_secs_f64();
237    let thermal_after = sample_gpu_thermal();
238    let thermal = capture_thermal(thermal_before, thermal_after, elapsed);
239    let energy_state = if thermal_before.is_some() && thermal_after.is_some() {
240        etalon_inference_lab::runner::MetricState::Estimated(thermal.energy_j)
241    } else {
242        etalon_inference_lab::runner::MetricState::Unavailable
243    };
244
245    let phase = crate::inference_bench::phase_snapshot();
246    let phase_serde = phase_serde(phase);
247
248    let vram_used = global_vram_ledger().used_bytes();
249
250    let mut evidence = TrialEvidence {
251        energy_j: energy_state,
252        schema_version: 1,
253        requested_token_budget: Some(cfg.decode_tokens),
254        actual_token_budget: Some(decode_tokens),
255        ..Default::default()
256    };
257    let (bench, quality, error) = match result {
258        Ok((text, tok_s)) => {
259            // Run post-turn verification on the generated text
260            let repair_start = std::time::Instant::now();
261            let verified = verify_and_heal_turn(&cfg.prompt, &text);
262            let qs = quality_from_verified(&verified);
263            evidence.raw_output = Some(text);
264            evidence.repair_duration_ms = etalon_inference_lab::runner::MetricState::Measured(
265                repair_start.elapsed().as_secs_f64() * 1000.0,
266            );
267            if verified.repaired {
268                evidence.repaired_output = Some(verified.final_text.clone());
269                evidence.repaired_quality = Some(qs.clone());
270            } else {
271                evidence.raw_quality = Some(qs.clone());
272            }
273
274            // Build a serializable bench result from the decode metrics
275            let bench = BenchResultSerde {
276                label: format!("{}_h{:016x}", cfg.space.name, config_hash),
277                quantization: cfg.quantization.clone(),
278                prompt_tokens: phase_serde.prefill_tokens,
279                output_tokens: phase_serde.decode_tokens,
280                cold_ttft_ms: 0.0,
281                cold_total_ms: 0.0,
282                warm_ttft_ms: 0.0,
283                warm_total_ms: elapsed * 1000.0,
284                load_ms: phase_serde.load_ns as f64 / 1_000_000.0,
285                prefill_ms: phase_serde.prefill_ns as f64 / 1_000_000.0,
286                prefill_tok_s: if phase_serde.prefill_ns > 0 && phase_serde.prefill_tokens > 0 {
287                    phase_serde.prefill_tokens as f64 / (phase_serde.prefill_ns as f64 / 1e9)
288                } else {
289                    0.0
290                },
291                decode_ms: phase_serde.decode_ns as f64 / 1_000_000.0,
292                decode_tok_s: tok_s,
293                mapped_bytes: 0,
294                kv_cache_bytes: 0,
295            };
296            (Some(bench), qs, None)
297        }
298        Err(e) => (None, QualityScore::default(), Some(e)),
299    };
300
301    set_decode_budget_override(0);
302
303    let receipt = if let Some(ref br) = bench {
304        let mut rc = ExecutionReceipt::new(BackendKind::Unknown, BackendKind::Unknown, "", "");
305        rc.counters.decode_steps = br.output_tokens;
306        rc.mark_measured(COUNTER_DECODE_STEPS);
307        rc.counters.submitted_prefill_tokens = br.prompt_tokens;
308        rc.mark_measured(COUNTER_SUBMITTED_PREFILL_TOKENS);
309        rc.counters.committed_prefill_tokens = br.prompt_tokens;
310        rc.mark_measured(COUNTER_COMMITTED_PREFILL_TOKENS);
311        rc.counters.pool_high_water_bytes = vram_used;
312        // A ledger snapshot is not a measured high-water mark.
313
314        let ttft_us = (br.warm_total_ms * 1000.0) as u64;
315        let itl_us = if br.output_tokens > 1 {
316            ((br.decode_ms / (br.output_tokens - 1) as f64) * 1000.0) as u64
317        } else {
318            0
319        };
320        rc.latency.time_to_first_token_us = ttft_us;
321        // Total elapsed time is retained as a legacy proxy, not TTFT evidence.
322        rc.latency.inter_token_latency_us = itl_us;
323        // Inter-token latency is derived from aggregate decode duration.
324        Some(rc)
325    } else {
326        None
327    };
328
329    ExperimentResult {
330        config_hash,
331        hypothesis_id: cfg.hypothesis_id.clone(),
332        evaluation_mode: cfg.evaluation_mode,
333        receipt,
334        bench,
335        phase: phase_serde,
336        quality,
337        thermal,
338        vram_used,
339        config_cbor,
340        timestamp_ms,
341        seed: cfg.seed,
342        error,
343        evidence,
344    }
345}
346
347pub struct QualiaExecutor;
348impl etalon_inference_lab::runner::ExperimentExecutor for QualiaExecutor {
349    fn execute(&mut self, cfg: &ExperimentConfig, with_quality: bool) -> ExperimentResult {
350        if with_quality {
351            run_experiment_with_quality(cfg)
352        } else {
353            run_experiment(cfg)
354        }
355    }
356}
357
358#[cfg(test)]
359#[path = "experiment_tests.rs"]
360mod tests;