qualia_core_db/inference/lab/
experiment.rs1use std::time::{SystemTime, UNIX_EPOCH};
12
13use crate::gpu_context::global_vram_ledger;
14use crate::inference::runtime::receipt::execution::{
15 BackendKind, ExecutionReceipt, COUNTER_COMMITTED_PREFILL_TOKENS, COUNTER_DECODE_STEPS,
16 COUNTER_SUBMITTED_PREFILL_TOKENS, COUNTER_TIME_TO_FIRST_TOKEN_US,
17};
18use crate::inference_bench::{
19 run_bench, set_attention_o_fuse, set_attention_preproject, set_coop_gemv,
20 set_decode_budget_override, set_ffn_fusion, set_gpu_topk, set_kv_dict, set_kv_int8,
21 set_kv_pool_budget_mb, set_prefill_chunk_size_override, set_prefix_cache_enabled,
22 set_prompt_precision_mode, set_resident_decode, set_resident_prefill, set_resident_weights,
23 set_spec_decode, set_ternary_ffn,
24};
25use crate::post_turn_verify::verify_and_heal_turn;
26use crate::thermal_telemetry::sample_gpu_thermal;
27
28use super::experiment_adapter::*;
29
30pub use etalon_inference_lab::experiment::*;
31pub trait ExperimentConfigExt {
32 fn apply_toggles(&self);
33}
34impl ExperimentConfigExt for ExperimentConfig {
35 fn apply_toggles(&self) {
38 if let Some(v) = self.config.get_bool("gpu_topk") {
39 set_gpu_topk(v);
40 }
41 if let Some(v) = self.config.get_bool("ternary_ffn") {
42 set_ternary_ffn(v);
43 }
44 if let Some(v) = self.config.get_bool("kv_dict") {
45 set_kv_dict(v);
46 }
47 if let Some(v) = self.config.get_bool("attention_preproject") {
48 set_attention_preproject(v);
49 }
50 if let Some(v) = self.config.get_bool("attention_o_fuse") {
51 set_attention_o_fuse(v);
52 }
53 if let Some(v) = self.config.get_bool("spec_decode") {
54 set_spec_decode(v);
55 }
56 if let Some(v) = self.config.get_bool("ffn_fusion") {
57 set_ffn_fusion(v);
58 }
59 if let Some(v) = self.config.get_bool("coop_gemv") {
60 set_coop_gemv(v);
61 }
62 if let Some(v) = self.config.get_bool("kv_int8") {
63 set_kv_int8(v);
64 }
65 if let Some(v) = self.config.get_bool("resident_decode") {
66 set_resident_decode(v);
67 }
68 if let Some(v) = self.config.get_bool("resident_prefill") {
69 set_resident_prefill(v);
70 }
71 if let Some(v) = self.config.get_bool("resident_weights") {
72 set_resident_weights(v);
73 }
74 if let Some(v) = self.config.get_bool("prefix_cache_enabled") {
75 set_prefix_cache_enabled(v);
76 }
77 if let Some(v) = self.config.get_int("kv_pool_budget_mb") {
78 set_kv_pool_budget_mb(v as u32);
79 }
80 if let Some(v) = self.config.get_int("prefill_chunk_size") {
81 set_prefill_chunk_size_override(v as u32);
82 }
83 if let Some(s) = self.config.get_string("prompt_precision_mode") {
84 set_prompt_precision_mode(s);
85 }
86 }
87}
88
89pub fn run_experiment(cfg: &ExperimentConfig) -> ExperimentResult {
92 let _state = super::state::TrialState::capture();
93 let config_hash = cfg.config.hash();
94 let config_cbor = cfg.config.to_cbor();
95 let timestamp_ms = SystemTime::now()
96 .duration_since(UNIX_EPOCH)
97 .map(|d| d.as_millis() as u64)
98 .unwrap_or(0);
99
100 cfg.apply_toggles();
102 set_decode_budget_override(cfg.decode_tokens);
103
104 let thermal_before = sample_gpu_thermal();
106 let t_start = std::time::Instant::now();
107
108 let bench_result = run_bench(&cfg.to_bench_config());
110
111 let elapsed = t_start.elapsed().as_secs_f64();
112
113 let thermal_after = sample_gpu_thermal();
115 let thermal = capture_thermal(thermal_before, thermal_after, elapsed);
116 let energy_state = if thermal_before.is_some() && thermal_after.is_some() {
117 etalon_inference_lab::runner::MetricState::Estimated(thermal.energy_j)
118 } else {
119 etalon_inference_lab::runner::MetricState::Unavailable
120 };
121
122 let phase = crate::inference_bench::phase_snapshot();
124 let phase_serde = phase_serde(phase);
125
126 let vram_used = global_vram_ledger().used_bytes();
128
129 let quality = match &bench_result {
131 Ok(br) => {
132 let has_output = br.output_tokens > 0;
137 QualityScore {
138 pass_rate: if has_output { 1.0 } else { 0.0 },
139 total_checks: 1,
140 passed: if has_output { 1 } else { 0 },
141 repaired: false,
142 text_len: 0,
143 }
144 }
145 Err(_) => QualityScore {
146 pass_rate: 0.0,
147 total_checks: 1,
148 passed: 0,
149 repaired: false,
150 text_len: 0,
151 },
152 };
153
154 set_decode_budget_override(0);
156
157 let (bench, error) = match bench_result {
158 Ok(r) => (Some(bench_serde(&r)), None),
159 Err(e) => (None, Some(e)),
160 };
161
162 let receipt = if let Some(ref br) = bench {
163 let mut rc = ExecutionReceipt::new(BackendKind::Unknown, BackendKind::Unknown, "", "");
164 rc.counters.decode_steps = br.output_tokens;
165 rc.mark_measured(COUNTER_DECODE_STEPS);
166 rc.counters.submitted_prefill_tokens = br.prompt_tokens;
167 rc.mark_measured(COUNTER_SUBMITTED_PREFILL_TOKENS);
168 rc.counters.committed_prefill_tokens = br.prompt_tokens;
169 rc.mark_measured(COUNTER_COMMITTED_PREFILL_TOKENS);
170 rc.counters.pool_high_water_bytes = vram_used;
171 let ttft_us = (br.cold_ttft_ms * 1000.0) as u64;
174 let itl_us = if br.output_tokens > 1 {
175 ((br.decode_ms / (br.output_tokens - 1) as f64) * 1000.0) as u64
176 } else {
177 0
178 };
179 rc.latency.time_to_first_token_us = ttft_us;
180 rc.mark_measured(COUNTER_TIME_TO_FIRST_TOKEN_US);
181 rc.latency.inter_token_latency_us = itl_us;
182 Some(rc)
184 } else {
185 None
186 };
187
188 ExperimentResult {
189 config_hash,
190 hypothesis_id: cfg.hypothesis_id.clone(),
191 evaluation_mode: cfg.evaluation_mode,
192 receipt,
193 bench,
194 phase: phase_serde,
195 quality,
196 thermal,
197 vram_used,
198 config_cbor,
199 timestamp_ms,
200 seed: cfg.seed,
201 error,
202 evidence: TrialEvidence {
203 schema_version: 1,
204 requested_token_budget: Some(cfg.decode_tokens),
205 actual_token_budget: Some(cfg.decode_tokens),
206 quality_is_token_proxy: true,
207 energy_j: energy_state,
208 ..Default::default()
209 },
210 }
211}
212
213pub fn run_experiment_with_quality(cfg: &ExperimentConfig) -> ExperimentResult {
216 let _state = super::state::TrialState::capture();
217 use crate::inference_bench::{decode_with_metrics, reset_phase_metrics};
218
219 let config_hash = cfg.config.hash();
220 let config_cbor = cfg.config.to_cbor();
221 let timestamp_ms = SystemTime::now()
222 .duration_since(UNIX_EPOCH)
223 .map(|d| d.as_millis() as u64)
224 .unwrap_or(0);
225
226 cfg.apply_toggles();
227
228 let thermal_before = sample_gpu_thermal();
229 let t_start = std::time::Instant::now();
230
231 reset_phase_metrics();
232
233 let decode_tokens = cfg.decode_tokens.max(8).min(128);
234 let result = decode_with_metrics(&cfg.model_path, &cfg.prompt, decode_tokens);
235
236 let elapsed = t_start.elapsed().as_secs_f64();
237 let thermal_after = sample_gpu_thermal();
238 let thermal = capture_thermal(thermal_before, thermal_after, elapsed);
239 let energy_state = if thermal_before.is_some() && thermal_after.is_some() {
240 etalon_inference_lab::runner::MetricState::Estimated(thermal.energy_j)
241 } else {
242 etalon_inference_lab::runner::MetricState::Unavailable
243 };
244
245 let phase = crate::inference_bench::phase_snapshot();
246 let phase_serde = phase_serde(phase);
247
248 let vram_used = global_vram_ledger().used_bytes();
249
250 let mut evidence = TrialEvidence {
251 energy_j: energy_state,
252 schema_version: 1,
253 requested_token_budget: Some(cfg.decode_tokens),
254 actual_token_budget: Some(decode_tokens),
255 ..Default::default()
256 };
257 let (bench, quality, error) = match result {
258 Ok((text, tok_s)) => {
259 let repair_start = std::time::Instant::now();
261 let verified = verify_and_heal_turn(&cfg.prompt, &text);
262 let qs = quality_from_verified(&verified);
263 evidence.raw_output = Some(text);
264 evidence.repair_duration_ms = etalon_inference_lab::runner::MetricState::Measured(
265 repair_start.elapsed().as_secs_f64() * 1000.0,
266 );
267 if verified.repaired {
268 evidence.repaired_output = Some(verified.final_text.clone());
269 evidence.repaired_quality = Some(qs.clone());
270 } else {
271 evidence.raw_quality = Some(qs.clone());
272 }
273
274 let bench = BenchResultSerde {
276 label: format!("{}_h{:016x}", cfg.space.name, config_hash),
277 quantization: cfg.quantization.clone(),
278 prompt_tokens: phase_serde.prefill_tokens,
279 output_tokens: phase_serde.decode_tokens,
280 cold_ttft_ms: 0.0,
281 cold_total_ms: 0.0,
282 warm_ttft_ms: 0.0,
283 warm_total_ms: elapsed * 1000.0,
284 load_ms: phase_serde.load_ns as f64 / 1_000_000.0,
285 prefill_ms: phase_serde.prefill_ns as f64 / 1_000_000.0,
286 prefill_tok_s: if phase_serde.prefill_ns > 0 && phase_serde.prefill_tokens > 0 {
287 phase_serde.prefill_tokens as f64 / (phase_serde.prefill_ns as f64 / 1e9)
288 } else {
289 0.0
290 },
291 decode_ms: phase_serde.decode_ns as f64 / 1_000_000.0,
292 decode_tok_s: tok_s,
293 mapped_bytes: 0,
294 kv_cache_bytes: 0,
295 };
296 (Some(bench), qs, None)
297 }
298 Err(e) => (None, QualityScore::default(), Some(e)),
299 };
300
301 set_decode_budget_override(0);
302
303 let receipt = if let Some(ref br) = bench {
304 let mut rc = ExecutionReceipt::new(BackendKind::Unknown, BackendKind::Unknown, "", "");
305 rc.counters.decode_steps = br.output_tokens;
306 rc.mark_measured(COUNTER_DECODE_STEPS);
307 rc.counters.submitted_prefill_tokens = br.prompt_tokens;
308 rc.mark_measured(COUNTER_SUBMITTED_PREFILL_TOKENS);
309 rc.counters.committed_prefill_tokens = br.prompt_tokens;
310 rc.mark_measured(COUNTER_COMMITTED_PREFILL_TOKENS);
311 rc.counters.pool_high_water_bytes = vram_used;
312 let ttft_us = (br.warm_total_ms * 1000.0) as u64;
315 let itl_us = if br.output_tokens > 1 {
316 ((br.decode_ms / (br.output_tokens - 1) as f64) * 1000.0) as u64
317 } else {
318 0
319 };
320 rc.latency.time_to_first_token_us = ttft_us;
321 rc.latency.inter_token_latency_us = itl_us;
323 Some(rc)
325 } else {
326 None
327 };
328
329 ExperimentResult {
330 config_hash,
331 hypothesis_id: cfg.hypothesis_id.clone(),
332 evaluation_mode: cfg.evaluation_mode,
333 receipt,
334 bench,
335 phase: phase_serde,
336 quality,
337 thermal,
338 vram_used,
339 config_cbor,
340 timestamp_ms,
341 seed: cfg.seed,
342 error,
343 evidence,
344 }
345}
346
347pub struct QualiaExecutor;
348impl etalon_inference_lab::runner::ExperimentExecutor for QualiaExecutor {
349 fn execute(&mut self, cfg: &ExperimentConfig, with_quality: bool) -> ExperimentResult {
350 if with_quality {
351 run_experiment_with_quality(cfg)
352 } else {
353 run_experiment(cfg)
354 }
355 }
356}
357
358#[cfg(test)]
359#[path = "experiment_tests.rs"]
360mod tests;