qualia_core_db/inference/lab/
pareto.rs

1//! 6-dimensional Pareto frontier computation for experiment scoring.
2//!
3//! Each experiment is scored on six dimensions:
4//! - Latency (lower is better)
5//! - Throughput (higher is better)
6//! - VRAM (lower is better)
7//! - Quality (higher is better)
8//! - Energy (lower is better)
9//! - Cost (lower is better)
10//!
11//! Dominated results are pruned but retained for analysis.
12
13use serde::{Deserialize, Serialize};
14
15use super::experiment::ExperimentResult;
16
17/// The six Pareto dimensions. "Lower is better" dimensions are negated
18/// internally so that "higher is always better" in the dominance comparison.
19#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
20pub struct ParetoPoint {
21    /// Latency in ms (lower is better → negated).
22    pub latency_ms: f64,
23    /// Throughput in tok/s (higher is better).
24    pub throughput_tok_s: f64,
25    /// VRAM in bytes (lower is better → negated).
26    pub vram_bytes: f64,
27    /// Quality score [0, 1] (higher is better).
28    pub quality: f64,
29    /// Energy in joules (lower is better → negated).
30    pub energy_j: f64,
31    /// Cost composite: VRAM × latency / throughput (lower is better → negated).
32    pub cost: f64,
33}
34
35impl ParetoPoint {
36    /// Extract a Pareto point from an experiment result.
37    pub fn from_result(r: &ExperimentResult) -> Option<Self> {
38        let bench = r.bench.as_ref()?;
39        if r.error.is_some() {
40            return None;
41        }
42
43        let latency_ms = bench.warm_total_ms.max(0.0);
44        let throughput_tok_s = bench.decode_tok_s.max(0.0);
45        let vram_bytes = r.vram_used as f64;
46        let quality = r.quality.composite();
47        let energy_j = r.thermal.energy_j.max(0.0);
48        let cost = if throughput_tok_s > 0.0 {
49            vram_bytes * latency_ms / throughput_tok_s
50        } else {
51            f64::INFINITY
52        };
53
54        Some(Self {
55            latency_ms,
56            throughput_tok_s,
57            vram_bytes,
58            quality,
59            energy_j,
60            cost,
61        })
62    }
63
64    /// Convert to a "higher is better" vector for dominance comparison.
65    fn to_higher_better(&self) -> [f64; 6] {
66        [
67            -self.latency_ms, // lower latency → higher negated
68            self.throughput_tok_s,
69            -self.vram_bytes, // lower VRAM → higher negated
70            self.quality,
71            -self.energy_j, // lower energy → higher negated
72            -self.cost,     // lower cost → higher negated
73        ]
74    }
75
76    /// Returns true if `self` dominates `other` (self is better or equal in all
77    /// dimensions, and strictly better in at least one).
78    pub fn dominates(&self, other: &ParetoPoint) -> bool {
79        let a = self.to_higher_better();
80        let b = other.to_higher_better();
81        let mut any_strictly_better = false;
82        for i in 0..6 {
83            if a[i] < b[i] {
84                return false;
85            }
86            if a[i] > b[i] {
87                any_strictly_better = true;
88            }
89        }
90        any_strictly_better
91    }
92}
93
94/// A Pareto frontier: the set of non-dominated points from a collection of experiments.
95#[derive(Debug, Clone, Serialize, Deserialize, Default)]
96pub struct ParetoFrontier {
97    /// Indices into the original results vector for non-dominated points.
98    pub non_dominated: Vec<usize>,
99    /// Indices of dominated points (retained for analysis).
100    pub dominated: Vec<usize>,
101}
102
103impl ParetoFrontier {
104    /// Compute the Pareto frontier from a slice of experiment results.
105    /// Returns a frontier with indices into the original slice.
106    pub fn compute(results: &[ExperimentResult]) -> Self {
107        let points: Vec<Option<ParetoPoint>> =
108            results.iter().map(ParetoPoint::from_result).collect();
109
110        let mut non_dominated = Vec::new();
111        let mut dominated = Vec::new();
112
113        for (i, pi) in points.iter().enumerate() {
114            let Some(p_i) = pi else {
115                // Failed experiments are neither dominated nor non-dominated.
116                continue;
117            };
118            let mut is_dominated = false;
119            for (j, pj) in points.iter().enumerate() {
120                if i == j {
121                    continue;
122                }
123                if let Some(p_j) = pj {
124                    if p_j.dominates(p_i) {
125                        is_dominated = true;
126                        break;
127                    }
128                }
129            }
130            if is_dominated {
131                dominated.push(i);
132            } else {
133                non_dominated.push(i);
134            }
135        }
136
137        Self {
138            non_dominated,
139            dominated,
140        }
141    }
142
143    /// Number of non-dominated points.
144    pub fn frontier_size(&self) -> usize {
145        self.non_dominated.len()
146    }
147
148    /// Get the non-dominated experiment results.
149    pub fn frontier_results<'a>(
150        &self,
151        results: &'a [ExperimentResult],
152    ) -> Vec<&'a ExperimentResult> {
153        self.non_dominated
154            .iter()
155            .filter_map(|&i| results.get(i))
156            .collect()
157    }
158
159    /// Serialize the frontier to JSON for external dashboard export.
160    pub fn to_json(&self, results: &[ExperimentResult]) -> String {
161        let frontier: Vec<&ExperimentResult> = self
162            .non_dominated
163            .iter()
164            .filter_map(|&i| results.get(i))
165            .collect();
166        serde_json::to_string_pretty(
167            &frontier
168                .iter()
169                .map(|r| {
170                    let p = ParetoPoint::from_result(r);
171                    serde_json::json!({
172                        "config_hash": r.config_hash,
173                        "hypothesis_id": r.hypothesis_id,
174                        "pareto": p,
175                        "quality": r.quality,
176                        "decode_tok_s": r.bench.as_ref().map(|b| b.decode_tok_s),
177                        "warm_total_ms": r.bench.as_ref().map(|b| b.warm_total_ms),
178                        "vram_used": r.vram_used,
179                        "energy_j": r.thermal.energy_j,
180                    })
181                })
182                .collect::<Vec<_>>(),
183        )
184        .unwrap_or_else(|e| format!("{{\"error\":\"{e}\"}}"))
185    }
186}
187
188/// Score a result against an application profile to select the "best" point
189/// from the Pareto frontier.
190#[derive(Debug, Clone, Copy, PartialEq, Eq)]
191pub enum ApplicationProfileWeight {
192    /// Optimize for latency (interactive).
193    Interactive,
194    /// Optimize for throughput (live-fast).
195    LiveFast,
196    /// Optimize for quality (batch-overnight).
197    BatchOvernight,
198}
199
200impl ApplicationProfileWeight {
201    /// Weight vector for scoring: (latency, throughput, vram, quality, energy, cost).
202    fn weights(&self) -> [f64; 6] {
203        match self {
204            Self::Interactive => [0.35, 0.20, 0.10, 0.15, 0.10, 0.10],
205            Self::LiveFast => [0.15, 0.40, 0.10, 0.10, 0.10, 0.15],
206            Self::BatchOvernight => [0.10, 0.10, 0.10, 0.50, 0.10, 0.10],
207        }
208    }
209
210    /// Score a Pareto point: higher is better. Normalizes each dimension to [0, 1]
211    /// relative to the frontier min/max, then applies the weight vector.
212    pub fn score(&self, point: &ParetoPoint, frontier: &[ParetoPoint]) -> f64 {
213        if frontier.is_empty() {
214            return 0.0;
215        }
216        let w = self.weights();
217        let hb = point.to_higher_better();
218
219        // Normalize each dimension relative to frontier min/max.
220        let mut score = 0.0;
221        for dim in 0..6 {
222            let vals: Vec<f64> = frontier.iter().map(|p| p.to_higher_better()[dim]).collect();
223            let lo = vals.iter().cloned().fold(f64::INFINITY, f64::min);
224            let hi = vals.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
225            let normalized = if (hi - lo).abs() < 1e-12 {
226                0.5
227            } else {
228                ((hb[dim] - lo) / (hi - lo)).clamp(0.0, 1.0)
229            };
230            score += w[dim] * normalized;
231        }
232        score
233    }
234
235    /// Select the best point from the frontier for this profile.
236    pub fn select_best<'a>(
237        &self,
238        results: &'a [ExperimentResult],
239        frontier: &ParetoFrontier,
240    ) -> Option<&'a ExperimentResult> {
241        let frontier_points: Vec<ParetoPoint> = frontier
242            .non_dominated
243            .iter()
244            .filter_map(|&i| results.get(i).and_then(ParetoPoint::from_result))
245            .collect();
246
247        let mut best_idx: Option<usize> = None;
248        let mut best_score = f64::NEG_INFINITY;
249        for (fi, &ri) in frontier.non_dominated.iter().enumerate() {
250            if let Some(ref p) = frontier_points.get(fi) {
251                let s = self.score(p, &frontier_points);
252                if s > best_score {
253                    best_score = s;
254                    best_idx = Some(ri);
255                }
256            }
257        }
258        best_idx.and_then(|i| results.get(i))
259    }
260}
261
262#[cfg(test)]
263mod tests {
264    use super::*;
265    use crate::inference::lab::experiment::{
266        BenchResultSerde, ExperimentResult, PhaseSnapshotSerde, QualityScore, ThermalSnapshot,
267    };
268
269    fn make_result(
270        warm_total_ms: f64,
271        decode_tok_s: f64,
272        vram: u64,
273        quality: f64,
274        energy: f64,
275    ) -> ExperimentResult {
276        ExperimentResult {
277            config_hash: rand::random(),
278            hypothesis_id: None,
279            bench: Some(BenchResultSerde {
280                warm_total_ms,
281                decode_tok_s,
282                ..Default::default()
283            }),
284            phase: PhaseSnapshotSerde::default(),
285            quality: QualityScore {
286                pass_rate: quality,
287                total_checks: 1,
288                passed: if quality >= 1.0 { 1 } else { 0 },
289                repaired: false,
290                text_len: 0,
291            },
292            thermal: ThermalSnapshot {
293                energy_j: energy,
294                ..Default::default()
295            },
296            vram_used: vram,
297            config_cbor: vec![],
298            timestamp_ms: 0,
299            seed: 0,
300            error: None,
301        }
302    }
303
304    #[test]
305    fn pareto_dominance_basic() {
306        let a = ParetoPoint {
307            latency_ms: 100.0,
308            throughput_tok_s: 50.0,
309            vram_bytes: 1e9,
310            quality: 0.95,
311            energy_j: 10.0,
312            cost: 2e9,
313        };
314        let b = ParetoPoint {
315            latency_ms: 200.0,
316            throughput_tok_s: 30.0,
317            vram_bytes: 2e9,
318            quality: 0.80,
319            energy_j: 20.0,
320            cost: 1.3e10,
321        };
322        // a dominates b in all dimensions.
323        assert!(a.dominates(&b));
324        assert!(!b.dominates(&a));
325    }
326
327    #[test]
328    fn pareto_frontier_computes() {
329        let results = vec![
330            make_result(100.0, 50.0, 1_000_000_000, 0.95, 10.0), // best latency + throughput
331            make_result(200.0, 30.0, 2_000_000_000, 0.80, 20.0), // dominated by 0
332            make_result(150.0, 40.0, 500_000_000, 0.99, 15.0),   // best quality + VRAM
333        ];
334        let frontier = ParetoFrontier::compute(&results);
335        assert_eq!(frontier.frontier_size(), 2);
336        assert!(frontier.non_dominated.contains(&0));
337        assert!(frontier.non_dominated.contains(&2));
338        assert!(frontier.dominated.contains(&1));
339    }
340
341    #[test]
342    fn profile_selects_best() {
343        let results = vec![
344            make_result(100.0, 50.0, 1_000_000_000, 0.90, 10.0),
345            make_result(200.0, 30.0, 500_000_000, 0.99, 20.0),
346        ];
347        let frontier = ParetoFrontier::compute(&results);
348        assert_eq!(frontier.frontier_size(), 2);
349
350        // Interactive should prefer the lower latency result.
351        let best = ApplicationProfileWeight::Interactive.select_best(&results, &frontier);
352        assert!(best.is_some());
353        let best = best.unwrap();
354        assert!(best.bench.as_ref().unwrap().warm_total_ms <= 150.0);
355
356        // BatchOvernight should prefer the higher quality result.
357        let best = ApplicationProfileWeight::BatchOvernight.select_best(&results, &frontier);
358        assert!(best.is_some());
359        let best = best.unwrap();
360        assert!(best.quality.pass_rate >= 0.95);
361    }
362
363    #[test]
364    fn frontier_json_exports() {
365        let results = vec![make_result(100.0, 50.0, 1_000_000_000, 0.95, 10.0)];
366        let frontier = ParetoFrontier::compute(&results);
367        let json = frontier.to_json(&results);
368        assert!(json.contains("config_hash"));
369        assert!(json.contains("pareto"));
370    }
371}
qualia_core_db/inference/lab/pareto.rs

qualia_core_db/inference/lab/
pareto.rs
