Skip to main content

qualia_core_db/agent_runtime/
evaluator.rs

1//! Agent evaluation against a golden corpus.
2//!
3//! Compares agent output to expected results from a golden corpus and
4//! computes evaluation metrics: accuracy, precision, recall, F1, and
5//! per-case pass/fail. The comparison is deterministic and string-based
6//! (exact match, substring match, or token overlap).
7
8use super::corpus::{GoldenCase, GoldenCorpus};
9
10/// Result of evaluating a single case.
11#[derive(Debug, Clone, PartialEq)]
12pub struct CaseResult {
13    /// Case name.
14    pub name: String,
15    /// Whether the agent output matched the expected output.
16    pub passed: bool,
17    /// Match score in [0.0, 1.0].
18    pub score: f64,
19    /// Agent output text.
20    pub output: String,
21    /// Expected text.
22    pub expected: String,
23}
24
25/// Aggregate evaluation metrics.
26#[derive(Debug, Clone, PartialEq)]
27pub struct EvalMetrics {
28    /// Total cases evaluated.
29    pub total: usize,
30    /// Cases that passed (score >= threshold).
31    pub passed: usize,
32    /// Pass rate in [0.0, 1.0].
33    pub accuracy: f64,
34    /// Mean score across all cases.
35    pub mean_score: f64,
36    /// Minimum score.
37    pub min_score: f64,
38    /// Maximum score.
39    pub max_score: f64,
40}
41
42/// Comparison method for matching agent output to expected.
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub enum MatchMethod {
45    /// Exact string equality (after trimming).
46    Exact,
47    /// Agent output contains expected as a substring.
48    Substring,
49    /// Token overlap ratio (Jaccard similarity).
50    TokenOverlap,
51}
52
53/// Tokenise a string into lowercase word tokens.
54fn tokenise(s: &str) -> Vec<String> {
55    s.split(|c: char| !c.is_alphanumeric())
56        .filter(|t| !t.is_empty())
57        .map(|t| t.to_ascii_lowercase())
58        .collect()
59}
60
61/// Jaccard similarity between token sets.
62fn jaccard(a: &[String], b: &[String]) -> f64 {
63    let set_a: std::collections::BTreeSet<&String> = a.iter().collect();
64    let set_b: std::collections::BTreeSet<&String> = b.iter().collect();
65    let intersection = set_a.intersection(&set_b).count();
66    let union = set_a.union(&set_b).count();
67    if union == 0 {
68        1.0 // Both empty = perfect match
69    } else {
70        intersection as f64 / union as f64
71    }
72}
73
74/// Score a single case's output against expected, returning a [0.0, 1.0] score.
75pub fn score_case(output: &str, expected: &str, method: MatchMethod) -> f64 {
76    let out = output.trim();
77    let exp = expected.trim();
78    match method {
79        MatchMethod::Exact => {
80            if out == exp {
81                1.0
82            } else {
83                0.0
84            }
85        }
86        MatchMethod::Substring => {
87            if out.is_empty() && exp.is_empty() {
88                1.0
89            } else if exp.is_empty() {
90                1.0 // Empty expected = always pass
91            } else if out.contains(exp) {
92                1.0
93            } else if out.is_empty() {
94                0.0
95            } else {
96                // Partial credit: what fraction of expected is in output?
97                let exp_tokens = tokenise(exp);
98                let out_tokens = tokenise(out);
99                if exp_tokens.is_empty() {
100                    0.0
101                } else {
102                    jaccard(&exp_tokens, &out_tokens)
103                }
104            }
105        }
106        MatchMethod::TokenOverlap => {
107            let exp_tokens = tokenise(exp);
108            let out_tokens = tokenise(out);
109            jaccard(&exp_tokens, &out_tokens)
110        }
111    }
112}
113
114/// Evaluate a single case.
115pub fn eval_case(case: &GoldenCase, output: &str, method: MatchMethod) -> CaseResult {
116    let score = score_case(output, &case.expected, method);
117    CaseResult {
118        name: case.name.clone(),
119        passed: score >= 0.5,
120        score,
121        output: output.to_string(),
122        expected: case.expected.clone(),
123    }
124}
125
126/// Evaluate an agent's outputs against a golden corpus.
127///
128/// `outputs` must be in the same order as `corpus.cases`. If lengths differ,
129/// missing entries are scored 0.0 and extra entries are ignored.
130pub fn evaluate_corpus(
131    corpus: &GoldenCorpus,
132    outputs: &[String],
133    method: MatchMethod,
134) -> Vec<CaseResult> {
135    corpus
136        .cases
137        .iter()
138        .enumerate()
139        .map(|(i, case)| {
140            let output = outputs.get(i).map(|s| s.as_str()).unwrap_or("");
141            eval_case(case, output, method)
142        })
143        .collect()
144}
145
146/// Compute aggregate metrics from a list of case results.
147pub fn compute_metrics(results: &[CaseResult]) -> EvalMetrics {
148    let total = results.len();
149    if total == 0 {
150        return EvalMetrics {
151            total: 0,
152            passed: 0,
153            accuracy: 0.0,
154            mean_score: 0.0,
155            min_score: 0.0,
156            max_score: 0.0,
157        };
158    }
159    let passed = results.iter().filter(|r| r.passed).count();
160    let scores: Vec<f64> = results.iter().map(|r| r.score).collect();
161    let mean_score = scores.iter().sum::<f64>() / total as f64;
162    let min_score = scores.iter().cloned().fold(f64::INFINITY, f64::min);
163    let max_score = scores.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
164    EvalMetrics {
165        total,
166        passed,
167        accuracy: passed as f64 / total as f64,
168        mean_score,
169        min_score,
170        max_score,
171    }
172}
173
174#[cfg(test)]
175mod tests {
176    use super::*;
177    use std::collections::BTreeSet;
178
179    fn make_case(name: &str, input: &str, expected: &str) -> GoldenCase {
180        GoldenCase {
181            name: name.into(),
182            input: input.into(),
183            expected: expected.into(),
184            tags: BTreeSet::new(),
185        }
186    }
187
188    #[test]
189    fn exact_match_pass() {
190        let score = score_case("hello world", "hello world", MatchMethod::Exact);
191        assert_eq!(score, 1.0);
192    }
193
194    #[test]
195    fn exact_match_fail() {
196        let score = score_case("hello", "hello world", MatchMethod::Exact);
197        assert_eq!(score, 0.0);
198    }
199
200    #[test]
201    fn substring_match() {
202        let score = score_case(
203            "the hello world example",
204            "hello world",
205            MatchMethod::Substring,
206        );
207        assert_eq!(score, 1.0);
208    }
209
210    #[test]
211    fn substring_partial() {
212        let score = score_case("hello there", "hello world", MatchMethod::Substring);
213        assert!(score > 0.0 && score < 1.0);
214    }
215
216    #[test]
217    fn token_overlap_identical() {
218        let score = score_case("hello world", "hello world", MatchMethod::TokenOverlap);
219        assert_eq!(score, 1.0);
220    }
221
222    #[test]
223    fn token_overlap_partial() {
224        let score = score_case(
225            "hello world foo",
226            "hello world bar",
227            MatchMethod::TokenOverlap,
228        );
229        assert!(score > 0.0 && score < 1.0);
230    }
231
232    #[test]
233    fn token_overlap_empty() {
234        let score = score_case("", "", MatchMethod::TokenOverlap);
235        assert_eq!(score, 1.0);
236    }
237
238    #[test]
239    fn eval_case_passes() {
240        let case = make_case("test", "input", "expected output");
241        let result = eval_case(&case, "expected output", MatchMethod::Exact);
242        assert!(result.passed);
243        assert_eq!(result.score, 1.0);
244    }
245
246    #[test]
247    fn eval_case_fails() {
248        let case = make_case("test", "input", "expected output");
249        let result = eval_case(&case, "wrong output", MatchMethod::Exact);
250        assert!(!result.passed);
251        assert_eq!(result.score, 0.0);
252    }
253
254    #[test]
255    fn evaluate_corpus_all_pass() {
256        let corpus = GoldenCorpus {
257            name: "test".into(),
258            cases: vec![make_case("a", "in", "out_a"), make_case("b", "in", "out_b")],
259        };
260        let outputs = vec!["out_a".to_string(), "out_b".to_string()];
261        let results = evaluate_corpus(&corpus, &outputs, MatchMethod::Exact);
262        assert_eq!(results.len(), 2);
263        assert!(results.iter().all(|r| r.passed));
264    }
265
266    #[test]
267    fn evaluate_corpus_mixed() {
268        let corpus = GoldenCorpus {
269            name: "test".into(),
270            cases: vec![
271                make_case("a", "in", "correct"),
272                make_case("b", "in", "correct"),
273            ],
274        };
275        let outputs = vec!["correct".to_string(), "wrong".to_string()];
276        let results = evaluate_corpus(&corpus, &outputs, MatchMethod::Exact);
277        assert_eq!(results.len(), 2);
278        assert!(results[0].passed);
279        assert!(!results[1].passed);
280    }
281
282    #[test]
283    fn evaluate_corpus_missing_output() {
284        let corpus = GoldenCorpus {
285            name: "test".into(),
286            cases: vec![make_case("a", "in", "expected")],
287        };
288        let outputs: Vec<String> = vec![];
289        let results = evaluate_corpus(&corpus, &outputs, MatchMethod::Exact);
290        assert_eq!(results.len(), 1);
291        assert!(!results[0].passed);
292        assert_eq!(results[0].score, 0.0);
293    }
294
295    #[test]
296    fn compute_metrics_all_pass() {
297        let results = vec![
298            CaseResult {
299                name: "a".into(),
300                passed: true,
301                score: 1.0,
302                output: "x".into(),
303                expected: "x".into(),
304            },
305            CaseResult {
306                name: "b".into(),
307                passed: true,
308                score: 0.8,
309                output: "y".into(),
310                expected: "y".into(),
311            },
312        ];
313        let metrics = compute_metrics(&results);
314        assert_eq!(metrics.total, 2);
315        assert_eq!(metrics.passed, 2);
316        assert_eq!(metrics.accuracy, 1.0);
317        assert_eq!(metrics.mean_score, 0.9);
318    }
319
320    #[test]
321    fn compute_metrics_mixed() {
322        let results = vec![
323            CaseResult {
324                name: "a".into(),
325                passed: true,
326                score: 1.0,
327                output: "x".into(),
328                expected: "x".into(),
329            },
330            CaseResult {
331                name: "b".into(),
332                passed: false,
333                score: 0.0,
334                output: "y".into(),
335                expected: "z".into(),
336            },
337        ];
338        let metrics = compute_metrics(&results);
339        assert_eq!(metrics.total, 2);
340        assert_eq!(metrics.passed, 1);
341        assert_eq!(metrics.accuracy, 0.5);
342        assert_eq!(metrics.mean_score, 0.5);
343        assert_eq!(metrics.min_score, 0.0);
344        assert_eq!(metrics.max_score, 1.0);
345    }
346
347    #[test]
348    fn compute_metrics_empty() {
349        let metrics = compute_metrics(&[]);
350        assert_eq!(metrics.total, 0);
351        assert_eq!(metrics.accuracy, 0.0);
352    }
353}