qualia_core_db/agent_runtime/
evaluator.rs1use super::corpus::{GoldenCase, GoldenCorpus};
9
10#[derive(Debug, Clone, PartialEq)]
12pub struct CaseResult {
13 pub name: String,
15 pub passed: bool,
17 pub score: f64,
19 pub output: String,
21 pub expected: String,
23}
24
25#[derive(Debug, Clone, PartialEq)]
27pub struct EvalMetrics {
28 pub total: usize,
30 pub passed: usize,
32 pub accuracy: f64,
34 pub mean_score: f64,
36 pub min_score: f64,
38 pub max_score: f64,
40}
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub enum MatchMethod {
45 Exact,
47 Substring,
49 TokenOverlap,
51}
52
53fn tokenise(s: &str) -> Vec<String> {
55 s.split(|c: char| !c.is_alphanumeric())
56 .filter(|t| !t.is_empty())
57 .map(|t| t.to_ascii_lowercase())
58 .collect()
59}
60
61fn jaccard(a: &[String], b: &[String]) -> f64 {
63 let set_a: std::collections::BTreeSet<&String> = a.iter().collect();
64 let set_b: std::collections::BTreeSet<&String> = b.iter().collect();
65 let intersection = set_a.intersection(&set_b).count();
66 let union = set_a.union(&set_b).count();
67 if union == 0 {
68 1.0 } else {
70 intersection as f64 / union as f64
71 }
72}
73
74pub fn score_case(output: &str, expected: &str, method: MatchMethod) -> f64 {
76 let out = output.trim();
77 let exp = expected.trim();
78 match method {
79 MatchMethod::Exact => {
80 if out == exp {
81 1.0
82 } else {
83 0.0
84 }
85 }
86 MatchMethod::Substring => {
87 if out.is_empty() && exp.is_empty() {
88 1.0
89 } else if exp.is_empty() {
90 1.0 } else if out.contains(exp) {
92 1.0
93 } else if out.is_empty() {
94 0.0
95 } else {
96 let exp_tokens = tokenise(exp);
98 let out_tokens = tokenise(out);
99 if exp_tokens.is_empty() {
100 0.0
101 } else {
102 jaccard(&exp_tokens, &out_tokens)
103 }
104 }
105 }
106 MatchMethod::TokenOverlap => {
107 let exp_tokens = tokenise(exp);
108 let out_tokens = tokenise(out);
109 jaccard(&exp_tokens, &out_tokens)
110 }
111 }
112}
113
114pub fn eval_case(case: &GoldenCase, output: &str, method: MatchMethod) -> CaseResult {
116 let score = score_case(output, &case.expected, method);
117 CaseResult {
118 name: case.name.clone(),
119 passed: score >= 0.5,
120 score,
121 output: output.to_string(),
122 expected: case.expected.clone(),
123 }
124}
125
126pub fn evaluate_corpus(
131 corpus: &GoldenCorpus,
132 outputs: &[String],
133 method: MatchMethod,
134) -> Vec<CaseResult> {
135 corpus
136 .cases
137 .iter()
138 .enumerate()
139 .map(|(i, case)| {
140 let output = outputs.get(i).map(|s| s.as_str()).unwrap_or("");
141 eval_case(case, output, method)
142 })
143 .collect()
144}
145
146pub fn compute_metrics(results: &[CaseResult]) -> EvalMetrics {
148 let total = results.len();
149 if total == 0 {
150 return EvalMetrics {
151 total: 0,
152 passed: 0,
153 accuracy: 0.0,
154 mean_score: 0.0,
155 min_score: 0.0,
156 max_score: 0.0,
157 };
158 }
159 let passed = results.iter().filter(|r| r.passed).count();
160 let scores: Vec<f64> = results.iter().map(|r| r.score).collect();
161 let mean_score = scores.iter().sum::<f64>() / total as f64;
162 let min_score = scores.iter().cloned().fold(f64::INFINITY, f64::min);
163 let max_score = scores.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
164 EvalMetrics {
165 total,
166 passed,
167 accuracy: passed as f64 / total as f64,
168 mean_score,
169 min_score,
170 max_score,
171 }
172}
173
174#[cfg(test)]
175mod tests {
176 use super::*;
177 use std::collections::BTreeSet;
178
179 fn make_case(name: &str, input: &str, expected: &str) -> GoldenCase {
180 GoldenCase {
181 name: name.into(),
182 input: input.into(),
183 expected: expected.into(),
184 tags: BTreeSet::new(),
185 }
186 }
187
188 #[test]
189 fn exact_match_pass() {
190 let score = score_case("hello world", "hello world", MatchMethod::Exact);
191 assert_eq!(score, 1.0);
192 }
193
194 #[test]
195 fn exact_match_fail() {
196 let score = score_case("hello", "hello world", MatchMethod::Exact);
197 assert_eq!(score, 0.0);
198 }
199
200 #[test]
201 fn substring_match() {
202 let score = score_case(
203 "the hello world example",
204 "hello world",
205 MatchMethod::Substring,
206 );
207 assert_eq!(score, 1.0);
208 }
209
210 #[test]
211 fn substring_partial() {
212 let score = score_case("hello there", "hello world", MatchMethod::Substring);
213 assert!(score > 0.0 && score < 1.0);
214 }
215
216 #[test]
217 fn token_overlap_identical() {
218 let score = score_case("hello world", "hello world", MatchMethod::TokenOverlap);
219 assert_eq!(score, 1.0);
220 }
221
222 #[test]
223 fn token_overlap_partial() {
224 let score = score_case(
225 "hello world foo",
226 "hello world bar",
227 MatchMethod::TokenOverlap,
228 );
229 assert!(score > 0.0 && score < 1.0);
230 }
231
232 #[test]
233 fn token_overlap_empty() {
234 let score = score_case("", "", MatchMethod::TokenOverlap);
235 assert_eq!(score, 1.0);
236 }
237
238 #[test]
239 fn eval_case_passes() {
240 let case = make_case("test", "input", "expected output");
241 let result = eval_case(&case, "expected output", MatchMethod::Exact);
242 assert!(result.passed);
243 assert_eq!(result.score, 1.0);
244 }
245
246 #[test]
247 fn eval_case_fails() {
248 let case = make_case("test", "input", "expected output");
249 let result = eval_case(&case, "wrong output", MatchMethod::Exact);
250 assert!(!result.passed);
251 assert_eq!(result.score, 0.0);
252 }
253
254 #[test]
255 fn evaluate_corpus_all_pass() {
256 let corpus = GoldenCorpus {
257 name: "test".into(),
258 cases: vec![make_case("a", "in", "out_a"), make_case("b", "in", "out_b")],
259 };
260 let outputs = vec!["out_a".to_string(), "out_b".to_string()];
261 let results = evaluate_corpus(&corpus, &outputs, MatchMethod::Exact);
262 assert_eq!(results.len(), 2);
263 assert!(results.iter().all(|r| r.passed));
264 }
265
266 #[test]
267 fn evaluate_corpus_mixed() {
268 let corpus = GoldenCorpus {
269 name: "test".into(),
270 cases: vec![
271 make_case("a", "in", "correct"),
272 make_case("b", "in", "correct"),
273 ],
274 };
275 let outputs = vec!["correct".to_string(), "wrong".to_string()];
276 let results = evaluate_corpus(&corpus, &outputs, MatchMethod::Exact);
277 assert_eq!(results.len(), 2);
278 assert!(results[0].passed);
279 assert!(!results[1].passed);
280 }
281
282 #[test]
283 fn evaluate_corpus_missing_output() {
284 let corpus = GoldenCorpus {
285 name: "test".into(),
286 cases: vec![make_case("a", "in", "expected")],
287 };
288 let outputs: Vec<String> = vec![];
289 let results = evaluate_corpus(&corpus, &outputs, MatchMethod::Exact);
290 assert_eq!(results.len(), 1);
291 assert!(!results[0].passed);
292 assert_eq!(results[0].score, 0.0);
293 }
294
295 #[test]
296 fn compute_metrics_all_pass() {
297 let results = vec![
298 CaseResult {
299 name: "a".into(),
300 passed: true,
301 score: 1.0,
302 output: "x".into(),
303 expected: "x".into(),
304 },
305 CaseResult {
306 name: "b".into(),
307 passed: true,
308 score: 0.8,
309 output: "y".into(),
310 expected: "y".into(),
311 },
312 ];
313 let metrics = compute_metrics(&results);
314 assert_eq!(metrics.total, 2);
315 assert_eq!(metrics.passed, 2);
316 assert_eq!(metrics.accuracy, 1.0);
317 assert_eq!(metrics.mean_score, 0.9);
318 }
319
320 #[test]
321 fn compute_metrics_mixed() {
322 let results = vec![
323 CaseResult {
324 name: "a".into(),
325 passed: true,
326 score: 1.0,
327 output: "x".into(),
328 expected: "x".into(),
329 },
330 CaseResult {
331 name: "b".into(),
332 passed: false,
333 score: 0.0,
334 output: "y".into(),
335 expected: "z".into(),
336 },
337 ];
338 let metrics = compute_metrics(&results);
339 assert_eq!(metrics.total, 2);
340 assert_eq!(metrics.passed, 1);
341 assert_eq!(metrics.accuracy, 0.5);
342 assert_eq!(metrics.mean_score, 0.5);
343 assert_eq!(metrics.min_score, 0.0);
344 assert_eq!(metrics.max_score, 1.0);
345 }
346
347 #[test]
348 fn compute_metrics_empty() {
349 let metrics = compute_metrics(&[]);
350 assert_eq!(metrics.total, 0);
351 assert_eq!(metrics.accuracy, 0.0);
352 }
353}