Skip to main content

qualia_core_db/agent_runtime/
corpus.rs

1//! Golden corpus loader for agent evaluation.
2//!
3//! A golden corpus is a collection of test cases — each with an input prompt,
4//! expected output (or expected substrate/relations), and optional scoring
5//! criteria. The loader reads a simple text-based format (one case per
6//! stanza, separated by `---`) and produces an in-memory corpus structure.
7//!
8//! Format:
9//! ```text
10//! # case: test_name
11//! input: <input text>
12//! expected: <expected output or relations>
13//! tags: tag1,tag2
14//! ---
15//! # case: another_test
16//! input: ...
17//! expected: ...
18//! ```
19//!
20//! Lines starting with `#` are comments (or case headers when prefixed with
21//! `case:`). The `---` separator delimits cases.
22
23use std::collections::BTreeSet;
24
25/// A single golden test case.
26#[derive(Debug, Clone, PartialEq)]
27pub struct GoldenCase {
28    /// Case name (from `# case:` header).
29    pub name: String,
30    /// Input text to feed to the agent.
31    pub input: String,
32    /// Expected output or extracted relations.
33    pub expected: String,
34    /// Optional tags for filtering.
35    pub tags: BTreeSet<String>,
36}
37
38/// A loaded golden corpus.
39#[derive(Debug, Clone, PartialEq)]
40pub struct GoldenCorpus {
41    /// Corpus name (typically derived from the file/IRI).
42    pub name: String,
43    /// All test cases in the corpus.
44    pub cases: Vec<GoldenCase>,
45}
46
47impl GoldenCorpus {
48    /// Number of cases in the corpus.
49    pub fn len(&self) -> usize {
50        self.cases.len()
51    }
52
53    /// Whether the corpus is empty.
54    pub fn is_empty(&self) -> bool {
55        self.cases.is_empty()
56    }
57
58    /// Filter cases by tag.
59    pub fn cases_with_tag(&self, tag: &str) -> Vec<&GoldenCase> {
60        self.cases.iter().filter(|c| c.tags.contains(tag)).collect()
61    }
62
63    /// All unique tags across all cases.
64    pub fn all_tags(&self) -> BTreeSet<String> {
65        let mut tags = BTreeSet::new();
66        for case in &self.cases {
67            tags.extend(case.tags.iter().cloned());
68        }
69        tags
70    }
71}
72
73/// Parse a golden corpus from a text string.
74///
75/// The format is stanza-based: cases are separated by `---` on its own line.
76/// Within a stanza, `# case:` sets the name, `input:` sets the input,
77/// `expected:` sets the expected output, and `tags:` sets comma-separated tags.
78/// Multi-line values are supported by indenting continuation lines.
79pub fn parse_corpus(name: &str, text: &str) -> GoldenCorpus {
80    let mut cases = Vec::new();
81    let mut current_name = String::new();
82    let mut current_input = String::new();
83    let mut current_expected = String::new();
84    let mut current_tags = BTreeSet::new();
85    let mut current_field: Option<&str> = None;
86
87    let flush = |cases: &mut Vec<GoldenCase>,
88                 name: &mut String,
89                 input: &mut String,
90                 expected: &mut String,
91                 tags: &mut BTreeSet<String>| {
92        if !name.is_empty() || !input.is_empty() || !expected.is_empty() {
93            cases.push(GoldenCase {
94                name: if name.is_empty() {
95                    format!("case_{}", cases.len())
96                } else {
97                    name.clone()
98                },
99                input: input.trim().to_string(),
100                expected: expected.trim().to_string(),
101                tags: tags.clone(),
102            });
103            name.clear();
104            input.clear();
105            expected.clear();
106            tags.clear();
107        }
108    };
109
110    for line in text.lines() {
111        let trimmed = line.trim();
112
113        if trimmed == "---" {
114            flush(
115                &mut cases,
116                &mut current_name,
117                &mut current_input,
118                &mut current_expected,
119                &mut current_tags,
120            );
121            current_field = None;
122            continue;
123        }
124
125        if let Some(rest) = trimmed.strip_prefix("# case:") {
126            current_name = rest.trim().to_string();
127            current_field = None;
128            continue;
129        }
130
131        // Skip pure comments (but not field continuations)
132        if trimmed.starts_with('#') && current_field.is_none() {
133            continue;
134        }
135
136        if let Some(rest) = trimmed.strip_prefix("input:") {
137            current_input = rest.trim().to_string();
138            current_field = Some("input");
139            continue;
140        }
141
142        if let Some(rest) = trimmed.strip_prefix("expected:") {
143            current_expected = rest.trim().to_string();
144            current_field = Some("expected");
145            continue;
146        }
147
148        if let Some(rest) = trimmed.strip_prefix("tags:") {
149            for tag in rest.split(',') {
150                let t = tag.trim();
151                if !t.is_empty() {
152                    current_tags.insert(t.to_string());
153                }
154            }
155            current_field = None;
156            continue;
157        }
158
159        // Continuation line for the current field
160        if let Some(field) = current_field {
161            match field {
162                "input" => {
163                    current_input.push('\n');
164                    current_input.push_str(trimmed);
165                }
166                "expected" => {
167                    current_expected.push('\n');
168                    current_expected.push_str(trimmed);
169                }
170                _ => {}
171            }
172        }
173    }
174
175    // Flush the last case
176    flush(
177        &mut cases,
178        &mut current_name,
179        &mut current_input,
180        &mut current_expected,
181        &mut current_tags,
182    );
183
184    GoldenCorpus {
185        name: name.to_string(),
186        cases,
187    }
188}
189
190/// Load a golden corpus from a file path.
191#[cfg(not(target_arch = "wasm32"))]
192pub fn load_corpus_from_file(path: &str) -> Result<GoldenCorpus, String> {
193    let text = std::fs::read_to_string(path)
194        .map_err(|e| format!("failed to read corpus file {path}: {e}"))?;
195    let name = std::path::Path::new(path)
196        .file_stem()
197        .and_then(|s| s.to_str())
198        .unwrap_or("corpus")
199        .to_string();
200    Ok(parse_corpus(&name, &text))
201}
202
203#[cfg(test)]
204mod tests {
205    use super::*;
206
207    #[test]
208    fn parse_single_case() {
209        let text = "# case: test_one\ninput: hello world\nexpected: greeting\n";
210        let corpus = parse_corpus("test", text);
211        assert_eq!(corpus.len(), 1);
212        assert_eq!(corpus.cases[0].name, "test_one");
213        assert_eq!(corpus.cases[0].input, "hello world");
214        assert_eq!(corpus.cases[0].expected, "greeting");
215    }
216
217    #[test]
218    fn parse_multiple_cases() {
219        let text = "\
220# case: first
221input: a
222expected: x
223---
224# case: second
225input: b
226expected: y
227";
228        let corpus = parse_corpus("test", text);
229        assert_eq!(corpus.len(), 2);
230        assert_eq!(corpus.cases[0].name, "first");
231        assert_eq!(corpus.cases[1].name, "second");
232    }
233
234    #[test]
235    fn parse_with_tags() {
236        let text = "\
237# case: tagged
238input: data
239expected: result
240tags: nlp,coref,important
241";
242        let corpus = parse_corpus("test", text);
243        assert_eq!(corpus.len(), 1);
244        let tags = &corpus.cases[0].tags;
245        assert!(tags.contains("nlp"));
246        assert!(tags.contains("coref"));
247        assert!(tags.contains("important"));
248    }
249
250    #[test]
251    fn parse_multiline_input() {
252        let text = "\
253# case: multi
254input: line one
255  line two
256  line three
257expected: result
258";
259        let corpus = parse_corpus("test", text);
260        assert_eq!(corpus.len(), 1);
261        assert!(corpus.cases[0].input.contains("line one"));
262        assert!(corpus.cases[0].input.contains("line two"));
263        assert!(corpus.cases[0].input.contains("line three"));
264    }
265
266    #[test]
267    fn parse_empty_corpus() {
268        let corpus = parse_corpus("empty", "");
269        assert!(corpus.is_empty());
270    }
271
272    #[test]
273    fn parse_auto_named_case() {
274        let text = "input: data\nexpected: result\n";
275        let corpus = parse_corpus("test", text);
276        assert_eq!(corpus.len(), 1);
277        assert!(corpus.cases[0].name.starts_with("case_"));
278    }
279
280    #[test]
281    fn cases_with_tag_filter() {
282        let text = "\
283# case: a
284input: x
285expected: y
286tags: fast
287---
288# case: b
289input: x
290expected: y
291tags: slow
292";
293        let corpus = parse_corpus("test", text);
294        let fast = corpus.cases_with_tag("fast");
295        assert_eq!(fast.len(), 1);
296        assert_eq!(fast[0].name, "a");
297    }
298
299    #[test]
300    fn all_tags_collects_unique() {
301        let text = "\
302# case: a
303input: x
304expected: y
305tags: nlp,fast
306---
307# case: b
308input: x
309expected: y
310tags: nlp,slow
311";
312        let corpus = parse_corpus("test", text);
313        let tags = corpus.all_tags();
314        assert_eq!(tags.len(), 3);
315        assert!(tags.contains("nlp"));
316        assert!(tags.contains("fast"));
317        assert!(tags.contains("slow"));
318    }
319
320    #[test]
321    fn comments_ignored() {
322        let text = "\
323# This is a comment
324# case: real
325input: data
326# Another comment
327expected: result
328";
329        let corpus = parse_corpus("test", text);
330        assert_eq!(corpus.len(), 1);
331        assert_eq!(corpus.cases[0].name, "real");
332    }
333}