Skip to main content

qualia_core_db/poet_host/invoke/agent/
runtime.rs

1//! Agent runtime invoke seams — planner, corpus, evaluator.
2//!
3//! Exposes the `agent_runtime` module's planner, corpus loader, and evaluator
4//! through VibeScript invoke IDs.
5
6use super::super::args;
7use crate::agent_runtime::{corpus, evaluator, planner};
8use vibe::{Diagnostic, Span, Value};
9
10/// `Agent.plan` — plan an agent task given a task description and capabilities.
11///
12/// Takes `task` (string) and `capabilities` (list of strings). Returns a
13/// record with `steps` (list of step records), `total_budget`, and `task`.
14pub fn agent_plan(args: &Value, span: Span) -> Result<Value, Diagnostic> {
15    let task =
16        args::rec_str(args, "task").ok_or_else(|| args::bad(span, "Agent.plan needs task"))?;
17    let capabilities: Vec<String> = match args {
18        Value::Record(map) => match map.get("capabilities") {
19            Some(Value::List(caps)) => caps
20                .iter()
21                .filter_map(|c| match c {
22                    Value::String(s) => Some(s.clone()),
23                    _ => None,
24                })
25                .collect(),
26            _ => vec![],
27        },
28        _ => vec![],
29    };
30
31    let plan = planner::plan_task(task, &capabilities);
32
33    let steps: Vec<Value> = plan
34        .steps
35        .iter()
36        .map(|step| {
37            let mut rec = std::collections::BTreeMap::new();
38            rec.insert("id".into(), Value::U64(step.id as u64));
39            rec.insert("name".into(), Value::String(step.name.clone()));
40            rec.insert("capability".into(), Value::String(step.capability.clone()));
41            rec.insert("effect".into(), Value::String(step.effect.into()));
42            rec.insert("budget".into(), Value::U64(step.budget as u64));
43            rec.insert(
44                "inputs".into(),
45                Value::List(
46                    step.inputs
47                        .iter()
48                        .map(|s| Value::String(s.clone()))
49                        .collect(),
50                ),
51            );
52            rec.insert(
53                "outputs".into(),
54                Value::List(
55                    step.outputs
56                        .iter()
57                        .map(|s| Value::String(s.clone()))
58                        .collect(),
59                ),
60            );
61            rec.insert(
62                "depends_on".into(),
63                Value::List(
64                    step.depends_on
65                        .iter()
66                        .map(|d| Value::U64(*d as u64))
67                        .collect(),
68                ),
69            );
70            Value::Record(rec)
71        })
72        .collect();
73
74    Ok(args::record([
75        ("task", Value::String(plan.task.clone())),
76        ("steps", Value::List(steps)),
77        ("total_budget", Value::U64(plan.total_budget as u64)),
78        ("step_count", Value::U64(plan.steps.len() as u64)),
79    ]))
80}
81
82/// `Agent.execute` — execute a planned agent task.
83///
84/// Currently this builds the plan and validates it (the actual DAG execution
85/// is handled by `agent.dag.execute`). Takes `task` and `capabilities`,
86/// returns a plan record with an `execution_ready` flag.
87pub fn agent_execute(args: &Value, span: Span) -> Result<Value, Diagnostic> {
88    let task =
89        args::rec_str(args, "task").ok_or_else(|| args::bad(span, "Agent.execute needs task"))?;
90    let capabilities: Vec<String> = match args {
91        Value::Record(map) => match map.get("capabilities") {
92            Some(Value::List(caps)) => caps
93                .iter()
94                .filter_map(|c| match c {
95                    Value::String(s) => Some(s.clone()),
96                    _ => None,
97                })
98                .collect(),
99            _ => vec![],
100        },
101        _ => vec![],
102    };
103
104    let plan = planner::plan_task(task, &capabilities);
105    let step_count = plan.steps.len();
106    let execution_ready = step_count > 0 && plan.total_budget > 0;
107
108    Ok(args::record([
109        ("task", Value::String(plan.task.clone())),
110        ("step_count", Value::U64(step_count as u64)),
111        ("total_budget", Value::U64(plan.total_budget as u64)),
112        ("execution_ready", Value::Bool(execution_ready)),
113        ("status", Value::String("planned".into())),
114    ]))
115}
116
117/// `Corpus.load` — load a golden corpus from a file path (native only).
118///
119/// Takes `path` (string). Returns a record with `name`, `case_count`, and
120/// `tags`. On WASM without filesystem access, returns E300.
121#[cfg(not(target_arch = "wasm32"))]
122pub fn corpus_load(args: &Value, span: Span) -> Result<Value, Diagnostic> {
123    let path =
124        args::rec_str(args, "path").ok_or_else(|| args::bad(span, "Corpus.load needs path"))?;
125    match corpus::load_corpus_from_file(path) {
126        Ok(c) => {
127            let case_count = c.len();
128            let tags: Vec<Value> = c.all_tags().into_iter().map(Value::String).collect();
129            Ok(args::record([
130                ("name", Value::String(c.name)),
131                ("case_count", Value::U64(case_count as u64)),
132                ("tags", Value::List(tags)),
133                ("loaded", Value::Bool(true)),
134            ]))
135        }
136        Err(e) => Err(args::bad(span, format!("Corpus.load: {e}"))),
137    }
138}
139
140#[cfg(target_arch = "wasm32")]
141/// `?::corpus_load`.
142pub fn corpus_load(_args: &Value, span: Span) -> Result<Value, Diagnostic> {
143    Err(args::need_scientific(span, "Corpus.load"))
144}
145
146/// `Corpus.parse` — parse a golden corpus from an inline text string.
147///
148/// Takes `name` (string) and `text` (string). Returns a record with `name`,
149/// `case_count`, and `tags`.
150pub fn corpus_parse(args: &Value, span: Span) -> Result<Value, Diagnostic> {
151    let name =
152        args::rec_str(args, "name").ok_or_else(|| args::bad(span, "Corpus.parse needs name"))?;
153    let text =
154        args::rec_str(args, "text").ok_or_else(|| args::bad(span, "Corpus.parse needs text"))?;
155    let c = corpus::parse_corpus(name, text);
156    let case_count = c.len();
157    let tags: Vec<Value> = c.all_tags().into_iter().map(Value::String).collect();
158    Ok(args::record([
159        ("name", Value::String(c.name)),
160        ("case_count", Value::U64(case_count as u64)),
161        ("tags", Value::List(tags)),
162    ]))
163}
164
165/// `Agent.evaluate` — evaluate agent outputs against a golden corpus.
166///
167/// Takes `expected` (list of strings), `outputs` (list of strings), and
168/// optional `method` (string: "exact", "substring", "token_overlap").
169/// Returns a record with `total`, `passed`, `accuracy`, `mean_score`,
170/// `min_score`, `max_score`, and `results` (list of per-case records).
171pub fn agent_evaluate(args: &Value, span: Span) -> Result<Value, Diagnostic> {
172    let expected = args::rec_str_list(args, "expected")
173        .ok_or_else(|| args::bad(span, "Agent.evaluate needs expected"))?;
174    let outputs = args::rec_str_list(args, "outputs")
175        .ok_or_else(|| args::bad(span, "Agent.evaluate needs outputs"))?;
176    let method_str = args::rec_str(args, "method").unwrap_or("exact");
177    let method = match method_str {
178        "exact" => evaluator::MatchMethod::Exact,
179        "substring" => evaluator::MatchMethod::Substring,
180        "token_overlap" => evaluator::MatchMethod::TokenOverlap,
181        _ => evaluator::MatchMethod::Exact,
182    };
183
184    if expected.len() != outputs.len() {
185        return Err(args::bad(
186            span,
187            "Agent.evaluate: expected and outputs must have the same length",
188        ));
189    }
190
191    // Build a temporary corpus from expected strings.
192    let cases: Vec<corpus::GoldenCase> = expected
193        .iter()
194        .enumerate()
195        .map(|(i, exp)| corpus::GoldenCase {
196            name: format!("case_{i}"),
197            input: String::new(),
198            expected: exp.clone(),
199            tags: std::collections::BTreeSet::new(),
200        })
201        .collect();
202    let corp = corpus::GoldenCorpus {
203        name: "eval".into(),
204        cases,
205    };
206
207    let results = evaluator::evaluate_corpus(&corp, &outputs, method);
208    let metrics = evaluator::compute_metrics(&results);
209
210    let result_values: Vec<Value> = results
211        .iter()
212        .map(|r| {
213            let mut rec = std::collections::BTreeMap::new();
214            rec.insert("name".into(), Value::String(r.name.clone()));
215            rec.insert("passed".into(), Value::Bool(r.passed));
216            rec.insert("score".into(), Value::F64(r.score));
217            Value::Record(rec)
218        })
219        .collect();
220
221    Ok(args::record([
222        ("total", Value::U64(metrics.total as u64)),
223        ("passed", Value::U64(metrics.passed as u64)),
224        ("accuracy", Value::F64(metrics.accuracy)),
225        ("mean_score", Value::F64(metrics.mean_score)),
226        ("min_score", Value::F64(metrics.min_score)),
227        ("max_score", Value::F64(metrics.max_score)),
228        ("results", Value::List(result_values)),
229    ]))
230}
231
232#[cfg(test)]
233mod tests {
234    use super::*;
235    use std::collections::BTreeMap;
236
237    #[test]
238    fn agent_plan_basic() {
239        let mut m = BTreeMap::new();
240        m.insert("task".into(), Value::String("research the topic".into()));
241        m.insert(
242            "capabilities".into(),
243            Value::List(vec![
244                Value::String("NLP.substrate_extract".into()),
245                Value::String("NLP.analyze".into()),
246                Value::String("Asset.create".into()),
247            ]),
248        );
249        let result = agent_plan(&Value::Record(m), Span { start: 0, end: 0 });
250        assert!(result.is_ok());
251        match result.unwrap() {
252            Value::Record(rec) => {
253                assert!(rec.contains_key("steps"));
254                assert!(rec.contains_key("total_budget"));
255                match rec.get("steps") {
256                    Some(Value::List(steps)) => assert!(!steps.is_empty()),
257                    _ => panic!("expected steps list"),
258                }
259            }
260            _ => panic!("expected record"),
261        }
262    }
263
264    #[test]
265    fn agent_plan_missing_task() {
266        let m = BTreeMap::new();
267        let result = agent_plan(&Value::Record(m), Span { start: 0, end: 0 });
268        assert!(result.is_err());
269    }
270
271    #[test]
272    fn agent_execute_basic() {
273        let mut m = BTreeMap::new();
274        m.insert("task".into(), Value::String("analyse the data".into()));
275        m.insert(
276            "capabilities".into(),
277            Value::List(vec![Value::String("NLP.analyze".into())]),
278        );
279        let result = agent_execute(&Value::Record(m), Span { start: 0, end: 0 });
280        assert!(result.is_ok());
281        match result.unwrap() {
282            Value::Record(rec) => {
283                assert_eq!(rec.get("execution_ready"), Some(&Value::Bool(true)));
284            }
285            _ => panic!("expected record"),
286        }
287    }
288
289    #[test]
290    fn corpus_parse_inline() {
291        let mut m = BTreeMap::new();
292        m.insert("name".into(), Value::String("test".into()));
293        m.insert(
294            "text".into(),
295            Value::String("# case: a\ninput: x\nexpected: y\n".into()),
296        );
297        let result = corpus_parse(&Value::Record(m), Span { start: 0, end: 0 });
298        assert!(result.is_ok());
299        match result.unwrap() {
300            Value::Record(rec) => {
301                assert_eq!(rec.get("case_count"), Some(&Value::U64(1)));
302            }
303            _ => panic!("expected record"),
304        }
305    }
306
307    #[test]
308    fn agent_evaluate_exact() {
309        let mut m = BTreeMap::new();
310        m.insert(
311            "expected".into(),
312            Value::List(vec![
313                Value::String("hello".into()),
314                Value::String("world".into()),
315            ]),
316        );
317        m.insert(
318            "outputs".into(),
319            Value::List(vec![
320                Value::String("hello".into()),
321                Value::String("world".into()),
322            ]),
323        );
324        m.insert("method".into(), Value::String("exact".into()));
325        let result = agent_evaluate(&Value::Record(m), Span { start: 0, end: 0 });
326        assert!(result.is_ok());
327        match result.unwrap() {
328            Value::Record(rec) => {
329                assert_eq!(rec.get("total"), Some(&Value::U64(2)));
330                assert_eq!(rec.get("passed"), Some(&Value::U64(2)));
331                match rec.get("accuracy") {
332                    Some(Value::F64(a)) => assert!((a - 1.0).abs() < 1e-9),
333                    _ => panic!("expected f64 accuracy"),
334                }
335            }
336            _ => panic!("expected record"),
337        }
338    }
339
340    #[test]
341    fn agent_evaluate_mismatch_length() {
342        let mut m = BTreeMap::new();
343        m.insert(
344            "expected".into(),
345            Value::List(vec![Value::String("hello".into())]),
346        );
347        m.insert(
348            "outputs".into(),
349            Value::List(vec![
350                Value::String("hello".into()),
351                Value::String("world".into()),
352            ]),
353        );
354        let result = agent_evaluate(&Value::Record(m), Span { start: 0, end: 0 });
355        assert!(result.is_err());
356    }
357
358    #[test]
359    fn agent_evaluate_token_overlap() {
360        let mut m = BTreeMap::new();
361        m.insert(
362            "expected".into(),
363            Value::List(vec![Value::String("hello world".into())]),
364        );
365        m.insert(
366            "outputs".into(),
367            Value::List(vec![Value::String("hello world foo".into())]),
368        );
369        m.insert("method".into(), Value::String("token_overlap".into()));
370        let result = agent_evaluate(&Value::Record(m), Span { start: 0, end: 0 });
371        assert!(result.is_ok());
372        match result.unwrap() {
373            Value::Record(rec) => match rec.get("mean_score") {
374                Some(Value::F64(s)) => assert!(*s > 0.0 && *s < 1.0),
375                _ => panic!("expected f64 mean_score"),
376            },
377            _ => panic!("expected record"),
378        }
379    }
380}