qualia_core_db/inference/lab/
hypothesis.rs

1//! Hypothesis backlog and typed belief graph with confidence cascade.
2//!
3//! The belief graph is a directed acyclic graph (DAG) of epistemic nodes:
4//! - **Hypothesis**: a testable claim about an optimization strategy.
5//! - **Experiment**: a trial that produces evidence for/against a hypothesis.
6//! - **Observation**: a measured datum from an experiment.
7//! - **Claim**: a derived conclusion from one or more observations.
8//!
9//! When an experiment produces a verdict on a hypothesis, the confidence
10//! cascades to dependent hypotheses via the DAG edges.
11
12use std::collections::HashMap;
13
14use serde::{Deserialize, Serialize};
15
16/// Unique identifier for a belief graph node.
17pub type NodeId = String;
18
19/// A hypothesis: a testable claim about an optimization strategy.
20#[derive(Debug, Clone, Serialize, Deserialize)]
21pub struct Hypothesis {
22    pub id: NodeId,
23    /// Human-readable statement, e.g. "coop_gemv improves decode tok/s by >20%".
24    pub statement: String,
25    /// The configuration space this hypothesis operates on.
26    pub space_name: String,
27    /// Expected direction: true = improvement, false = regression.
28    pub expects_improvement: bool,
29    /// Current confidence in [-1, 1]: -1 = refuted, 0 = unknown, +1 = confirmed.
30    pub confidence: f64,
31    /// IDs of hypotheses this depends on (predecessors in the DAG).
32    pub depends_on: Vec<NodeId>,
33    /// IDs of experiments that have tested this hypothesis.
34    pub experiments: Vec<NodeId>,
35    /// Whether this hypothesis is active (eligible for testing).
36    pub active: bool,
37    /// Creation timestamp (unix ms).
38    pub created_ms: u64,
39}
40
41impl Hypothesis {
42    pub fn new(
43        id: impl Into<String>,
44        statement: impl Into<String>,
45        space_name: impl Into<String>,
46    ) -> Self {
47        let now = std::time::SystemTime::now()
48            .duration_since(std::time::UNIX_EPOCH)
49            .map(|d| d.as_millis() as u64)
50            .unwrap_or(0);
51        Self {
52            id: id.into(),
53            statement: statement.into(),
54            space_name: space_name.into(),
55            expects_improvement: true,
56            confidence: 0.0,
57            depends_on: Vec::new(),
58            experiments: Vec::new(),
59            active: true,
60            created_ms: now,
61        }
62    }
63
64    pub fn with_dependency(mut self, dep: impl Into<String>) -> Self {
65        self.depends_on.push(dep.into());
66        self
67    }
68
69    pub fn expects_regression(mut self) -> Self {
70        self.expects_improvement = false;
71        self
72    }
73}
74
75/// The verdict of an experiment on a hypothesis.
76#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
77pub enum ExperimentVerdict {
78    /// The hypothesis is supported by the evidence.
79    Confirmed,
80    /// The hypothesis is refuted by the evidence.
81    Refuted,
82    /// The evidence is inconclusive.
83    Inconclusive,
84    /// The experiment failed (no data).
85    Failed,
86}
87
88impl ExperimentVerdict {
89    /// Convert to a confidence delta.
90    pub fn confidence_delta(&self, weight: f64) -> f64 {
91        match self {
92            Self::Confirmed => weight,
93            Self::Refuted => -weight,
94            Self::Inconclusive => 0.0,
95            Self::Failed => 0.0,
96        }
97    }
98}
99
100/// An experiment node in the belief graph.
101#[derive(Debug, Clone, Serialize, Deserialize)]
102pub struct ExperimentNode {
103    pub id: NodeId,
104    /// The hypothesis this experiment tests.
105    pub hypothesis_id: NodeId,
106    /// The configuration tested.
107    pub config_hash: u64,
108    /// The verdict.
109    pub verdict: ExperimentVerdict,
110    /// Weight of this experiment's evidence [0, 1].
111    pub weight: f64,
112    /// Timestamp (unix ms).
113    pub timestamp_ms: u64,
114}
115
116/// An observation node: a single measured datum.
117#[derive(Debug, Clone, Serialize, Deserialize)]
118pub struct Observation {
119    pub id: NodeId,
120    pub experiment_id: NodeId,
121    pub metric: String,
122    pub value: f64,
123    pub unit: String,
124}
125
126/// A claim node: a derived conclusion.
127#[derive(Debug, Clone, Serialize, Deserialize)]
128pub struct Claim {
129    pub id: NodeId,
130    pub statement: String,
131    /// Observations that support this claim.
132    pub supported_by: Vec<NodeId>,
133    pub confidence: f64,
134}
135
136/// The typed belief graph.
137#[derive(Debug, Clone, Serialize, Deserialize, Default)]
138pub struct BeliefGraph {
139    pub hypotheses: HashMap<NodeId, Hypothesis>,
140    pub experiments: HashMap<NodeId, ExperimentNode>,
141    pub observations: HashMap<NodeId, Observation>,
142    pub claims: HashMap<NodeId, Claim>,
143}
144
145impl BeliefGraph {
146    pub fn new() -> Self {
147        Self::default()
148    }
149
150    /// Add a hypothesis to the graph.
151    pub fn add_hypothesis(&mut self, h: Hypothesis) {
152        self.hypotheses.insert(h.id.clone(), h);
153    }
154
155    /// Record an experiment result against a hypothesis and update confidence.
156    pub fn record_experiment(
157        &mut self,
158        experiment_id: impl Into<String>,
159        hypothesis_id: &str,
160        config_hash: u64,
161        verdict: ExperimentVerdict,
162        weight: f64,
163    ) {
164        let now = std::time::SystemTime::now()
165            .duration_since(std::time::UNIX_EPOCH)
166            .map(|d| d.as_millis() as u64)
167            .unwrap_or(0);
168
169        let exp = ExperimentNode {
170            id: experiment_id.into(),
171            hypothesis_id: hypothesis_id.to_string(),
172            config_hash,
173            verdict,
174            weight,
175            timestamp_ms: now,
176        };
177
178        // Update hypothesis confidence.
179        if let Some(h) = self.hypotheses.get_mut(hypothesis_id) {
180            let delta = verdict.confidence_delta(weight);
181            // Exponential moving average for confidence.
182            let alpha = 0.3;
183            h.confidence = h.confidence * (1.0 - alpha) + delta * alpha;
184            h.confidence = h.confidence.clamp(-1.0, 1.0);
185            h.experiments.push(exp.id.clone());
186
187            // If strongly refuted, deactivate.
188            if h.confidence < -0.5 {
189                h.active = false;
190            }
191        }
192
193        self.experiments.insert(exp.id.clone(), exp);
194
195        // Cascade confidence to dependent hypotheses.
196        self.cascade_confidence(hypothesis_id);
197    }
198
199    /// Cascade confidence from a hypothesis to its dependents.
200    fn cascade_confidence(&mut self, source_id: &str) {
201        // Find hypotheses that depend on the source.
202        let dependents: Vec<NodeId> = self
203            .hypotheses
204            .values()
205            .filter(|h| h.depends_on.iter().any(|d| d == source_id))
206            .map(|h| h.id.clone())
207            .collect();
208
209        for dep_id in dependents {
210            // The dependent's confidence is influenced by the source's confidence.
211            let source_confidence = self
212                .hypotheses
213                .get(source_id)
214                .map(|s| s.confidence)
215                .unwrap_or(0.0);
216            if let Some(dep) = self.hypotheses.get_mut(&dep_id) {
217                // If the source is confirmed, boost the dependent's confidence.
218                // If refuted, reduce it.
219                let influence = source_confidence * 0.2;
220                dep.confidence = (dep.confidence + influence).clamp(-1.0, 1.0);
221            }
222        }
223    }
224
225    /// Add an observation linked to an experiment.
226    pub fn add_observation(
227        &mut self,
228        id: impl Into<String>,
229        experiment_id: impl Into<String>,
230        metric: impl Into<String>,
231        value: f64,
232        unit: impl Into<String>,
233    ) {
234        let obs = Observation {
235            id: id.into(),
236            experiment_id: experiment_id.into(),
237            metric: metric.into(),
238            value,
239            unit: unit.into(),
240        };
241        self.observations.insert(obs.id.clone(), obs);
242    }
243
244    /// Add a claim supported by observations.
245    pub fn add_claim(
246        &mut self,
247        id: impl Into<String>,
248        statement: impl Into<String>,
249        supported_by: Vec<NodeId>,
250    ) {
251        let confidence = self.compute_claim_confidence(&supported_by);
252        let claim = Claim {
253            id: id.into(),
254            statement: statement.into(),
255            supported_by,
256            confidence,
257        };
258        self.claims.insert(claim.id.clone(), claim);
259    }
260
261    /// Compute confidence for a claim based on supporting observations.
262    fn compute_claim_confidence(&self, obs_ids: &[NodeId]) -> f64 {
263        if obs_ids.is_empty() {
264            return 0.0;
265        }
266        // Average the experiment confidences that produced these observations.
267        let mut total = 0.0;
268        let mut count = 0;
269        for obs_id in obs_ids {
270            if let Some(obs) = self.observations.get(obs_id) {
271                if let Some(exp) = self.experiments.get(&obs.experiment_id) {
272                    if let Some(h) = self.hypotheses.get(&exp.hypothesis_id) {
273                        total += h.confidence;
274                        count += 1;
275                    }
276                }
277            }
278        }
279        if count == 0 {
280            0.0
281        } else {
282            (total / count as f64).clamp(-1.0, 1.0)
283        }
284    }
285
286    /// Get all active hypotheses sorted by confidence (descending).
287    pub fn active_hypotheses(&self) -> Vec<&Hypothesis> {
288        let mut active: Vec<&Hypothesis> = self.hypotheses.values().filter(|h| h.active).collect();
289        active.sort_by(|a, b| {
290            b.confidence
291                .partial_cmp(&a.confidence)
292                .unwrap_or(std::cmp::Ordering::Equal)
293        });
294        active
295    }
296
297    /// Get the next hypothesis to test (highest confidence uncertainty).
298    pub fn next_to_test(&self) -> Option<&Hypothesis> {
299        self.active_hypotheses().into_iter().min_by(|a, b| {
300            a.confidence
301                .abs()
302                .partial_cmp(&b.confidence.abs())
303                .unwrap_or(std::cmp::Ordering::Equal)
304        })
305    }
306
307    /// Serialize the entire belief graph to JSON.
308    pub fn to_json(&self) -> String {
309        serde_json::to_string_pretty(self).unwrap_or_else(|e| format!("{{\"error\":\"{e}\"}}"))
310    }
311
312    /// Save to a JSON file.
313    pub fn save(&self, path: &std::path::Path) -> Result<(), String> {
314        if let Some(parent) = path.parent() {
315            std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
316        }
317        let json = self.to_json();
318        std::fs::write(path, json).map_err(|e| e.to_string())
319    }
320
321    /// Load from a JSON file.
322    pub fn load(path: &std::path::Path) -> Result<Self, String> {
323        let content = std::fs::read_to_string(path).map_err(|e| e.to_string())?;
324        serde_json::from_str(&content).map_err(|e| e.to_string())
325    }
326}
327
328/// Determine the verdict of an experiment against a hypothesis.
329/// This compares the experiment result against a baseline.
330pub fn evaluate_verdict(
331    hypothesis: &Hypothesis,
332    treatment_tok_s: f64,
333    baseline_tok_s: f64,
334    improvement_threshold: f64,
335) -> ExperimentVerdict {
336    if baseline_tok_s <= 0.0 || treatment_tok_s <= 0.0 {
337        return ExperimentVerdict::Failed;
338    }
339    let relative = (treatment_tok_s - baseline_tok_s) / baseline_tok_s;
340    let improved = relative > improvement_threshold;
341    let regressed = relative < -improvement_threshold;
342
343    match (hypothesis.expects_improvement, improved, regressed) {
344        (true, true, false) => ExperimentVerdict::Confirmed,
345        (true, false, true) => ExperimentVerdict::Refuted,
346        (false, false, true) => ExperimentVerdict::Confirmed,
347        (false, true, false) => ExperimentVerdict::Refuted,
348        _ => ExperimentVerdict::Inconclusive,
349    }
350}
351
352#[cfg(test)]
353mod tests {
354    use super::*;
355
356    #[test]
357    fn hypothesis_confidence_updates() {
358        let mut graph = BeliefGraph::new();
359        let h = Hypothesis::new("H-001", "coop_gemv improves decode by >20%", "toggle_space");
360        graph.add_hypothesis(h);
361
362        graph.record_experiment("E-001", "H-001", 42, ExperimentVerdict::Confirmed, 0.8);
363        let h = graph.hypotheses.get("H-001").unwrap();
364        assert!(h.confidence > 0.0);
365        assert_eq!(h.experiments.len(), 1);
366    }
367
368    #[test]
369    fn confidence_cascade() {
370        let mut graph = BeliefGraph::new();
371        let h1 = Hypothesis::new("H-001", "coop_gemv improves decode", "space");
372        graph.add_hypothesis(h1);
373        let h2 =
374            Hypothesis::new("H-002", "fused_ffn improves decode", "space").with_dependency("H-001");
375        graph.add_hypothesis(h2);
376
377        // Confirm H-001 strongly.
378        graph.record_experiment("E-001", "H-001", 42, ExperimentVerdict::Confirmed, 1.0);
379        graph.record_experiment("E-002", "H-001", 43, ExperimentVerdict::Confirmed, 1.0);
380
381        // H-002 should have positive confidence from cascade.
382        let h2 = graph.hypotheses.get("H-002").unwrap();
383        assert!(h2.confidence > 0.0);
384    }
385
386    #[test]
387    fn refuted_hypothesis_deactivates() {
388        let mut graph = BeliefGraph::new();
389        graph.add_hypothesis(Hypothesis::new("H-001", "X improves Y", "space"));
390
391        // Strongly refute.
392        graph.record_experiment("E-001", "H-001", 1, ExperimentVerdict::Refuted, 1.0);
393        graph.record_experiment("E-002", "H-001", 2, ExperimentVerdict::Refuted, 1.0);
394        graph.record_experiment("E-003", "H-001", 3, ExperimentVerdict::Refuted, 1.0);
395
396        let h = graph.hypotheses.get("H-001").unwrap();
397        assert!(!h.active);
398    }
399
400    #[test]
401    fn evaluate_verdict_improvement() {
402        let h = Hypothesis::new("H-001", "coop improves tok/s", "space");
403        let v = evaluate_verdict(&h, 60.0, 40.0, 0.20);
404        assert_eq!(v, ExperimentVerdict::Confirmed);
405
406        let v = evaluate_verdict(&h, 41.0, 40.0, 0.20);
407        assert_eq!(v, ExperimentVerdict::Inconclusive);
408
409        let v = evaluate_verdict(&h, 30.0, 40.0, 0.20);
410        assert_eq!(v, ExperimentVerdict::Refuted);
411    }
412
413    #[test]
414    fn evaluate_verdict_regression() {
415        let h =
416            Hypothesis::new("H-001", "naive GEMV regresses tok/s", "space").expects_regression();
417        let v = evaluate_verdict(&h, 30.0, 40.0, 0.20);
418        assert_eq!(v, ExperimentVerdict::Confirmed);
419
420        let v = evaluate_verdict(&h, 60.0, 40.0, 0.20);
421        assert_eq!(v, ExperimentVerdict::Refuted);
422    }
423
424    #[test]
425    fn next_to_test_picks_uncertain() {
426        let mut graph = BeliefGraph::new();
427        graph.add_hypothesis(Hypothesis::new("H-001", "certain claim", "space"));
428        graph.add_hypothesis(Hypothesis::new("H-002", "uncertain claim", "space"));
429
430        // Make H-001 very confident.
431        graph.record_experiment("E-001", "H-001", 1, ExperimentVerdict::Confirmed, 1.0);
432        graph.record_experiment("E-002", "H-001", 2, ExperimentVerdict::Confirmed, 1.0);
433
434        // H-002 is still at 0.0 — most uncertain.
435        let next = graph.next_to_test();
436        assert!(next.is_some());
437        assert_eq!(next.unwrap().id, "H-002");
438    }
439
440    #[test]
441    fn belief_graph_save_load() {
442        let mut graph = BeliefGraph::new();
443        graph.add_hypothesis(Hypothesis::new("H-001", "test", "space"));
444        graph.record_experiment("E-001", "H-001", 42, ExperimentVerdict::Confirmed, 0.5);
445
446        let tmp =
447            std::env::temp_dir().join(format!("qualia_belief_test_{}.json", std::process::id()));
448        graph.save(&tmp).unwrap();
449        let loaded = BeliefGraph::load(&tmp).unwrap();
450        assert!(loaded.hypotheses.contains_key("H-001"));
451        assert!(loaded.experiments.contains_key("E-001"));
452        let _ = std::fs::remove_file(&tmp);
453    }
454}
qualia_core_db/inference/lab/hypothesis.rs

qualia_core_db/inference/lab/
hypothesis.rs
