Skip to main content

qualia_vision/
metrics.rs

1//! Swarm W — evaluation metrics (synthetic always; real H1 when corpus supplied).
2//!
3//! Synthetic and real metrics are **never mixed** into one score.
4
5use crate::preprocess::iou_u16;
6use crate::synthetic::{generate_scene_rgb8, match_accuracy, sample_id, DatasetSplit};
7use crate::types::{Detection, ImageView, PixelFormat, VisualModel, MAX_DETECTIONS};
8use crate::weights::VisionBackendKind;
9
10#[derive(Debug, Clone, Copy, PartialEq)]
11pub struct MetricsReport {
12    pub backend: VisionBackendKind,
13    pub model_hash: u64,
14    pub n_samples: u32,
15    pub mean_match_acc: f32,
16    pub mean_detections: f32,
17    /// True only when metrics ran on an external real set (H1).
18    pub is_real_eval: bool,
19    /// Always set when synthetic partition used.
20    pub is_synthetic_eval: bool,
21}
22
23/// Run detector on `n` synthetic **test** samples; returns mean match accuracy vs GT.
24pub fn evaluate_synthetic<M: VisualModel>(
25    model: &mut M,
26    backend: VisionBackendKind,
27    model_hash: u64,
28    n_samples: u32,
29    width: u32,
30    height: u32,
31) -> MetricsReport {
32    let n = n_samples.max(1).min(64);
33    let mut sum_acc = 0.0f32;
34    let mut sum_det = 0.0f32;
35    let px = (width as usize) * (height as usize) * 3;
36    let mut rgb = vec![0u8; px];
37    let mut gt = [Detection::empty(); MAX_DETECTIONS];
38    let mut pred = [Detection::empty(); MAX_DETECTIONS];
39    let mut emb = [0.0f32; 32];
40    let mut ws = [0u8; MAX_DETECTIONS];
41
42    for i in 0..n {
43        let sample = sample_id(DatasetSplit::Test, i, width, height);
44        let n_gt = generate_scene_rgb8(&sample, &mut rgb, &mut gt).unwrap_or(0);
45        let img = ImageView {
46            bytes: &rgb,
47            width,
48            height,
49            row_stride: width * 3,
50            format: PixelFormat::Rgb8,
51        };
52        let counts = model.infer(img, &mut pred, &mut emb, &mut ws).unwrap_or(
53            crate::types::VisualOutputCounts {
54                detections: 0,
55                embedding_written: 0,
56            },
57        );
58        sum_acc += match_accuracy(&gt, n_gt, &pred, counts.detections, 0.15);
59        sum_det += counts.detections as f32;
60    }
61    MetricsReport {
62        backend,
63        model_hash,
64        n_samples: n,
65        mean_match_acc: sum_acc / n as f32,
66        mean_detections: sum_det / n as f32,
67        is_real_eval: false,
68        is_synthetic_eval: true,
69    }
70}
71
72/// Placeholder for H1 real eval: caller supplies (image, gt) pairs.
73/// Returns None if empty — never invents real metrics.
74pub fn evaluate_real_held_out(
75    pairs: &[(ImageView<'_>, &[Detection])],
76    model: &mut impl VisualModel,
77    backend: VisionBackendKind,
78    model_hash: u64,
79    iou_thresh: f32,
80) -> Option<MetricsReport> {
81    if pairs.is_empty() {
82        return None;
83    }
84    let mut sum_acc = 0.0f32;
85    let mut sum_det = 0.0f32;
86    let mut pred = [Detection::empty(); MAX_DETECTIONS];
87    let mut emb = [0.0f32; 32];
88    let mut ws = [0u8; MAX_DETECTIONS];
89    for (img, gt) in pairs {
90        let counts = model.infer(*img, &mut pred, &mut emb, &mut ws).ok()?;
91        sum_acc += match_accuracy(gt, gt.len(), &pred, counts.detections, iou_thresh);
92        sum_det += counts.detections as f32;
93    }
94    let n = pairs.len() as u32;
95    Some(MetricsReport {
96        backend,
97        model_hash,
98        n_samples: n,
99        mean_match_acc: sum_acc / n as f32,
100        mean_detections: sum_det / n as f32,
101        is_real_eval: true,
102        is_synthetic_eval: false,
103    })
104}
105
106/// Mean IoU of best-matching same-class pairs (diagnostic).
107pub fn mean_best_iou(gt: &[Detection], n_gt: usize, pred: &[Detection], n_pred: usize) -> f32 {
108    if n_gt == 0 {
109        return 1.0;
110    }
111    let mut s = 0.0f32;
112    for g in gt.iter().take(n_gt) {
113        let mut best = 0.0f32;
114        for p in pred.iter().take(n_pred) {
115            if p.class_hash == g.class_hash {
116                best = best.max(iou_u16(g, p));
117            }
118        }
119        s += best;
120    }
121    s / n_gt as f32
122}
123
124#[cfg(test)]
125mod tests {
126    use super::*;
127    use crate::detector::{GridMultiObjectDetector, CLASS_MOSTLY_BLUE, CLASS_MOSTLY_RED};
128    use crate::weights::{ProductionVision, VisionWeightBundle};
129
130    #[test]
131    fn synthetic_metrics_reference() {
132        let mut m = GridMultiObjectDetector::new(4, 3);
133        let mh = m.model_hash();
134        let r = evaluate_synthetic(&mut m, VisionBackendKind::Reference, mh, 4, 48, 32);
135        assert!(r.is_synthetic_eval);
136        assert!(!r.is_real_eval);
137        assert_eq!(r.n_samples, 4);
138        assert!(r.mean_detections >= 0.0);
139    }
140
141    #[test]
142    fn synthetic_metrics_production() {
143        let b = VisionWeightBundle::from_seed(7, 16, &[CLASS_MOSTLY_RED, CLASS_MOSTLY_BLUE]);
144        let h = b.model_hash();
145        let mut m = ProductionVision::new(b);
146        let r = evaluate_synthetic(&mut m, VisionBackendKind::ProductionWeights, h, 3, 32, 32);
147        assert!(r.is_synthetic_eval);
148        assert_eq!(r.backend, VisionBackendKind::ProductionWeights);
149    }
150
151    #[test]
152    fn real_eval_empty_is_none() {
153        let mut m = GridMultiObjectDetector::new(2, 2);
154        assert!(
155            evaluate_real_held_out(&[], &mut m, VisionBackendKind::Reference, 0, 0.5).is_none()
156        );
157    }
158}