1use crate::preprocess::iou_u16;
6use crate::synthetic::{generate_scene_rgb8, match_accuracy, sample_id, DatasetSplit};
7use crate::types::{Detection, ImageView, PixelFormat, VisualModel, MAX_DETECTIONS};
8use crate::weights::VisionBackendKind;
9
10#[derive(Debug, Clone, Copy, PartialEq)]
11pub struct MetricsReport {
12 pub backend: VisionBackendKind,
13 pub model_hash: u64,
14 pub n_samples: u32,
15 pub mean_match_acc: f32,
16 pub mean_detections: f32,
17 pub is_real_eval: bool,
19 pub is_synthetic_eval: bool,
21}
22
23pub fn evaluate_synthetic<M: VisualModel>(
25 model: &mut M,
26 backend: VisionBackendKind,
27 model_hash: u64,
28 n_samples: u32,
29 width: u32,
30 height: u32,
31) -> MetricsReport {
32 let n = n_samples.max(1).min(64);
33 let mut sum_acc = 0.0f32;
34 let mut sum_det = 0.0f32;
35 let px = (width as usize) * (height as usize) * 3;
36 let mut rgb = vec![0u8; px];
37 let mut gt = [Detection::empty(); MAX_DETECTIONS];
38 let mut pred = [Detection::empty(); MAX_DETECTIONS];
39 let mut emb = [0.0f32; 32];
40 let mut ws = [0u8; MAX_DETECTIONS];
41
42 for i in 0..n {
43 let sample = sample_id(DatasetSplit::Test, i, width, height);
44 let n_gt = generate_scene_rgb8(&sample, &mut rgb, &mut gt).unwrap_or(0);
45 let img = ImageView {
46 bytes: &rgb,
47 width,
48 height,
49 row_stride: width * 3,
50 format: PixelFormat::Rgb8,
51 };
52 let counts = model.infer(img, &mut pred, &mut emb, &mut ws).unwrap_or(
53 crate::types::VisualOutputCounts {
54 detections: 0,
55 embedding_written: 0,
56 },
57 );
58 sum_acc += match_accuracy(>, n_gt, &pred, counts.detections, 0.15);
59 sum_det += counts.detections as f32;
60 }
61 MetricsReport {
62 backend,
63 model_hash,
64 n_samples: n,
65 mean_match_acc: sum_acc / n as f32,
66 mean_detections: sum_det / n as f32,
67 is_real_eval: false,
68 is_synthetic_eval: true,
69 }
70}
71
72pub fn evaluate_real_held_out(
75 pairs: &[(ImageView<'_>, &[Detection])],
76 model: &mut impl VisualModel,
77 backend: VisionBackendKind,
78 model_hash: u64,
79 iou_thresh: f32,
80) -> Option<MetricsReport> {
81 if pairs.is_empty() {
82 return None;
83 }
84 let mut sum_acc = 0.0f32;
85 let mut sum_det = 0.0f32;
86 let mut pred = [Detection::empty(); MAX_DETECTIONS];
87 let mut emb = [0.0f32; 32];
88 let mut ws = [0u8; MAX_DETECTIONS];
89 for (img, gt) in pairs {
90 let counts = model.infer(*img, &mut pred, &mut emb, &mut ws).ok()?;
91 sum_acc += match_accuracy(gt, gt.len(), &pred, counts.detections, iou_thresh);
92 sum_det += counts.detections as f32;
93 }
94 let n = pairs.len() as u32;
95 Some(MetricsReport {
96 backend,
97 model_hash,
98 n_samples: n,
99 mean_match_acc: sum_acc / n as f32,
100 mean_detections: sum_det / n as f32,
101 is_real_eval: true,
102 is_synthetic_eval: false,
103 })
104}
105
106pub fn mean_best_iou(gt: &[Detection], n_gt: usize, pred: &[Detection], n_pred: usize) -> f32 {
108 if n_gt == 0 {
109 return 1.0;
110 }
111 let mut s = 0.0f32;
112 for g in gt.iter().take(n_gt) {
113 let mut best = 0.0f32;
114 for p in pred.iter().take(n_pred) {
115 if p.class_hash == g.class_hash {
116 best = best.max(iou_u16(g, p));
117 }
118 }
119 s += best;
120 }
121 s / n_gt as f32
122}
123
124#[cfg(test)]
125mod tests {
126 use super::*;
127 use crate::detector::{GridMultiObjectDetector, CLASS_MOSTLY_BLUE, CLASS_MOSTLY_RED};
128 use crate::weights::{ProductionVision, VisionWeightBundle};
129
130 #[test]
131 fn synthetic_metrics_reference() {
132 let mut m = GridMultiObjectDetector::new(4, 3);
133 let mh = m.model_hash();
134 let r = evaluate_synthetic(&mut m, VisionBackendKind::Reference, mh, 4, 48, 32);
135 assert!(r.is_synthetic_eval);
136 assert!(!r.is_real_eval);
137 assert_eq!(r.n_samples, 4);
138 assert!(r.mean_detections >= 0.0);
139 }
140
141 #[test]
142 fn synthetic_metrics_production() {
143 let b = VisionWeightBundle::from_seed(7, 16, &[CLASS_MOSTLY_RED, CLASS_MOSTLY_BLUE]);
144 let h = b.model_hash();
145 let mut m = ProductionVision::new(b);
146 let r = evaluate_synthetic(&mut m, VisionBackendKind::ProductionWeights, h, 3, 32, 32);
147 assert!(r.is_synthetic_eval);
148 assert_eq!(r.backend, VisionBackendKind::ProductionWeights);
149 }
150
151 #[test]
152 fn real_eval_empty_is_none() {
153 let mut m = GridMultiObjectDetector::new(2, 2);
154 assert!(
155 evaluate_real_held_out(&[], &mut m, VisionBackendKind::Reference, 0, 0.5).is_none()
156 );
157 }
158}