Skip to main content

qualia_vision/
detector.rs

1//! Phase 5 / V5 — multi-object detector pipeline (grid head + NMS).
2//!
3//! Honest scope: pure-Rust, fixed-buffer path over a configurable cell grid.
4//! Not a neural YOLO head — scores each cell by channel mean / edge energy
5//! (same family as `CpuReferenceVision`) then applies class-agnostic NMS.
6//! Replace the cell scorer with a P64 encoder when weights land; keep the
7//! pipeline + `VisualModel` surface.
8
9use crate::preprocess::nms_class_agnostic;
10use crate::semantic::q_hash;
11use crate::types::{
12    Detection, ImageView, PixelFormat, VisionError, VisualCapabilities, VisualModel,
13    VisualOutputCounts, MAX_DETECTIONS,
14};
15
16/// Stable class IRIs (shared taxonomy with cpu_reference).
17pub const CLASS_MOSTLY_RED: &str = "https://ns.webizen.org/q42/vision/class/mostly-red";
18pub const CLASS_MOSTLY_GREEN: &str = "https://ns.webizen.org/q42/vision/class/mostly-green";
19pub const CLASS_MOSTLY_BLUE: &str = "https://ns.webizen.org/q42/vision/class/mostly-blue";
20pub const CLASS_HIGH_EDGE: &str = "https://ns.webizen.org/q42/vision/class/high-edge-energy";
21pub const CLASS_LOW_CONTRAST: &str = "https://ns.webizen.org/q42/vision/class/low-contrast";
22
23const MODEL_ID: &str = "qualia-vision-grid-detector-v1";
24
25/// Maximum grid dimension (NxN cells before NMS). 8×8 = 64 = MAX_DETECTIONS.
26pub const MAX_GRID: u32 = 8;
27
28/// Configurable multi-object grid detector.
29#[derive(Debug, Clone)]
30pub struct GridMultiObjectDetector {
31    pub grid_w: u32,
32    pub grid_h: u32,
33    pub min_score: f32,
34    pub nms_iou: f32,
35    model_hash: u64,
36}
37
38impl Default for GridMultiObjectDetector {
39    fn default() -> Self {
40        Self::new(2, 2)
41    }
42}
43
44impl GridMultiObjectDetector {
45    /// Creates a new instance.
46    pub fn new(grid_w: u32, grid_h: u32) -> Self {
47        let gw = grid_w.clamp(1, MAX_GRID);
48        let gh = grid_h.clamp(1, MAX_GRID);
49        Self {
50            grid_w: gw,
51            grid_h: gh,
52            min_score: 0.15,
53            nms_iou: 0.45,
54            model_hash: q_hash(MODEL_ID) ^ ((gw as u64) << 8) ^ (gh as u64),
55        }
56    }
57
58    /// `model` hash.
59    pub fn model_hash(&self) -> u64 {
60        self.model_hash
61    }
62
63    fn sample_rgb(img: ImageView<'_>, x: u32, y: u32) -> (u8, u8, u8) {
64        let bpp = img.bytes_per_pixel() as usize;
65        let off = (y as usize)
66            .saturating_mul(img.row_stride as usize)
67            .saturating_add((x as usize).saturating_mul(bpp));
68        if off + bpp > img.bytes.len() {
69            return (0, 0, 0);
70        }
71        match img.format {
72            PixelFormat::Gray8 => {
73                let g = img.bytes[off];
74                (g, g, g)
75            }
76            PixelFormat::Rgb8 | PixelFormat::Rgba8 => {
77                (img.bytes[off], img.bytes[off + 1], img.bytes[off + 2])
78            }
79            PixelFormat::Bgr8 => (img.bytes[off + 2], img.bytes[off + 1], img.bytes[off]),
80            PixelFormat::RgbF32 => (0, 0, 0),
81        }
82    }
83
84    /// Score one cell into a Detection (class + box + score). Returns None if below min_score.
85    fn score_cell(
86        &self,
87        image: ImageView<'_>,
88        cx: u32,
89        cy: u32,
90        frame_index: u32,
91    ) -> Option<Detection> {
92        let w = image.width;
93        let h = image.height;
94        let x0 = cx * w / self.grid_w;
95        let y0 = cy * h / self.grid_h;
96        let x1 = if cx + 1 >= self.grid_w {
97            w
98        } else {
99            (cx + 1) * w / self.grid_w
100        };
101        let y1 = if cy + 1 >= self.grid_h {
102            h
103        } else {
104            (cy + 1) * h / self.grid_h
105        };
106        if x1 <= x0 || y1 <= y0 {
107            return None;
108        }
109
110        let mut sum_r = 0u64;
111        let mut sum_g = 0u64;
112        let mut sum_b = 0u64;
113        let mut edge = 0u64;
114        let mut n = 0u64;
115        let mut y = y0;
116        while y < y1 {
117            let mut x = x0;
118            while x < x1 {
119                let (r, g, b) = Self::sample_rgb(image, x, y);
120                sum_r += r as u64;
121                sum_g += g as u64;
122                sum_b += b as u64;
123                if x + 1 < x1 {
124                    let (r2, g2, b2) = Self::sample_rgb(image, x + 1, y);
125                    let d = (r as i16 - r2 as i16).unsigned_abs() as u64
126                        + (g as i16 - g2 as i16).unsigned_abs() as u64
127                        + (b as i16 - b2 as i16).unsigned_abs() as u64;
128                    edge += d;
129                }
130                n += 1;
131                x += 2;
132            }
133            y += 2;
134        }
135        if n == 0 {
136            return None;
137        }
138        let mr = sum_r / n;
139        let mg = sum_g / n;
140        let mb = sum_b / n;
141        let mean_edge = edge / n;
142
143        let (class_iri, score) = if mean_edge > 40 {
144            (CLASS_HIGH_EDGE, (mean_edge.min(255) as f32) / 255.0)
145        } else if mr >= mg && mr >= mb && mr > 40 {
146            (CLASS_MOSTLY_RED, (mr as f32) / 255.0)
147        } else if mg >= mr && mg >= mb && mg > 40 {
148            (CLASS_MOSTLY_GREEN, (mg as f32) / 255.0)
149        } else if mb >= mr && mb >= mg && mb > 40 {
150            (CLASS_MOSTLY_BLUE, (mb as f32) / 255.0)
151        } else {
152            (CLASS_LOW_CONTRAST, 0.35)
153        };
154
155        if score < self.min_score {
156            return None;
157        }
158
159        let mut d = Detection::empty();
160        d.class_hash = q_hash(class_iri);
161        d.instance_hash = self.model_hash.wrapping_mul(0x9e37_79b9_7f4a_7c15)
162            ^ ((cx as u64) << 32)
163            ^ (cy as u64)
164            ^ (mr << 16)
165            ^ mg
166            ^ ((frame_index as u64) << 48);
167        d.score_u16 = (score.clamp(0.0, 1.0) * 65535.0) as u16;
168        d.x_min_u16 = ((x0 as f32 / w as f32) * 65535.0) as u16;
169        d.y_min_u16 = ((y0 as f32 / h as f32) * 65535.0) as u16;
170        d.x_max_u16 = ((x1 as f32 / w as f32) * 65535.0) as u16;
171        d.y_max_u16 = ((y1 as f32 / h as f32) * 65535.0) as u16;
172        d.frame_index = frame_index;
173        d.flags = Detection::FLAG_REFERENCE_BACKEND | Detection::FLAG_LOW_ASSURANCE;
174        Some(d)
175    }
176
177    /// Emit raw cell proposals (no NMS). Writes up to `out.len()` detections.
178    pub fn propose_cells(
179        &self,
180        image: ImageView<'_>,
181        frame_index: u32,
182        out: &mut [Detection],
183    ) -> Result<usize, VisionError> {
184        if !image.is_well_formed() {
185            return Err(VisionError::MalformedImage);
186        }
187        if out.is_empty() {
188            return Err(VisionError::OutputBufferTooSmall);
189        }
190        let mut n = 0usize;
191        for cy in 0..self.grid_h {
192            for cx in 0..self.grid_w {
193                if n >= out.len() || n >= MAX_DETECTIONS {
194                    return Ok(n);
195                }
196                if let Some(d) = self.score_cell(image, cx, cy, frame_index) {
197                    out[n] = d;
198                    n += 1;
199                }
200            }
201        }
202        for d in out.iter_mut().skip(n) {
203            *d = Detection::empty();
204        }
205        Ok(n)
206    }
207
208    /// Full pipeline: cell proposals → NMS → `out`.
209    /// `workspace` must hold ≥ MAX_DETECTIONS bytes (NMS flags) + room for proposal scratch
210    /// if using the in-place path; we use stack proposal buffer of MAX_DETECTIONS.
211    pub fn detect(
212        &self,
213        image: ImageView<'_>,
214        frame_index: u32,
215        out: &mut [Detection],
216        workspace: &mut [u8],
217    ) -> Result<usize, VisionError> {
218        let mut proposals = [Detection::empty(); MAX_DETECTIONS];
219        let n_prop = self.propose_cells(image, frame_index, &mut proposals)?;
220        if n_prop == 0 {
221            for d in out.iter_mut() {
222                *d = Detection::empty();
223            }
224            return Ok(0);
225        }
226        nms_class_agnostic(&proposals, n_prop, self.nms_iou, out, workspace)
227    }
228}
229
230impl VisualModel for GridMultiObjectDetector {
231    fn capabilities(&self) -> VisualCapabilities {
232        let max_cells = (self.grid_w * self.grid_h).min(MAX_DETECTIONS as u32) as u16;
233        VisualCapabilities {
234            max_detections: max_cells,
235            embed_dim: 0,
236            supports_boxes: true,
237            supports_embedding: false,
238            is_reference_backend: true,
239        }
240    }
241
242    fn infer(
243        &mut self,
244        image: ImageView<'_>,
245        detections_out: &mut [Detection],
246        embedding_out: &mut [f32],
247        workspace: &mut [u8],
248    ) -> Result<VisualOutputCounts, VisionError> {
249        let n = self.detect(image, 0, detections_out, workspace)?;
250        for e in embedding_out.iter_mut() {
251            *e = 0.0;
252        }
253        Ok(VisualOutputCounts {
254            detections: n,
255            embedding_written: 0,
256        })
257    }
258}
259
260/// Deterministic frame indices for video sampling (cold path helper).
261/// Writes every `stride`-th frame into `out`, starting at 0. Returns count written.
262/// Overflow: if more indices than `out.len()`, only first `out.len()` are kept (deterministic).
263pub fn sample_frame_indices(total_frames: u32, stride: u32, out: &mut [u32]) -> usize {
264    if total_frames == 0 || out.is_empty() {
265        return 0;
266    }
267    let step = stride.max(1);
268    let mut w = 0usize;
269    let mut i = 0u32;
270    while i < total_frames && w < out.len() {
271        out[w] = i;
272        w += 1;
273        i = i.saturating_add(step);
274    }
275    w
276}
277
278#[cfg(test)]
279mod tests {
280    use super::*;
281
282    fn checkerboard_rgb(w: u32, h: u32) -> Vec<u8> {
283        // Left half red, right half blue — two objects for multi-det.
284        let mut v = vec![0u8; (w * h * 3) as usize];
285        for y in 0..h {
286            for x in 0..w {
287                let i = ((y * w + x) * 3) as usize;
288                if x < w / 2 {
289                    v[i] = 220;
290                    v[i + 1] = 10;
291                    v[i + 2] = 10;
292                } else {
293                    v[i] = 10;
294                    v[i + 1] = 10;
295                    v[i + 2] = 220;
296                }
297            }
298        }
299        v
300    }
301
302    #[test]
303    fn multi_object_two_cells() {
304        let px = checkerboard_rgb(16, 8);
305        let img = ImageView {
306            bytes: &px,
307            width: 16,
308            height: 8,
309            row_stride: 48,
310            format: PixelFormat::Rgb8,
311        };
312        let det = GridMultiObjectDetector::new(2, 1);
313        let mut out = [Detection::empty(); 16];
314        let mut ws = [0u8; 64];
315        let n = det.detect(img, 0, &mut out, &mut ws).unwrap();
316        assert!(n >= 2, "expected multi-object, got {n}");
317        let classes: Vec<u64> = out[..n].iter().map(|d| d.class_hash).collect();
318        assert!(classes.contains(&q_hash(CLASS_MOSTLY_RED)));
319        assert!(classes.contains(&q_hash(CLASS_MOSTLY_BLUE)));
320    }
321
322    #[test]
323    fn frame_sampler_stride() {
324        let mut out = [0u32; 8];
325        let n = sample_frame_indices(10, 3, &mut out);
326        assert_eq!(n, 4);
327        assert_eq!(&out[..4], &[0, 3, 6, 9]);
328    }
329
330    #[test]
331    fn frame_sampler_overflow_deterministic() {
332        let mut out = [0u32; 2];
333        let n = sample_frame_indices(100, 1, &mut out);
334        assert_eq!(n, 2);
335        assert_eq!(out[0], 0);
336        assert_eq!(out[1], 1);
337    }
338
339    #[test]
340    fn nms_path_via_visual_model() {
341        let px = checkerboard_rgb(8, 8);
342        let img = ImageView {
343            bytes: &px,
344            width: 8,
345            height: 8,
346            row_stride: 24,
347            format: PixelFormat::Rgb8,
348        };
349        let mut det = GridMultiObjectDetector::new(4, 4);
350        let mut dets = [Detection::empty(); MAX_DETECTIONS];
351        let mut emb = [0.0f32; 4];
352        let mut ws = [0u8; MAX_DETECTIONS];
353        let c = det.infer(img, &mut dets, &mut emb, &mut ws).unwrap();
354        assert!(c.detections >= 1);
355        assert!(c.detections <= MAX_DETECTIONS);
356    }
357}