1use crate::preprocess::nms_class_agnostic;
10use crate::semantic::q_hash;
11use crate::types::{
12 Detection, ImageView, PixelFormat, VisionError, VisualCapabilities, VisualModel,
13 VisualOutputCounts, MAX_DETECTIONS,
14};
15
16pub const CLASS_MOSTLY_RED: &str = "https://ns.webizen.org/q42/vision/class/mostly-red";
18pub const CLASS_MOSTLY_GREEN: &str = "https://ns.webizen.org/q42/vision/class/mostly-green";
19pub const CLASS_MOSTLY_BLUE: &str = "https://ns.webizen.org/q42/vision/class/mostly-blue";
20pub const CLASS_HIGH_EDGE: &str = "https://ns.webizen.org/q42/vision/class/high-edge-energy";
21pub const CLASS_LOW_CONTRAST: &str = "https://ns.webizen.org/q42/vision/class/low-contrast";
22
23const MODEL_ID: &str = "qualia-vision-grid-detector-v1";
24
25pub const MAX_GRID: u32 = 8;
27
28#[derive(Debug, Clone)]
30pub struct GridMultiObjectDetector {
31 pub grid_w: u32,
32 pub grid_h: u32,
33 pub min_score: f32,
34 pub nms_iou: f32,
35 model_hash: u64,
36}
37
38impl Default for GridMultiObjectDetector {
39 fn default() -> Self {
40 Self::new(2, 2)
41 }
42}
43
44impl GridMultiObjectDetector {
45 pub fn new(grid_w: u32, grid_h: u32) -> Self {
47 let gw = grid_w.clamp(1, MAX_GRID);
48 let gh = grid_h.clamp(1, MAX_GRID);
49 Self {
50 grid_w: gw,
51 grid_h: gh,
52 min_score: 0.15,
53 nms_iou: 0.45,
54 model_hash: q_hash(MODEL_ID) ^ ((gw as u64) << 8) ^ (gh as u64),
55 }
56 }
57
58 pub fn model_hash(&self) -> u64 {
60 self.model_hash
61 }
62
63 fn sample_rgb(img: ImageView<'_>, x: u32, y: u32) -> (u8, u8, u8) {
64 let bpp = img.bytes_per_pixel() as usize;
65 let off = (y as usize)
66 .saturating_mul(img.row_stride as usize)
67 .saturating_add((x as usize).saturating_mul(bpp));
68 if off + bpp > img.bytes.len() {
69 return (0, 0, 0);
70 }
71 match img.format {
72 PixelFormat::Gray8 => {
73 let g = img.bytes[off];
74 (g, g, g)
75 }
76 PixelFormat::Rgb8 | PixelFormat::Rgba8 => {
77 (img.bytes[off], img.bytes[off + 1], img.bytes[off + 2])
78 }
79 PixelFormat::Bgr8 => (img.bytes[off + 2], img.bytes[off + 1], img.bytes[off]),
80 PixelFormat::RgbF32 => (0, 0, 0),
81 }
82 }
83
84 fn score_cell(
86 &self,
87 image: ImageView<'_>,
88 cx: u32,
89 cy: u32,
90 frame_index: u32,
91 ) -> Option<Detection> {
92 let w = image.width;
93 let h = image.height;
94 let x0 = cx * w / self.grid_w;
95 let y0 = cy * h / self.grid_h;
96 let x1 = if cx + 1 >= self.grid_w {
97 w
98 } else {
99 (cx + 1) * w / self.grid_w
100 };
101 let y1 = if cy + 1 >= self.grid_h {
102 h
103 } else {
104 (cy + 1) * h / self.grid_h
105 };
106 if x1 <= x0 || y1 <= y0 {
107 return None;
108 }
109
110 let mut sum_r = 0u64;
111 let mut sum_g = 0u64;
112 let mut sum_b = 0u64;
113 let mut edge = 0u64;
114 let mut n = 0u64;
115 let mut y = y0;
116 while y < y1 {
117 let mut x = x0;
118 while x < x1 {
119 let (r, g, b) = Self::sample_rgb(image, x, y);
120 sum_r += r as u64;
121 sum_g += g as u64;
122 sum_b += b as u64;
123 if x + 1 < x1 {
124 let (r2, g2, b2) = Self::sample_rgb(image, x + 1, y);
125 let d = (r as i16 - r2 as i16).unsigned_abs() as u64
126 + (g as i16 - g2 as i16).unsigned_abs() as u64
127 + (b as i16 - b2 as i16).unsigned_abs() as u64;
128 edge += d;
129 }
130 n += 1;
131 x += 2;
132 }
133 y += 2;
134 }
135 if n == 0 {
136 return None;
137 }
138 let mr = sum_r / n;
139 let mg = sum_g / n;
140 let mb = sum_b / n;
141 let mean_edge = edge / n;
142
143 let (class_iri, score) = if mean_edge > 40 {
144 (CLASS_HIGH_EDGE, (mean_edge.min(255) as f32) / 255.0)
145 } else if mr >= mg && mr >= mb && mr > 40 {
146 (CLASS_MOSTLY_RED, (mr as f32) / 255.0)
147 } else if mg >= mr && mg >= mb && mg > 40 {
148 (CLASS_MOSTLY_GREEN, (mg as f32) / 255.0)
149 } else if mb >= mr && mb >= mg && mb > 40 {
150 (CLASS_MOSTLY_BLUE, (mb as f32) / 255.0)
151 } else {
152 (CLASS_LOW_CONTRAST, 0.35)
153 };
154
155 if score < self.min_score {
156 return None;
157 }
158
159 let mut d = Detection::empty();
160 d.class_hash = q_hash(class_iri);
161 d.instance_hash = self.model_hash.wrapping_mul(0x9e37_79b9_7f4a_7c15)
162 ^ ((cx as u64) << 32)
163 ^ (cy as u64)
164 ^ (mr << 16)
165 ^ mg
166 ^ ((frame_index as u64) << 48);
167 d.score_u16 = (score.clamp(0.0, 1.0) * 65535.0) as u16;
168 d.x_min_u16 = ((x0 as f32 / w as f32) * 65535.0) as u16;
169 d.y_min_u16 = ((y0 as f32 / h as f32) * 65535.0) as u16;
170 d.x_max_u16 = ((x1 as f32 / w as f32) * 65535.0) as u16;
171 d.y_max_u16 = ((y1 as f32 / h as f32) * 65535.0) as u16;
172 d.frame_index = frame_index;
173 d.flags = Detection::FLAG_REFERENCE_BACKEND | Detection::FLAG_LOW_ASSURANCE;
174 Some(d)
175 }
176
177 pub fn propose_cells(
179 &self,
180 image: ImageView<'_>,
181 frame_index: u32,
182 out: &mut [Detection],
183 ) -> Result<usize, VisionError> {
184 if !image.is_well_formed() {
185 return Err(VisionError::MalformedImage);
186 }
187 if out.is_empty() {
188 return Err(VisionError::OutputBufferTooSmall);
189 }
190 let mut n = 0usize;
191 for cy in 0..self.grid_h {
192 for cx in 0..self.grid_w {
193 if n >= out.len() || n >= MAX_DETECTIONS {
194 return Ok(n);
195 }
196 if let Some(d) = self.score_cell(image, cx, cy, frame_index) {
197 out[n] = d;
198 n += 1;
199 }
200 }
201 }
202 for d in out.iter_mut().skip(n) {
203 *d = Detection::empty();
204 }
205 Ok(n)
206 }
207
208 pub fn detect(
212 &self,
213 image: ImageView<'_>,
214 frame_index: u32,
215 out: &mut [Detection],
216 workspace: &mut [u8],
217 ) -> Result<usize, VisionError> {
218 let mut proposals = [Detection::empty(); MAX_DETECTIONS];
219 let n_prop = self.propose_cells(image, frame_index, &mut proposals)?;
220 if n_prop == 0 {
221 for d in out.iter_mut() {
222 *d = Detection::empty();
223 }
224 return Ok(0);
225 }
226 nms_class_agnostic(&proposals, n_prop, self.nms_iou, out, workspace)
227 }
228}
229
230impl VisualModel for GridMultiObjectDetector {
231 fn capabilities(&self) -> VisualCapabilities {
232 let max_cells = (self.grid_w * self.grid_h).min(MAX_DETECTIONS as u32) as u16;
233 VisualCapabilities {
234 max_detections: max_cells,
235 embed_dim: 0,
236 supports_boxes: true,
237 supports_embedding: false,
238 is_reference_backend: true,
239 }
240 }
241
242 fn infer(
243 &mut self,
244 image: ImageView<'_>,
245 detections_out: &mut [Detection],
246 embedding_out: &mut [f32],
247 workspace: &mut [u8],
248 ) -> Result<VisualOutputCounts, VisionError> {
249 let n = self.detect(image, 0, detections_out, workspace)?;
250 for e in embedding_out.iter_mut() {
251 *e = 0.0;
252 }
253 Ok(VisualOutputCounts {
254 detections: n,
255 embedding_written: 0,
256 })
257 }
258}
259
260pub fn sample_frame_indices(total_frames: u32, stride: u32, out: &mut [u32]) -> usize {
264 if total_frames == 0 || out.is_empty() {
265 return 0;
266 }
267 let step = stride.max(1);
268 let mut w = 0usize;
269 let mut i = 0u32;
270 while i < total_frames && w < out.len() {
271 out[w] = i;
272 w += 1;
273 i = i.saturating_add(step);
274 }
275 w
276}
277
278#[cfg(test)]
279mod tests {
280 use super::*;
281
282 fn checkerboard_rgb(w: u32, h: u32) -> Vec<u8> {
283 let mut v = vec![0u8; (w * h * 3) as usize];
285 for y in 0..h {
286 for x in 0..w {
287 let i = ((y * w + x) * 3) as usize;
288 if x < w / 2 {
289 v[i] = 220;
290 v[i + 1] = 10;
291 v[i + 2] = 10;
292 } else {
293 v[i] = 10;
294 v[i + 1] = 10;
295 v[i + 2] = 220;
296 }
297 }
298 }
299 v
300 }
301
302 #[test]
303 fn multi_object_two_cells() {
304 let px = checkerboard_rgb(16, 8);
305 let img = ImageView {
306 bytes: &px,
307 width: 16,
308 height: 8,
309 row_stride: 48,
310 format: PixelFormat::Rgb8,
311 };
312 let det = GridMultiObjectDetector::new(2, 1);
313 let mut out = [Detection::empty(); 16];
314 let mut ws = [0u8; 64];
315 let n = det.detect(img, 0, &mut out, &mut ws).unwrap();
316 assert!(n >= 2, "expected multi-object, got {n}");
317 let classes: Vec<u64> = out[..n].iter().map(|d| d.class_hash).collect();
318 assert!(classes.contains(&q_hash(CLASS_MOSTLY_RED)));
319 assert!(classes.contains(&q_hash(CLASS_MOSTLY_BLUE)));
320 }
321
322 #[test]
323 fn frame_sampler_stride() {
324 let mut out = [0u32; 8];
325 let n = sample_frame_indices(10, 3, &mut out);
326 assert_eq!(n, 4);
327 assert_eq!(&out[..4], &[0, 3, 6, 9]);
328 }
329
330 #[test]
331 fn frame_sampler_overflow_deterministic() {
332 let mut out = [0u32; 2];
333 let n = sample_frame_indices(100, 1, &mut out);
334 assert_eq!(n, 2);
335 assert_eq!(out[0], 0);
336 assert_eq!(out[1], 1);
337 }
338
339 #[test]
340 fn nms_path_via_visual_model() {
341 let px = checkerboard_rgb(8, 8);
342 let img = ImageView {
343 bytes: &px,
344 width: 8,
345 height: 8,
346 row_stride: 24,
347 format: PixelFormat::Rgb8,
348 };
349 let mut det = GridMultiObjectDetector::new(4, 4);
350 let mut dets = [Detection::empty(); MAX_DETECTIONS];
351 let mut emb = [0.0f32; 4];
352 let mut ws = [0u8; MAX_DETECTIONS];
353 let c = det.infer(img, &mut dets, &mut emb, &mut ws).unwrap();
354 assert!(c.detections >= 1);
355 assert!(c.detections <= MAX_DETECTIONS);
356 }
357}