qualia_vision/
cpu_reference.rs1use crate::semantic::q_hash;
13use crate::types::{
14 Detection, ImageView, PixelFormat, VisionError, VisualCapabilities, VisualModel,
15 VisualOutputCounts, MAX_DETECTIONS, MAX_EMBED_DIM,
16};
17
18pub const CLASS_MOSTLY_RED: &str = "https://ns.webizen.org/q42/vision/class/mostly-red";
20pub const CLASS_MOSTLY_GREEN: &str = "https://ns.webizen.org/q42/vision/class/mostly-green";
21pub const CLASS_MOSTLY_BLUE: &str = "https://ns.webizen.org/q42/vision/class/mostly-blue";
22pub const CLASS_HIGH_EDGE: &str = "https://ns.webizen.org/q42/vision/class/high-edge-energy";
23pub const CLASS_LOW_CONTRAST: &str = "https://ns.webizen.org/q42/vision/class/low-contrast";
24
25const MODEL_ID: &str = "qualia-vision-cpu-reference-v1";
26
27pub struct CpuReferenceVision {
28 model_hash: u64,
29}
30
31impl Default for CpuReferenceVision {
32 fn default() -> Self {
33 Self::new()
34 }
35}
36
37impl CpuReferenceVision {
38 pub fn new() -> Self {
40 Self {
41 model_hash: q_hash(MODEL_ID),
42 }
43 }
44
45 pub fn model_hash(&self) -> u64 {
47 self.model_hash
48 }
49
50 fn sample_rgb(img: ImageView<'_>, x: u32, y: u32) -> (u8, u8, u8) {
51 let bpp = img.bytes_per_pixel() as usize;
52 let off = (y as usize)
53 .saturating_mul(img.row_stride as usize)
54 .saturating_add((x as usize).saturating_mul(bpp));
55 if off + bpp > img.bytes.len() {
56 return (0, 0, 0);
57 }
58 match img.format {
59 PixelFormat::Gray8 => {
60 let g = img.bytes[off];
61 (g, g, g)
62 }
63 PixelFormat::Rgb8 | PixelFormat::Rgba8 => {
64 (img.bytes[off], img.bytes[off + 1], img.bytes[off + 2])
65 }
66 PixelFormat::Bgr8 => (img.bytes[off + 2], img.bytes[off + 1], img.bytes[off]),
67 PixelFormat::RgbF32 => {
68 (0, 0, 0)
70 }
71 }
72 }
73}
74
75impl VisualModel for CpuReferenceVision {
76 fn capabilities(&self) -> VisualCapabilities {
77 VisualCapabilities {
78 max_detections: 4,
79 embed_dim: 16,
80 supports_boxes: true,
81 supports_embedding: true,
82 is_reference_backend: true,
83 }
84 }
85
86 fn infer(
87 &mut self,
88 image: ImageView<'_>,
89 detections_out: &mut [Detection],
90 embedding_out: &mut [f32],
91 _workspace: &mut [u8],
92 ) -> Result<VisualOutputCounts, VisionError> {
93 if !image.is_well_formed() {
94 return Err(VisionError::MalformedImage);
95 }
96 if detections_out.len() < 4 {
97 return Err(VisionError::OutputBufferTooSmall);
98 }
99
100 let w = image.width;
101 let h = image.height;
102 let mut det_n = 0usize;
103
104 for cy in 0..2u32 {
106 for cx in 0..2u32 {
107 let x0 = cx * w / 2;
108 let y0 = cy * h / 2;
109 let x1 = if cx == 1 { w } else { (cx + 1) * w / 2 };
110 let y1 = if cy == 1 { h } else { (cy + 1) * h / 2 };
111 if x1 <= x0 || y1 <= y0 {
112 continue;
113 }
114
115 let mut sum_r = 0u64;
116 let mut sum_g = 0u64;
117 let mut sum_b = 0u64;
118 let mut edge = 0u64;
119 let mut n = 0u64;
120 let mut y = y0;
122 while y < y1 {
123 let mut x = x0;
124 while x < x1 {
125 let (r, g, b) = Self::sample_rgb(image, x, y);
126 sum_r += r as u64;
127 sum_g += g as u64;
128 sum_b += b as u64;
129 if x + 1 < x1 {
130 let (r2, g2, b2) = Self::sample_rgb(image, x + 1, y);
131 let d = (r as i16 - r2 as i16).unsigned_abs() as u64
132 + (g as i16 - g2 as i16).unsigned_abs() as u64
133 + (b as i16 - b2 as i16).unsigned_abs() as u64;
134 edge += d;
135 }
136 n += 1;
137 x += 2;
138 }
139 y += 2;
140 }
141 if n == 0 {
142 continue;
143 }
144 let mr = sum_r / n;
145 let mg = sum_g / n;
146 let mb = sum_b / n;
147 let mean_edge = edge / n;
148
149 let (class_iri, score) = if mean_edge > 40 {
150 (CLASS_HIGH_EDGE, (mean_edge.min(255) as f32) / 255.0)
151 } else if mr >= mg && mr >= mb && mr > 40 {
152 (CLASS_MOSTLY_RED, (mr as f32) / 255.0)
153 } else if mg >= mr && mg >= mb && mg > 40 {
154 (CLASS_MOSTLY_GREEN, (mg as f32) / 255.0)
155 } else if mb >= mr && mb >= mg && mb > 40 {
156 (CLASS_MOSTLY_BLUE, (mb as f32) / 255.0)
157 } else {
158 (CLASS_LOW_CONTRAST, 0.35)
159 };
160
161 let mut d = Detection::empty();
162 d.class_hash = q_hash(class_iri);
163 d.instance_hash = self.model_hash.wrapping_mul(0x9e37_79b9_7f4a_7c15)
164 ^ ((cx as u64) << 32)
165 ^ (cy as u64)
166 ^ (mr << 16)
167 ^ mg;
168 d.score_u16 = (score.clamp(0.0, 1.0) * 65535.0) as u16;
169 d.x_min_u16 = ((x0 as f32 / w as f32) * 65535.0) as u16;
171 d.y_min_u16 = ((y0 as f32 / h as f32) * 65535.0) as u16;
172 d.x_max_u16 = ((x1 as f32 / w as f32) * 65535.0) as u16;
173 d.y_max_u16 = ((y1 as f32 / h as f32) * 65535.0) as u16;
174 d.flags = Detection::FLAG_REFERENCE_BACKEND | Detection::FLAG_LOW_ASSURANCE;
175 detections_out[det_n] = d;
176 det_n += 1;
177 if det_n >= MAX_DETECTIONS.min(4) {
178 break;
179 }
180 }
181 if det_n >= 4 {
182 break;
183 }
184 }
185
186 let emb_n = embedding_out.len().min(MAX_EMBED_DIM).min(16);
188 if emb_n > 0 {
189 let (r, g, b) = Self::sample_rgb(image, w / 2, h / 2);
190 embedding_out[..emb_n].fill(0.0);
191 if emb_n > 0 {
192 embedding_out[0] = r as f32 / 255.0;
193 }
194 if emb_n > 1 {
195 embedding_out[1] = g as f32 / 255.0;
196 }
197 if emb_n > 2 {
198 embedding_out[2] = b as f32 / 255.0;
199 }
200 if emb_n > 3 {
201 embedding_out[3] = w as f32 / (h.max(1) as f32);
202 }
203 for i in 0..det_n.min(emb_n.saturating_sub(4)) {
204 embedding_out[4 + i] = detections_out[i].score_f32();
205 }
206 }
207
208 for d in detections_out.iter_mut().skip(det_n) {
210 *d = Detection::empty();
211 }
212
213 Ok(VisualOutputCounts {
214 detections: det_n,
215 embedding_written: emb_n,
216 })
217 }
218}
219
220#[cfg(test)]
221mod tests {
222 use super::*;
223 use crate::semantic::{compile_observation_quins, media_digest};
224
225 #[test]
226 fn fixture_solid_red_loads() {
227 let px = include_bytes!("../fixtures/solid_red_8x8.rgb");
228 assert_eq!(px.len(), 8 * 8 * 3);
229 let img = ImageView {
230 bytes: px,
231 width: 8,
232 height: 8,
233 row_stride: 24,
234 format: PixelFormat::Rgb8,
235 };
236 let mut model = CpuReferenceVision::new();
237 let mut dets = [Detection::empty(); 8];
238 let mut emb = [0.0f32; 16];
239 let mut ws = [0u8; 64];
240 let counts = model.infer(img, &mut dets, &mut emb, &mut ws).unwrap();
241 assert!(counts.detections >= 1);
242 assert_eq!(dets[0].class_hash, q_hash(CLASS_MOSTLY_RED));
244 }
245
246 #[test]
247 fn red_cell_emits_detection() {
248 let mut px = vec![0u8; 4 * 4 * 3];
250 for p in px.chunks_mut(3) {
251 p[0] = 220;
252 p[1] = 10;
253 p[2] = 10;
254 }
255 let img = ImageView {
256 bytes: &px,
257 width: 4,
258 height: 4,
259 row_stride: 12,
260 format: PixelFormat::Rgb8,
261 };
262 let mut model = CpuReferenceVision::new();
263 let mut dets = [Detection::empty(); 8];
264 let mut emb = [0.0f32; 16];
265 let mut ws = [0u8; 64];
266 let counts = model
267 .infer(img, &mut dets, &mut emb, &mut ws)
268 .expect("infer");
269 assert!(counts.detections >= 1);
270 assert!(dets[0].score_u16 > 0);
271 assert!(model.capabilities().is_reference_backend);
272
273 let digest = media_digest(&px);
274 let mut quins = [crate::semantic::VisionQuin::with_parity(0, 0, 0, 0, 0); 16];
275 let n = compile_observation_quins(
276 digest,
277 &dets[..counts.detections],
278 model.model_hash(),
279 &mut quins,
280 );
281 assert!(n >= 2);
282 }
283}