1pub mod load_litert_bytes;
13pub mod load_onnx_bytes;
14pub mod onnx_session;
15pub mod resolve_vision_asset;
16pub use load_litert_bytes::{
17 load_litert_file, validate_litert_bytes, LiteRtFileMeta, LiteRtLoadError, LITER_TFLITE_MAGIC,
18};
19pub use load_onnx_bytes::{load_onnx_file, validate_onnx_bytes, OnnxFileMeta, OnnxLoadError};
20pub use onnx_session::{
21 probe_onnx_asset, sface_infer_rgb8, yunet_infer_rgb8, OnnxSessionError, OnnxSessionInfo,
22};
23pub use resolve_vision_asset::{
24 resolve_vision_asset, AssetLicenceTag, ResolvedAsset, VisionAssetError, VisionAssetId,
25};
26
27use crate::classifier::LinearHead;
28use crate::semantic::q_hash;
29use crate::types::{
30 Detection, ImageView, PixelFormat, VisionError, VisualCapabilities, VisualModel,
31 VisualOutputCounts, MAX_DETECTIONS, MAX_EMBED_DIM,
32};
33
34pub const QVWT_MAGIC: u32 = 0x5457_5651; pub const QVWT_VERSION: u32 = 1;
37
38#[repr(u8)]
40#[derive(Debug, Clone, Copy, PartialEq, Eq)]
41pub enum VisionBackendKind {
42 Reference = 0,
44 ProductionWeights = 1,
46}
47
48#[derive(Debug, Clone)]
50pub struct VisionWeightBundle {
51 pub embed_dim: usize,
52 pub n_classes: usize,
53 pub proj: Vec<f32>,
55 pub head: LinearHead,
56 pub model_id: String,
57 pub content_hash: u64,
58}
59
60impl VisionWeightBundle {
61 pub fn model_hash(&self) -> u64 {
63 self.content_hash
64 }
65
66 pub fn from_seed(seed: u64, embed_dim: usize, class_iris: &[&str]) -> Self {
68 let dim = embed_dim.min(MAX_EMBED_DIM).max(8);
69 let n = class_iris.len().max(1);
70 let mut proj = vec![0.0f32; dim * 16];
71 let mut h = seed ^ 0xA5A5_5A5A_C3C3_3C3C;
72 for v in proj.iter_mut() {
73 h = h
74 .wrapping_mul(0x9e37_79b9_7f4a_7c15)
75 .rotate_left(13)
76 .wrapping_add(1);
77 *v = ((h as i64 as f32) / (u64::MAX as f32)) * 0.5;
78 }
79 let mut head = LinearHead::zeros(dim, class_iris);
80 for c in 0..n {
81 for i in 0..dim {
82 h = h.wrapping_mul(0x85eb_ca6b).wrapping_add(c as u64 + 1);
83 head.weight[c * dim + i] = ((h % 1000) as f32 / 1000.0) - 0.5;
84 }
85 head.bias[c] = 0.01 * (c as f32 + 1.0);
86 }
87 let content_hash = q_hash(&format!("qvwt-seed:{seed}:{dim}:{n}"));
88 Self {
89 embed_dim: dim,
90 n_classes: n,
91 proj,
92 head,
93 model_id: format!("qvwt-seed-{seed:016x}"),
94 content_hash,
95 }
96 }
97
98 pub fn to_bytes(&self) -> Vec<u8> {
100 let mut out = Vec::with_capacity(
101 64 + (self.proj.len() + self.head.weight.len() + self.head.bias.len()) * 4,
102 );
103 out.extend_from_slice(&QVWT_MAGIC.to_le_bytes());
104 out.extend_from_slice(&QVWT_VERSION.to_le_bytes());
105 out.extend_from_slice(&(self.embed_dim as u32).to_le_bytes());
106 out.extend_from_slice(&(self.n_classes as u32).to_le_bytes());
107 out.extend_from_slice(&self.content_hash.to_le_bytes());
108 while out.len() < 32 {
110 out.push(0);
111 }
112 for f in self
113 .proj
114 .iter()
115 .chain(self.head.weight.iter())
116 .chain(self.head.bias.iter())
117 {
118 out.extend_from_slice(&f.to_le_bytes());
119 }
120 for &ch in &self.head.class_hashes {
122 out.extend_from_slice(&ch.to_le_bytes());
123 }
124 out
125 }
126
127 pub fn save_path(&self, path: &std::path::Path) -> Result<(), String> {
129 if let Some(p) = path.parent() {
130 std::fs::create_dir_all(p).map_err(|e| e.to_string())?;
131 }
132 std::fs::write(path, self.to_bytes()).map_err(|e| e.to_string())
133 }
134
135 pub fn load_path(path: &std::path::Path, class_iris: &[&str]) -> Result<Self, String> {
137 let b = std::fs::read(path).map_err(|e| e.to_string())?;
138 Self::from_bytes(&b, class_iris).map_err(|e| format!("{e:?}"))
139 }
140
141 pub fn from_bytes(bytes: &[u8], class_iris: &[&str]) -> Result<Self, VisionError> {
143 if bytes.len() < 32 {
144 return Err(VisionError::MalformedImage);
145 }
146 let magic = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
147 if magic != QVWT_MAGIC {
148 return Err(VisionError::BackendUnavailable);
149 }
150 let ver = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
151 if ver != QVWT_VERSION {
152 return Err(VisionError::BackendUnavailable);
153 }
154 let embed_dim = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;
155 let n_classes = u32::from_le_bytes([bytes[12], bytes[13], bytes[14], bytes[15]]) as usize;
156 let content_hash = u64::from_le_bytes([
157 bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21], bytes[22], bytes[23],
158 ]);
159 if embed_dim == 0 || embed_dim > MAX_EMBED_DIM || n_classes == 0 || n_classes > 64 {
160 return Err(VisionError::MalformedImage);
161 }
162 let mut off = 32usize;
163 let proj_n = embed_dim * 16;
164 let w_n = n_classes * embed_dim;
165 let need = (proj_n + w_n + n_classes) * 4 + n_classes * 8;
166 if bytes.len() < off + need {
167 return Err(VisionError::OutputBufferTooSmall);
168 }
169 let mut proj = vec![0.0f32; proj_n];
170 for v in &mut proj {
171 *v = f32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]);
172 off += 4;
173 }
174 let names: Vec<&str> = if class_iris.len() >= n_classes {
175 class_iris[..n_classes].to_vec()
176 } else {
177 (0..n_classes)
179 .map(|_| "https://ns.webizen.org/q42/vision/class/loaded")
180 .collect()
181 };
182 let mut head = LinearHead::zeros(embed_dim, &names);
184 for v in &mut head.weight {
185 *v = f32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]);
186 off += 4;
187 }
188 for v in &mut head.bias {
189 *v = f32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]);
190 off += 4;
191 }
192 for c in 0..n_classes {
193 let ch = u64::from_le_bytes([
194 bytes[off],
195 bytes[off + 1],
196 bytes[off + 2],
197 bytes[off + 3],
198 bytes[off + 4],
199 bytes[off + 5],
200 bytes[off + 6],
201 bytes[off + 7],
202 ]);
203 off += 8;
204 if c < head.class_hashes.len() {
205 head.class_hashes[c] = ch;
206 }
207 }
208 Ok(Self {
209 embed_dim,
210 n_classes,
211 proj,
212 head,
213 model_id: format!("qvwt-{content_hash:016x}"),
214 content_hash,
215 })
216 }
217}
218
219pub struct ProductionVision {
221 pub bundle: VisionWeightBundle,
222 pub min_score: f32,
223 pub grid: u32,
224}
225
226impl ProductionVision {
227 pub fn new(bundle: VisionWeightBundle) -> Self {
229 Self {
230 bundle,
231 min_score: 0.2,
232 grid: 2,
233 }
234 }
235
236 pub fn backend_kind(&self) -> VisionBackendKind {
238 VisionBackendKind::ProductionWeights
239 }
240
241 fn sample_rgb(img: ImageView<'_>, x: u32, y: u32) -> (u8, u8, u8) {
242 let bpp = img.bytes_per_pixel() as usize;
243 let off = (y as usize)
244 .saturating_mul(img.row_stride as usize)
245 .saturating_add((x as usize).saturating_mul(bpp));
246 if off + bpp > img.bytes.len() {
247 return (0, 0, 0);
248 }
249 match img.format {
250 PixelFormat::Gray8 => {
251 let g = img.bytes[off];
252 (g, g, g)
253 }
254 PixelFormat::Rgb8 | PixelFormat::Rgba8 => {
255 (img.bytes[off], img.bytes[off + 1], img.bytes[off + 2])
256 }
257 PixelFormat::Bgr8 => (img.bytes[off + 2], img.bytes[off + 1], img.bytes[off]),
258 PixelFormat::RgbF32 => (0, 0, 0),
259 }
260 }
261
262 fn cell_features(img: ImageView<'_>, x0: u32, y0: u32, x1: u32, y1: u32) -> [f32; 16] {
263 let mut sum = [0.0f32; 3];
264 let mut n = 0u32;
265 let mut edge = 0.0f32;
266 let mut y = y0;
267 while y < y1 {
268 let mut x = x0;
269 while x < x1 {
270 let (r, g, b) = Self::sample_rgb(img, x, y);
271 sum[0] += r as f32 / 255.0;
272 sum[1] += g as f32 / 255.0;
273 sum[2] += b as f32 / 255.0;
274 if x + 1 < x1 {
275 let (r2, g2, b2) = Self::sample_rgb(img, x + 1, y);
276 edge += ((r as i16 - r2 as i16).unsigned_abs() as f32
277 + (g as i16 - g2 as i16).unsigned_abs() as f32
278 + (b as i16 - b2 as i16).unsigned_abs() as f32)
279 / (3.0 * 255.0);
280 }
281 n += 1;
282 x += 2;
283 }
284 y += 2;
285 }
286 let inv = if n == 0 { 0.0 } else { 1.0 / n as f32 };
287 let mut f = [0.0f32; 16];
288 f[0] = sum[0] * inv;
289 f[1] = sum[1] * inv;
290 f[2] = sum[2] * inv;
291 f[3] = edge * inv;
292 f[4] = (x1 - x0) as f32 / img.width.max(1) as f32;
293 f[5] = (y1 - y0) as f32 / img.height.max(1) as f32;
294 for i in 6..16 {
295 f[i] = f[i % 3] * f[3];
296 }
297 f
298 }
299
300 fn project(&self, feat: &[f32; 16], out: &mut [f32]) {
301 let dim = self.bundle.embed_dim.min(out.len());
302 for e in 0..dim {
303 let mut s = 0.0f32;
304 for i in 0..16 {
305 s += feat[i] * self.bundle.proj[e * 16 + i];
306 }
307 out[e] = s.tanh();
308 }
309 }
310}
311
312impl VisualModel for ProductionVision {
313 fn capabilities(&self) -> VisualCapabilities {
314 VisualCapabilities {
315 max_detections: (self.grid * self.grid).min(MAX_DETECTIONS as u32) as u16,
316 embed_dim: self.bundle.embed_dim as u16,
317 supports_boxes: true,
318 supports_embedding: true,
319 is_reference_backend: false,
320 }
321 }
322
323 fn infer(
324 &mut self,
325 image: ImageView<'_>,
326 detections_out: &mut [Detection],
327 embedding_out: &mut [f32],
328 _workspace: &mut [u8],
329 ) -> Result<VisualOutputCounts, VisionError> {
330 if !image.is_well_formed() {
331 return Err(VisionError::MalformedImage);
332 }
333 if detections_out.is_empty() {
334 return Err(VisionError::OutputBufferTooSmall);
335 }
336 let g = self.grid.max(1);
337 let w = image.width;
338 let h = image.height;
339 let mut det_n = 0usize;
340 let mut global_emb = [0.0f32; MAX_EMBED_DIM];
341 let gf = Self::cell_features(image, 0, 0, w, h);
342 self.project(&gf, &mut global_emb);
343 let emb_n = embedding_out.len().min(self.bundle.embed_dim);
344 embedding_out[..emb_n].copy_from_slice(&global_emb[..emb_n]);
345
346 for cy in 0..g {
347 for cx in 0..g {
348 if det_n >= detections_out.len() || det_n >= MAX_DETECTIONS {
349 break;
350 }
351 let x0 = cx * w / g;
352 let y0 = cy * h / g;
353 let x1 = if cx + 1 >= g { w } else { (cx + 1) * w / g };
354 let y1 = if cy + 1 >= g { h } else { (cy + 1) * h / g };
355 let feat = Self::cell_features(image, x0, y0, x1, y1);
356 let mut emb = [0.0f32; MAX_EMBED_DIM];
357 self.project(&feat, &mut emb);
358 if let Some(mut d) = self
359 .bundle
360 .head
361 .classify_embedding(&emb[..self.bundle.embed_dim], self.min_score)
362 {
363 d.x_min_u16 = ((x0 as f32 / w as f32) * 65535.0) as u16;
364 d.y_min_u16 = ((y0 as f32 / h as f32) * 65535.0) as u16;
365 d.x_max_u16 = ((x1 as f32 / w as f32) * 65535.0) as u16;
366 d.y_max_u16 = ((y1 as f32 / h as f32) * 65535.0) as u16;
367 d.instance_hash ^= self.bundle.content_hash ^ ((cx as u64) << 8) ^ cy as u64;
368 d.flags = 0; detections_out[det_n] = d;
370 det_n += 1;
371 }
372 }
373 }
374 for d in detections_out.iter_mut().skip(det_n) {
375 *d = Detection::empty();
376 }
377 Ok(VisualOutputCounts {
378 detections: det_n,
379 embedding_written: emb_n,
380 })
381 }
382}
383
384#[cfg(test)]
385mod tests {
386 use super::*;
387 use crate::detector::{CLASS_MOSTLY_BLUE, CLASS_MOSTLY_RED};
388
389 #[test]
390 fn qvwt_roundtrip() {
391 let b = VisionWeightBundle::from_seed(42, 16, &[CLASS_MOSTLY_RED, CLASS_MOSTLY_BLUE]);
392 let bytes = b.to_bytes();
393 let b2 =
394 VisionWeightBundle::from_bytes(&bytes, &[CLASS_MOSTLY_RED, CLASS_MOSTLY_BLUE]).unwrap();
395 assert_eq!(b.embed_dim, b2.embed_dim);
396 assert_eq!(b.content_hash, b2.content_hash);
397 assert_eq!(b.proj, b2.proj);
398 }
399
400 #[test]
401 fn production_not_labelled_reference() {
402 let b = VisionWeightBundle::from_seed(1, 16, &[CLASS_MOSTLY_RED, CLASS_MOSTLY_BLUE]);
403 let mut m = ProductionVision::new(b);
404 assert_eq!(m.backend_kind(), VisionBackendKind::ProductionWeights);
405 assert!(!m.capabilities().is_reference_backend);
406 let mut rgb = vec![200u8; 8 * 8 * 3];
407 for p in rgb.chunks_mut(3) {
408 p[0] = 220;
409 p[1] = 20;
410 p[2] = 20;
411 }
412 let img = ImageView {
413 bytes: &rgb,
414 width: 8,
415 height: 8,
416 row_stride: 24,
417 format: PixelFormat::Rgb8,
418 };
419 let mut dets = [Detection::empty(); 8];
420 let mut emb = [0.0f32; 16];
421 let mut ws = [0u8; 8];
422 let c = m.infer(img, &mut dets, &mut emb, &mut ws).unwrap();
423 assert!(c.embedding_written > 0);
424 }
425}