Skip to main content

qualia_vision/weights/
mod.rs

1//! Swarm W — production weight bundle + encoder/detector backend.
2//!
3//! Native **QVWT** format (Qualia Vision Weights): mmap-friendly flat f32 tables.
4//! Offline conversion from GGUF/P64 may write this format; no Python at runtime.
5//!
6//! Honest labels: `VisionBackendKind::ProductionWeights` only when a bundle is loaded.
7//! Seed-built fixtures are regenerable test weights, not a third-party foundation model.
8//!
9//! Vendor pack (MIT/Apache ONNX/TFLite): see [`resolve_vision_asset()`] — **PermissiveReady**,
10//! never commercial-licence gated. States: WeightAbsent / AdapterMissing.
11
12pub mod load_litert_bytes;
13pub mod load_onnx_bytes;
14pub mod onnx_session;
15pub mod resolve_vision_asset;
16pub use load_litert_bytes::{
17    load_litert_file, validate_litert_bytes, LiteRtFileMeta, LiteRtLoadError, LITER_TFLITE_MAGIC,
18};
19pub use load_onnx_bytes::{load_onnx_file, validate_onnx_bytes, OnnxFileMeta, OnnxLoadError};
20pub use onnx_session::{
21    probe_onnx_asset, sface_infer_rgb8, yunet_infer_rgb8, OnnxSessionError, OnnxSessionInfo,
22};
23pub use resolve_vision_asset::{
24    resolve_vision_asset, AssetLicenceTag, ResolvedAsset, VisionAssetError, VisionAssetId,
25};
26
27use crate::classifier::LinearHead;
28use crate::semantic::q_hash;
29use crate::types::{
30    Detection, ImageView, PixelFormat, VisionError, VisualCapabilities, VisualModel,
31    VisualOutputCounts, MAX_DETECTIONS, MAX_EMBED_DIM,
32};
33
34/// File magic: `QVWT` little-endian.
35pub const QVWT_MAGIC: u32 = 0x5457_5651; // 'QVWT' as LE bytes Q V W T
36pub const QVWT_VERSION: u32 = 1;
37
38/// Which detector path is active (product must label this).
39#[repr(u8)]
40#[derive(Debug, Clone, Copy, PartialEq, Eq)]
41pub enum VisionBackendKind {
42    /// Heuristic grid / CPU reference — not production eval.
43    Reference = 0,
44    /// Loaded QVWT (or equivalent) weight tables.
45    ProductionWeights = 1,
46}
47
48/// On-disk / in-memory vision weight bundle.
49#[derive(Debug, Clone)]
50pub struct VisionWeightBundle {
51    pub embed_dim: usize,
52    pub n_classes: usize,
53    /// Projects 16-d image features → embed_dim. Row-major `[embed_dim * 16]`.
54    pub proj: Vec<f32>,
55    pub head: LinearHead,
56    pub model_id: String,
57    pub content_hash: u64,
58}
59
60impl VisionWeightBundle {
61    /// `model` hash.
62    pub fn model_hash(&self) -> u64 {
63        self.content_hash
64    }
65
66    /// Regenerable fixture weights from seed (tests + offline demo without external files).
67    pub fn from_seed(seed: u64, embed_dim: usize, class_iris: &[&str]) -> Self {
68        let dim = embed_dim.min(MAX_EMBED_DIM).max(8);
69        let n = class_iris.len().max(1);
70        let mut proj = vec![0.0f32; dim * 16];
71        let mut h = seed ^ 0xA5A5_5A5A_C3C3_3C3C;
72        for v in proj.iter_mut() {
73            h = h
74                .wrapping_mul(0x9e37_79b9_7f4a_7c15)
75                .rotate_left(13)
76                .wrapping_add(1);
77            *v = ((h as i64 as f32) / (u64::MAX as f32)) * 0.5;
78        }
79        let mut head = LinearHead::zeros(dim, class_iris);
80        for c in 0..n {
81            for i in 0..dim {
82                h = h.wrapping_mul(0x85eb_ca6b).wrapping_add(c as u64 + 1);
83                head.weight[c * dim + i] = ((h % 1000) as f32 / 1000.0) - 0.5;
84            }
85            head.bias[c] = 0.01 * (c as f32 + 1.0);
86        }
87        let content_hash = q_hash(&format!("qvwt-seed:{seed}:{dim}:{n}"));
88        Self {
89            embed_dim: dim,
90            n_classes: n,
91            proj,
92            head,
93            model_id: format!("qvwt-seed-{seed:016x}"),
94            content_hash,
95        }
96    }
97
98    /// Serialize to QVWT bytes (cold path).
99    pub fn to_bytes(&self) -> Vec<u8> {
100        let mut out = Vec::with_capacity(
101            64 + (self.proj.len() + self.head.weight.len() + self.head.bias.len()) * 4,
102        );
103        out.extend_from_slice(&QVWT_MAGIC.to_le_bytes());
104        out.extend_from_slice(&QVWT_VERSION.to_le_bytes());
105        out.extend_from_slice(&(self.embed_dim as u32).to_le_bytes());
106        out.extend_from_slice(&(self.n_classes as u32).to_le_bytes());
107        out.extend_from_slice(&self.content_hash.to_le_bytes());
108        // pad to 32
109        while out.len() < 32 {
110            out.push(0);
111        }
112        for f in self
113            .proj
114            .iter()
115            .chain(self.head.weight.iter())
116            .chain(self.head.bias.iter())
117        {
118            out.extend_from_slice(&f.to_le_bytes());
119        }
120        // class hashes
121        for &ch in &self.head.class_hashes {
122            out.extend_from_slice(&ch.to_le_bytes());
123        }
124        out
125    }
126
127    /// Saves path.
128    pub fn save_path(&self, path: &std::path::Path) -> Result<(), String> {
129        if let Some(p) = path.parent() {
130            std::fs::create_dir_all(p).map_err(|e| e.to_string())?;
131        }
132        std::fs::write(path, self.to_bytes()).map_err(|e| e.to_string())
133    }
134
135    /// Loads path.
136    pub fn load_path(path: &std::path::Path, class_iris: &[&str]) -> Result<Self, String> {
137        let b = std::fs::read(path).map_err(|e| e.to_string())?;
138        Self::from_bytes(&b, class_iris).map_err(|e| format!("{e:?}"))
139    }
140
141    /// Load QVWT from bytes. Class IRIs optional: if fewer names than n_classes, hashes from file used.
142    pub fn from_bytes(bytes: &[u8], class_iris: &[&str]) -> Result<Self, VisionError> {
143        if bytes.len() < 32 {
144            return Err(VisionError::MalformedImage);
145        }
146        let magic = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
147        if magic != QVWT_MAGIC {
148            return Err(VisionError::BackendUnavailable);
149        }
150        let ver = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
151        if ver != QVWT_VERSION {
152            return Err(VisionError::BackendUnavailable);
153        }
154        let embed_dim = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;
155        let n_classes = u32::from_le_bytes([bytes[12], bytes[13], bytes[14], bytes[15]]) as usize;
156        let content_hash = u64::from_le_bytes([
157            bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21], bytes[22], bytes[23],
158        ]);
159        if embed_dim == 0 || embed_dim > MAX_EMBED_DIM || n_classes == 0 || n_classes > 64 {
160            return Err(VisionError::MalformedImage);
161        }
162        let mut off = 32usize;
163        let proj_n = embed_dim * 16;
164        let w_n = n_classes * embed_dim;
165        let need = (proj_n + w_n + n_classes) * 4 + n_classes * 8;
166        if bytes.len() < off + need {
167            return Err(VisionError::OutputBufferTooSmall);
168        }
169        let mut proj = vec![0.0f32; proj_n];
170        for v in &mut proj {
171            *v = f32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]);
172            off += 4;
173        }
174        let names: Vec<&str> = if class_iris.len() >= n_classes {
175            class_iris[..n_classes].to_vec()
176        } else {
177            // placeholder iris; hashes overwritten from file
178            (0..n_classes)
179                .map(|_| "https://ns.webizen.org/q42/vision/class/loaded")
180                .collect()
181        };
182        // Build head then fill
183        let mut head = LinearHead::zeros(embed_dim, &names);
184        for v in &mut head.weight {
185            *v = f32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]);
186            off += 4;
187        }
188        for v in &mut head.bias {
189            *v = f32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]);
190            off += 4;
191        }
192        for c in 0..n_classes {
193            let ch = u64::from_le_bytes([
194                bytes[off],
195                bytes[off + 1],
196                bytes[off + 2],
197                bytes[off + 3],
198                bytes[off + 4],
199                bytes[off + 5],
200                bytes[off + 6],
201                bytes[off + 7],
202            ]);
203            off += 8;
204            if c < head.class_hashes.len() {
205                head.class_hashes[c] = ch;
206            }
207        }
208        Ok(Self {
209            embed_dim,
210            n_classes,
211            proj,
212            head,
213            model_id: format!("qvwt-{content_hash:016x}"),
214            content_hash,
215        })
216    }
217}
218
219/// Production path: image features → projection → linear head → whole-image or grid cells.
220pub struct ProductionVision {
221    pub bundle: VisionWeightBundle,
222    pub min_score: f32,
223    pub grid: u32,
224}
225
226impl ProductionVision {
227    /// Creates a new instance.
228    pub fn new(bundle: VisionWeightBundle) -> Self {
229        Self {
230            bundle,
231            min_score: 0.2,
232            grid: 2,
233        }
234    }
235
236    /// `backend` kind.
237    pub fn backend_kind(&self) -> VisionBackendKind {
238        VisionBackendKind::ProductionWeights
239    }
240
241    fn sample_rgb(img: ImageView<'_>, x: u32, y: u32) -> (u8, u8, u8) {
242        let bpp = img.bytes_per_pixel() as usize;
243        let off = (y as usize)
244            .saturating_mul(img.row_stride as usize)
245            .saturating_add((x as usize).saturating_mul(bpp));
246        if off + bpp > img.bytes.len() {
247            return (0, 0, 0);
248        }
249        match img.format {
250            PixelFormat::Gray8 => {
251                let g = img.bytes[off];
252                (g, g, g)
253            }
254            PixelFormat::Rgb8 | PixelFormat::Rgba8 => {
255                (img.bytes[off], img.bytes[off + 1], img.bytes[off + 2])
256            }
257            PixelFormat::Bgr8 => (img.bytes[off + 2], img.bytes[off + 1], img.bytes[off]),
258            PixelFormat::RgbF32 => (0, 0, 0),
259        }
260    }
261
262    fn cell_features(img: ImageView<'_>, x0: u32, y0: u32, x1: u32, y1: u32) -> [f32; 16] {
263        let mut sum = [0.0f32; 3];
264        let mut n = 0u32;
265        let mut edge = 0.0f32;
266        let mut y = y0;
267        while y < y1 {
268            let mut x = x0;
269            while x < x1 {
270                let (r, g, b) = Self::sample_rgb(img, x, y);
271                sum[0] += r as f32 / 255.0;
272                sum[1] += g as f32 / 255.0;
273                sum[2] += b as f32 / 255.0;
274                if x + 1 < x1 {
275                    let (r2, g2, b2) = Self::sample_rgb(img, x + 1, y);
276                    edge += ((r as i16 - r2 as i16).unsigned_abs() as f32
277                        + (g as i16 - g2 as i16).unsigned_abs() as f32
278                        + (b as i16 - b2 as i16).unsigned_abs() as f32)
279                        / (3.0 * 255.0);
280                }
281                n += 1;
282                x += 2;
283            }
284            y += 2;
285        }
286        let inv = if n == 0 { 0.0 } else { 1.0 / n as f32 };
287        let mut f = [0.0f32; 16];
288        f[0] = sum[0] * inv;
289        f[1] = sum[1] * inv;
290        f[2] = sum[2] * inv;
291        f[3] = edge * inv;
292        f[4] = (x1 - x0) as f32 / img.width.max(1) as f32;
293        f[5] = (y1 - y0) as f32 / img.height.max(1) as f32;
294        for i in 6..16 {
295            f[i] = f[i % 3] * f[3];
296        }
297        f
298    }
299
300    fn project(&self, feat: &[f32; 16], out: &mut [f32]) {
301        let dim = self.bundle.embed_dim.min(out.len());
302        for e in 0..dim {
303            let mut s = 0.0f32;
304            for i in 0..16 {
305                s += feat[i] * self.bundle.proj[e * 16 + i];
306            }
307            out[e] = s.tanh();
308        }
309    }
310}
311
312impl VisualModel for ProductionVision {
313    fn capabilities(&self) -> VisualCapabilities {
314        VisualCapabilities {
315            max_detections: (self.grid * self.grid).min(MAX_DETECTIONS as u32) as u16,
316            embed_dim: self.bundle.embed_dim as u16,
317            supports_boxes: true,
318            supports_embedding: true,
319            is_reference_backend: false,
320        }
321    }
322
323    fn infer(
324        &mut self,
325        image: ImageView<'_>,
326        detections_out: &mut [Detection],
327        embedding_out: &mut [f32],
328        _workspace: &mut [u8],
329    ) -> Result<VisualOutputCounts, VisionError> {
330        if !image.is_well_formed() {
331            return Err(VisionError::MalformedImage);
332        }
333        if detections_out.is_empty() {
334            return Err(VisionError::OutputBufferTooSmall);
335        }
336        let g = self.grid.max(1);
337        let w = image.width;
338        let h = image.height;
339        let mut det_n = 0usize;
340        let mut global_emb = [0.0f32; MAX_EMBED_DIM];
341        let gf = Self::cell_features(image, 0, 0, w, h);
342        self.project(&gf, &mut global_emb);
343        let emb_n = embedding_out.len().min(self.bundle.embed_dim);
344        embedding_out[..emb_n].copy_from_slice(&global_emb[..emb_n]);
345
346        for cy in 0..g {
347            for cx in 0..g {
348                if det_n >= detections_out.len() || det_n >= MAX_DETECTIONS {
349                    break;
350                }
351                let x0 = cx * w / g;
352                let y0 = cy * h / g;
353                let x1 = if cx + 1 >= g { w } else { (cx + 1) * w / g };
354                let y1 = if cy + 1 >= g { h } else { (cy + 1) * h / g };
355                let feat = Self::cell_features(image, x0, y0, x1, y1);
356                let mut emb = [0.0f32; MAX_EMBED_DIM];
357                self.project(&feat, &mut emb);
358                if let Some(mut d) = self
359                    .bundle
360                    .head
361                    .classify_embedding(&emb[..self.bundle.embed_dim], self.min_score)
362                {
363                    d.x_min_u16 = ((x0 as f32 / w as f32) * 65535.0) as u16;
364                    d.y_min_u16 = ((y0 as f32 / h as f32) * 65535.0) as u16;
365                    d.x_max_u16 = ((x1 as f32 / w as f32) * 65535.0) as u16;
366                    d.y_max_u16 = ((y1 as f32 / h as f32) * 65535.0) as u16;
367                    d.instance_hash ^= self.bundle.content_hash ^ ((cx as u64) << 8) ^ cy as u64;
368                    d.flags = 0; // production path — not FLAG_REFERENCE_BACKEND
369                    detections_out[det_n] = d;
370                    det_n += 1;
371                }
372            }
373        }
374        for d in detections_out.iter_mut().skip(det_n) {
375            *d = Detection::empty();
376        }
377        Ok(VisualOutputCounts {
378            detections: det_n,
379            embedding_written: emb_n,
380        })
381    }
382}
383
384#[cfg(test)]
385mod tests {
386    use super::*;
387    use crate::detector::{CLASS_MOSTLY_BLUE, CLASS_MOSTLY_RED};
388
389    #[test]
390    fn qvwt_roundtrip() {
391        let b = VisionWeightBundle::from_seed(42, 16, &[CLASS_MOSTLY_RED, CLASS_MOSTLY_BLUE]);
392        let bytes = b.to_bytes();
393        let b2 =
394            VisionWeightBundle::from_bytes(&bytes, &[CLASS_MOSTLY_RED, CLASS_MOSTLY_BLUE]).unwrap();
395        assert_eq!(b.embed_dim, b2.embed_dim);
396        assert_eq!(b.content_hash, b2.content_hash);
397        assert_eq!(b.proj, b2.proj);
398    }
399
400    #[test]
401    fn production_not_labelled_reference() {
402        let b = VisionWeightBundle::from_seed(1, 16, &[CLASS_MOSTLY_RED, CLASS_MOSTLY_BLUE]);
403        let mut m = ProductionVision::new(b);
404        assert_eq!(m.backend_kind(), VisionBackendKind::ProductionWeights);
405        assert!(!m.capabilities().is_reference_backend);
406        let mut rgb = vec![200u8; 8 * 8 * 3];
407        for p in rgb.chunks_mut(3) {
408            p[0] = 220;
409            p[1] = 20;
410            p[2] = 20;
411        }
412        let img = ImageView {
413            bytes: &rgb,
414            width: 8,
415            height: 8,
416            row_stride: 24,
417            format: PixelFormat::Rgb8,
418        };
419        let mut dets = [Detection::empty(); 8];
420        let mut emb = [0.0f32; 16];
421        let mut ws = [0u8; 8];
422        let c = m.infer(img, &mut dets, &mut emb, &mut ws).unwrap();
423        assert!(c.embedding_written > 0);
424    }
425}