Skip to main content

qualia_vision/
classifier.rs

1//! Phase 4 / V4 — frozen embedding + compact linear classifier head.
2//!
3//! Uses the existing CPU-reference embedding (or any `VisualModel` embed) and a
4//! caller-supplied weight matrix for a closed vocabulary. No Python, no deep
5//! training loop — head can be filled offline or by simple accumulators.
6
7use crate::semantic::q_hash;
8use crate::types::{
9    Detection, ImageView, VisionError, VisualCapabilities, VisualModel, VisualOutputCounts,
10    MAX_DETECTIONS,
11};
12
13/// Compact linear head: `logits[c] = bias[c] + sum_i embed[i] * weight[c * dim + i]`.
14#[derive(Debug, Clone)]
15pub struct LinearHead {
16    pub dim: usize,
17    pub n_classes: usize,
18    /// Row-major `[n_classes * dim]`.
19    pub weight: Vec<f32>,
20    pub bias: Vec<f32>,
21    /// Class IRI hashes parallel to rows.
22    pub class_hashes: Vec<u64>,
23}
24
25impl LinearHead {
26    /// Build a zero head (always abstains until weights set).
27    pub fn zeros(dim: usize, class_iris: &[&str]) -> Self {
28        let n = class_iris.len();
29        Self {
30            dim,
31            n_classes: n,
32            weight: vec![0.0; n * dim],
33            bias: vec![0.0; n],
34            class_hashes: class_iris.iter().map(|s| q_hash(s)).collect(),
35        }
36    }
37
38    /// Nearest-centroid style: set each class row to the provided centroid vector.
39    pub fn from_centroids(
40        dim: usize,
41        class_iris: &[&str],
42        centroids: &[&[f32]],
43    ) -> Result<Self, VisionError> {
44        if class_iris.len() != centroids.len() {
45            return Err(VisionError::MalformedImage);
46        }
47        let mut h = Self::zeros(dim, class_iris);
48        for (c, cent) in centroids.iter().enumerate() {
49            if cent.len() < dim {
50                return Err(VisionError::OutputBufferTooSmall);
51            }
52            for i in 0..dim {
53                h.weight[c * dim + i] = cent[i];
54            }
55        }
56        Ok(h)
57    }
58
59    /// Softmax-free argmax score into detection (whole-image box).
60    pub fn classify_embedding(&self, embed: &[f32], min_score: f32) -> Option<Detection> {
61        if embed.len() < self.dim || self.n_classes == 0 {
62            return None;
63        }
64        let mut best_c = 0usize;
65        let mut best = f32::NEG_INFINITY;
66        for c in 0..self.n_classes {
67            let mut logit = self.bias[c];
68            for i in 0..self.dim {
69                logit += embed[i] * self.weight[c * self.dim + i];
70            }
71            if logit > best {
72                best = logit;
73                best_c = c;
74            }
75        }
76        // Map logit to (0,1) via tanh for score_u16 packing.
77        let score = ((best.tanh() + 1.0) * 0.5).clamp(0.0, 1.0);
78        if score < min_score {
79            return None;
80        }
81        let mut d = Detection::empty();
82        d.class_hash = self.class_hashes[best_c];
83        d.instance_hash = d.class_hash ^ (best.to_bits() as u64);
84        d.score_u16 = (score * 65535.0) as u16;
85        d.x_min_u16 = 0;
86        d.y_min_u16 = 0;
87        d.x_max_u16 = 65535;
88        d.y_max_u16 = 65535;
89        d.flags = Detection::FLAG_LOW_ASSURANCE;
90        Some(d)
91    }
92}
93
94/// Wraps a base visual model: takes its embedding, runs linear head, emits 0–1 detection.
95pub struct LinearProbeVision<M: VisualModel> {
96    pub base: M,
97    pub head: LinearHead,
98    pub min_score: f32,
99    pub model_hash: u64,
100}
101
102impl<M: VisualModel> LinearProbeVision<M> {
103    /// Creates a new instance.
104    pub fn new(base: M, head: LinearHead, min_score: f32) -> Self {
105        let model_hash = q_hash("qualia-vision-linear-probe-v1");
106        Self {
107            base,
108            head,
109            min_score,
110            model_hash,
111        }
112    }
113
114    /// `model` hash.
115    pub fn model_hash(&self) -> u64 {
116        self.model_hash
117    }
118}
119
120impl<M: VisualModel> VisualModel for LinearProbeVision<M> {
121    fn capabilities(&self) -> VisualCapabilities {
122        let mut c = self.base.capabilities();
123        c.supports_boxes = true;
124        c.max_detections = 1;
125        c.is_reference_backend = c.is_reference_backend || true;
126        c
127    }
128
129    fn infer(
130        &mut self,
131        image: ImageView<'_>,
132        detections_out: &mut [Detection],
133        embedding_out: &mut [f32],
134        workspace: &mut [u8],
135    ) -> Result<VisualOutputCounts, VisionError> {
136        if detections_out.is_empty() {
137            return Err(VisionError::OutputBufferTooSmall);
138        }
139        // Base may write multiple dets; we only need embed.
140        let mut scratch_dets = [Detection::empty(); MAX_DETECTIONS];
141        let base_counts = self
142            .base
143            .infer(image, &mut scratch_dets, embedding_out, workspace)?;
144        let emb_n = base_counts.embedding_written.min(embedding_out.len());
145        for d in detections_out.iter_mut() {
146            *d = Detection::empty();
147        }
148        let mut det_n = 0usize;
149        if emb_n >= self.head.dim {
150            if let Some(d) = self
151                .head
152                .classify_embedding(&embedding_out[..emb_n], self.min_score)
153            {
154                detections_out[0] = d;
155                det_n = 1;
156            }
157        }
158        Ok(VisualOutputCounts {
159            detections: det_n,
160            embedding_written: emb_n,
161        })
162    }
163}
164
165/// Train a 2-class head by averaging embeddings of two example batches (centroid).
166pub fn fit_two_class_centroids(
167    dim: usize,
168    class_a: &str,
169    embeds_a: &[&[f32]],
170    class_b: &str,
171    embeds_b: &[&[f32]],
172) -> Result<LinearHead, VisionError> {
173    if embeds_a.is_empty() || embeds_b.is_empty() {
174        return Err(VisionError::MalformedImage);
175    }
176    let mut ca = vec![0.0f32; dim];
177    let mut cb = vec![0.0f32; dim];
178    for e in embeds_a {
179        if e.len() < dim {
180            return Err(VisionError::OutputBufferTooSmall);
181        }
182        for i in 0..dim {
183            ca[i] += e[i];
184        }
185    }
186    for e in embeds_b {
187        if e.len() < dim {
188            return Err(VisionError::OutputBufferTooSmall);
189        }
190        for i in 0..dim {
191            cb[i] += e[i];
192        }
193    }
194    let na = embeds_a.len() as f32;
195    let nb = embeds_b.len() as f32;
196    for i in 0..dim {
197        ca[i] /= na;
198        cb[i] /= nb;
199    }
200    LinearHead::from_centroids(dim, &[class_a, class_b], &[&ca, &cb])
201}
202
203#[cfg(test)]
204mod tests {
205    use super::*;
206    use crate::cpu_reference::{CpuReferenceVision, CLASS_MOSTLY_BLUE, CLASS_MOSTLY_RED};
207    use crate::types::PixelFormat;
208
209    fn solid(rgb: [u8; 3], w: u32, h: u32) -> Vec<u8> {
210        let mut v = vec![0u8; (w * h * 3) as usize];
211        for p in v.chunks_mut(3) {
212            p.copy_from_slice(&rgb);
213        }
214        v
215    }
216
217    #[test]
218    fn linear_probe_prefers_matching_centroid() {
219        let mut base = CpuReferenceVision::new();
220        let red = solid([220, 10, 10], 8, 8);
221        let blue = solid([10, 10, 220], 8, 8);
222        let img_r = ImageView {
223            bytes: &red,
224            width: 8,
225            height: 8,
226            row_stride: 24,
227            format: PixelFormat::Rgb8,
228        };
229        let img_b = ImageView {
230            bytes: &blue,
231            width: 8,
232            height: 8,
233            row_stride: 24,
234            format: PixelFormat::Rgb8,
235        };
236        let mut dets = [Detection::empty(); 8];
237        let mut emb_r = [0.0f32; 16];
238        let mut emb_b = [0.0f32; 16];
239        let mut ws = [0u8; 64];
240        base.infer(img_r, &mut dets, &mut emb_r, &mut ws).unwrap();
241        base.infer(img_b, &mut dets, &mut emb_b, &mut ws).unwrap();
242        let head = fit_two_class_centroids(
243            16,
244            CLASS_MOSTLY_RED,
245            &[&emb_r],
246            CLASS_MOSTLY_BLUE,
247            &[&emb_b],
248        )
249        .unwrap();
250        let mut probe = LinearProbeVision::new(base, head, 0.01);
251        let c = probe.infer(img_r, &mut dets, &mut emb_r, &mut ws).unwrap();
252        assert_eq!(c.detections, 1);
253        assert_eq!(dets[0].class_hash, q_hash(CLASS_MOSTLY_RED));
254    }
255}