1use crate::semantic::q_hash;
8use crate::types::{
9 Detection, ImageView, VisionError, VisualCapabilities, VisualModel, VisualOutputCounts,
10 MAX_DETECTIONS,
11};
12
13#[derive(Debug, Clone)]
15pub struct LinearHead {
16 pub dim: usize,
17 pub n_classes: usize,
18 pub weight: Vec<f32>,
20 pub bias: Vec<f32>,
21 pub class_hashes: Vec<u64>,
23}
24
25impl LinearHead {
26 pub fn zeros(dim: usize, class_iris: &[&str]) -> Self {
28 let n = class_iris.len();
29 Self {
30 dim,
31 n_classes: n,
32 weight: vec![0.0; n * dim],
33 bias: vec![0.0; n],
34 class_hashes: class_iris.iter().map(|s| q_hash(s)).collect(),
35 }
36 }
37
38 pub fn from_centroids(
40 dim: usize,
41 class_iris: &[&str],
42 centroids: &[&[f32]],
43 ) -> Result<Self, VisionError> {
44 if class_iris.len() != centroids.len() {
45 return Err(VisionError::MalformedImage);
46 }
47 let mut h = Self::zeros(dim, class_iris);
48 for (c, cent) in centroids.iter().enumerate() {
49 if cent.len() < dim {
50 return Err(VisionError::OutputBufferTooSmall);
51 }
52 for i in 0..dim {
53 h.weight[c * dim + i] = cent[i];
54 }
55 }
56 Ok(h)
57 }
58
59 pub fn classify_embedding(&self, embed: &[f32], min_score: f32) -> Option<Detection> {
61 if embed.len() < self.dim || self.n_classes == 0 {
62 return None;
63 }
64 let mut best_c = 0usize;
65 let mut best = f32::NEG_INFINITY;
66 for c in 0..self.n_classes {
67 let mut logit = self.bias[c];
68 for i in 0..self.dim {
69 logit += embed[i] * self.weight[c * self.dim + i];
70 }
71 if logit > best {
72 best = logit;
73 best_c = c;
74 }
75 }
76 let score = ((best.tanh() + 1.0) * 0.5).clamp(0.0, 1.0);
78 if score < min_score {
79 return None;
80 }
81 let mut d = Detection::empty();
82 d.class_hash = self.class_hashes[best_c];
83 d.instance_hash = d.class_hash ^ (best.to_bits() as u64);
84 d.score_u16 = (score * 65535.0) as u16;
85 d.x_min_u16 = 0;
86 d.y_min_u16 = 0;
87 d.x_max_u16 = 65535;
88 d.y_max_u16 = 65535;
89 d.flags = Detection::FLAG_LOW_ASSURANCE;
90 Some(d)
91 }
92}
93
94pub struct LinearProbeVision<M: VisualModel> {
96 pub base: M,
97 pub head: LinearHead,
98 pub min_score: f32,
99 pub model_hash: u64,
100}
101
102impl<M: VisualModel> LinearProbeVision<M> {
103 pub fn new(base: M, head: LinearHead, min_score: f32) -> Self {
105 let model_hash = q_hash("qualia-vision-linear-probe-v1");
106 Self {
107 base,
108 head,
109 min_score,
110 model_hash,
111 }
112 }
113
114 pub fn model_hash(&self) -> u64 {
116 self.model_hash
117 }
118}
119
120impl<M: VisualModel> VisualModel for LinearProbeVision<M> {
121 fn capabilities(&self) -> VisualCapabilities {
122 let mut c = self.base.capabilities();
123 c.supports_boxes = true;
124 c.max_detections = 1;
125 c.is_reference_backend = c.is_reference_backend || true;
126 c
127 }
128
129 fn infer(
130 &mut self,
131 image: ImageView<'_>,
132 detections_out: &mut [Detection],
133 embedding_out: &mut [f32],
134 workspace: &mut [u8],
135 ) -> Result<VisualOutputCounts, VisionError> {
136 if detections_out.is_empty() {
137 return Err(VisionError::OutputBufferTooSmall);
138 }
139 let mut scratch_dets = [Detection::empty(); MAX_DETECTIONS];
141 let base_counts = self
142 .base
143 .infer(image, &mut scratch_dets, embedding_out, workspace)?;
144 let emb_n = base_counts.embedding_written.min(embedding_out.len());
145 for d in detections_out.iter_mut() {
146 *d = Detection::empty();
147 }
148 let mut det_n = 0usize;
149 if emb_n >= self.head.dim {
150 if let Some(d) = self
151 .head
152 .classify_embedding(&embedding_out[..emb_n], self.min_score)
153 {
154 detections_out[0] = d;
155 det_n = 1;
156 }
157 }
158 Ok(VisualOutputCounts {
159 detections: det_n,
160 embedding_written: emb_n,
161 })
162 }
163}
164
165pub fn fit_two_class_centroids(
167 dim: usize,
168 class_a: &str,
169 embeds_a: &[&[f32]],
170 class_b: &str,
171 embeds_b: &[&[f32]],
172) -> Result<LinearHead, VisionError> {
173 if embeds_a.is_empty() || embeds_b.is_empty() {
174 return Err(VisionError::MalformedImage);
175 }
176 let mut ca = vec![0.0f32; dim];
177 let mut cb = vec![0.0f32; dim];
178 for e in embeds_a {
179 if e.len() < dim {
180 return Err(VisionError::OutputBufferTooSmall);
181 }
182 for i in 0..dim {
183 ca[i] += e[i];
184 }
185 }
186 for e in embeds_b {
187 if e.len() < dim {
188 return Err(VisionError::OutputBufferTooSmall);
189 }
190 for i in 0..dim {
191 cb[i] += e[i];
192 }
193 }
194 let na = embeds_a.len() as f32;
195 let nb = embeds_b.len() as f32;
196 for i in 0..dim {
197 ca[i] /= na;
198 cb[i] /= nb;
199 }
200 LinearHead::from_centroids(dim, &[class_a, class_b], &[&ca, &cb])
201}
202
203#[cfg(test)]
204mod tests {
205 use super::*;
206 use crate::cpu_reference::{CpuReferenceVision, CLASS_MOSTLY_BLUE, CLASS_MOSTLY_RED};
207 use crate::types::PixelFormat;
208
209 fn solid(rgb: [u8; 3], w: u32, h: u32) -> Vec<u8> {
210 let mut v = vec![0u8; (w * h * 3) as usize];
211 for p in v.chunks_mut(3) {
212 p.copy_from_slice(&rgb);
213 }
214 v
215 }
216
217 #[test]
218 fn linear_probe_prefers_matching_centroid() {
219 let mut base = CpuReferenceVision::new();
220 let red = solid([220, 10, 10], 8, 8);
221 let blue = solid([10, 10, 220], 8, 8);
222 let img_r = ImageView {
223 bytes: &red,
224 width: 8,
225 height: 8,
226 row_stride: 24,
227 format: PixelFormat::Rgb8,
228 };
229 let img_b = ImageView {
230 bytes: &blue,
231 width: 8,
232 height: 8,
233 row_stride: 24,
234 format: PixelFormat::Rgb8,
235 };
236 let mut dets = [Detection::empty(); 8];
237 let mut emb_r = [0.0f32; 16];
238 let mut emb_b = [0.0f32; 16];
239 let mut ws = [0u8; 64];
240 base.infer(img_r, &mut dets, &mut emb_r, &mut ws).unwrap();
241 base.infer(img_b, &mut dets, &mut emb_b, &mut ws).unwrap();
242 let head = fit_two_class_centroids(
243 16,
244 CLASS_MOSTLY_RED,
245 &[&emb_r],
246 CLASS_MOSTLY_BLUE,
247 &[&emb_b],
248 )
249 .unwrap();
250 let mut probe = LinearProbeVision::new(base, head, 0.01);
251 let c = probe.infer(img_r, &mut dets, &mut emb_r, &mut ws).unwrap();
252 assert_eq!(c.detections, 1);
253 assert_eq!(dets[0].class_hash, q_hash(CLASS_MOSTLY_RED));
254 }
255}