1use std::io::Read;
8use std::path::{Path, PathBuf};
9
10use qualia_core_db::{gguf_sharder::GGufSharder, q_hash, wal::WriteAheadLog, NQuin};
11use serde::Serialize;
12use sha2::{Digest, Sha256};
13
14use crate::model_lifecycle::{self, ActiveModelRecord, InstallManifest};
15
16#[derive(Debug)]
17pub enum VisionError {
18 NoActiveModel,
19 NotMultimodal,
20 MissingProjector,
21 InactiveLifecycle,
22 Io(std::io::Error),
23 Wal(String),
24 Json(serde_json::Error),
25 Buffer(String),
27}
28
29impl std::fmt::Display for VisionError {
30 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
31 match self {
32 VisionError::NoActiveModel => {
33 write!(
34 f,
35 "No active model — activate a multimodal model in LLM Hub"
36 )
37 }
38 VisionError::NotMultimodal => write!(f, "Active model is text-only; install a VLM"),
39 VisionError::MissingProjector => {
40 write!(f, "Active model is missing mmproj projector path")
41 }
42 VisionError::InactiveLifecycle => {
43 write!(
44 f,
45 "Model lifecycle is not Active — activate model before image ingest"
46 )
47 }
48 VisionError::Io(e) => write!(f, "IO error: {e}"),
49 VisionError::Wal(e) => write!(f, "WAL error: {e}"),
50 VisionError::Json(e) => write!(f, "JSON error: {e}"),
51 VisionError::Buffer(e) => write!(f, "Buffer error: {e}"),
52 }
53 }
54}
55
56impl From<std::io::Error> for VisionError {
57 fn from(e: std::io::Error) -> Self {
58 VisionError::Io(e)
59 }
60}
61
62impl From<serde_json::Error> for VisionError {
63 fn from(e: serde_json::Error) -> Self {
64 VisionError::Json(e)
65 }
66}
67
68#[derive(Debug, Clone, Serialize)]
69pub struct VisionIngestResult {
70 pub status: String,
71 pub file: String,
72 pub typology: String,
73 pub lexicon_id: String,
74 pub image_sha256: String,
75 pub model_id: String,
76 pub mmproj_path: String,
77 pub architecture: Option<String>,
78 pub facet: String,
79 pub wal_path: String,
80 pub vision_quins_appended: usize,
81}
82
83fn sha256_file(path: &Path) -> Result<String, std::io::Error> {
84 let mut file = std::fs::File::open(path)?;
85 let mut hasher = Sha256::new();
86 let mut buf = [0u8; 65_536];
87 loop {
88 let n = file.read(&mut buf)?;
89 if n == 0 {
90 break;
91 }
92 hasher.update(&buf[..n]);
93 }
94 Ok(hex::encode(hasher.finalize()))
95}
96
97fn facet_for_typology(typology: &str, image_hash: &str, arch: Option<&str>) -> String {
98 let arch_label = arch.unwrap_or("vlm");
99 match typology {
100 "Meme" => format!("{arch_label} meme tensor | irony-bound | sha256:{image_hash}"),
101 "Heraldry" => {
102 format!("{arch_label} heraldry charge tensor | tincture-bound | sha256:{image_hash}")
103 }
104 "Clinical" | "DICOM" => {
105 format!("{arch_label} clinical imaging facet | sha256:{image_hash}")
106 }
107 _ => format!("{arch_label} asset facet | typology:{typology} | sha256:{image_hash}"),
108 }
109}
110
111fn provenance_quin(image_path: &str, typology: &str, timestamp: u64) -> NQuin {
112 let subject = q_hash(&format!("vision:{}", image_path));
113 let predicate = q_hash("prov:wasDerivedFrom");
114 let object = q_hash(typology);
115 let context = q_hash("ctx:vision_ingest");
116 let metadata = timestamp & 0xFFFF_FFFF;
117 let parity = subject ^ predicate ^ object ^ context ^ metadata;
118 NQuin {
119 subject,
120 predicate,
121 object,
122 context,
123 metadata,
124 parity,
125 }
126}
127
128pub fn resolve_active_multimodal(
130 storage_root: &Path,
131 active: &ActiveModelRecord,
132) -> Result<(InstallManifest, PathBuf), VisionError> {
133 if active.modality != "multimodal" {
134 return Err(VisionError::NotMultimodal);
135 }
136 if active.lifecycle_state != "Active" {
137 return Err(VisionError::InactiveLifecycle);
138 }
139 let manifest = model_lifecycle::load_install_manifest(storage_root, &active.model_id)
140 .ok_or(VisionError::NoActiveModel)?;
141 let mmproj = active
142 .mmproj_path
143 .as_deref()
144 .or(manifest.mmproj_path.as_deref())
145 .ok_or(VisionError::MissingProjector)?;
146 let mmproj_path = PathBuf::from(mmproj);
147 if !mmproj_path.is_file() {
148 return Err(VisionError::MissingProjector);
149 }
150 Ok((manifest, mmproj_path))
151}
152
153pub fn ingest_image_file(
155 storage_root: &Path,
156 active: &ActiveModelRecord,
157 file_path: &Path,
158 typology: &str,
159) -> Result<VisionIngestResult, VisionError> {
160 let (manifest, mmproj_path) = resolve_active_multimodal(storage_root, active)?;
161
162 if !file_path.is_file() {
163 return Err(VisionError::Io(std::io::Error::new(
164 std::io::ErrorKind::NotFound,
165 format!("Image not found: {}", file_path.display()),
166 )));
167 }
168
169 let image_sha256 = sha256_file(file_path)?;
170 let lexicon_id = format!("0x{:016X}", q_hash(&image_sha256) & 0xFFFF_FFFF_FFFF_FFFF);
171
172 let mmproj_str = mmproj_path.to_string_lossy().into_owned();
173 let vision_quins = GGufSharder::new(mmproj_str).generate_bidx_pointer_map();
174
175 let wal_path = model_lifecycle::models_dir(storage_root).join("vision_ingest.wal");
176 let mut wal = WriteAheadLog::open(&wal_path)
177 .map_err(|e| VisionError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
178
179 let timestamp = std::time::SystemTime::now()
180 .duration_since(std::time::UNIX_EPOCH)
181 .unwrap_or_default()
182 .as_secs();
183
184 let prov = provenance_quin(&file_path.to_string_lossy(), typology, timestamp);
185 wal.append_mutation(&prov)
186 .map_err(|e| VisionError::Wal(e.to_string()))?;
187
188 for q in &vision_quins {
189 wal.append_mutation(q)
190 .map_err(|e| VisionError::Wal(e.to_string()))?;
191 }
192
193 let facet = facet_for_typology(typology, &image_sha256, manifest.architecture.as_deref());
194
195 Ok(VisionIngestResult {
196 status: "success".to_string(),
197 file: file_path.to_string_lossy().into_owned(),
198 typology: typology.to_string(),
199 lexicon_id,
200 image_sha256,
201 model_id: active.model_id.clone(),
202 mmproj_path: mmproj_path.to_string_lossy().into_owned(),
203 architecture: manifest.architecture.clone(),
204 facet,
205 wal_path: wal_path.to_string_lossy().into_owned(),
206 vision_quins_appended: vision_quins.len(),
207 })
208}
209
210pub fn ingest_image_with_active_record(
212 storage_root: &Path,
213 active: Option<ActiveModelRecord>,
214 file_path: &Path,
215 typology: &str,
216) -> Result<VisionIngestResult, VisionError> {
217 let active = active.ok_or(VisionError::NoActiveModel)?;
218 ingest_image_file(storage_root, &active, file_path, typology)
219}
220
221#[derive(Debug, Clone, Copy)]
225pub struct NativeDetection {
226 pub class_hash: u64,
227 pub instance_hash: u64,
228 pub score_u16: u16,
229 pub x_min_u16: u16,
230 pub y_min_u16: u16,
231 pub x_max_u16: u16,
232 pub y_max_u16: u16,
233 pub frame_index: u32,
234 pub track_id: u32,
235 pub flags: u32,
236}
237
238const P_VISUAL_OBSERVATION: &str = "https://ns.webizen.org/q42/VisualObservation";
239const P_PROPOSES_CLASS: &str = "https://ns.webizen.org/q42/proposesClass";
240const P_HAS_BBOX: &str = "https://ns.webizen.org/q42/hasBoundingBox";
241const P_HAS_TRACK: &str = "https://ns.webizen.org/q42/hasTrackId";
242const P_MODEL_DIGEST: &str = "https://ns.webizen.org/q42/modelDigest";
243const P_HUMAN_REJECTS: &str = "https://ns.webizen.org/q42/humanRejects";
244const P_HUMAN_CORRECTS: &str = "https://ns.webizen.org/q42/humanCorrectsClass";
245const CTX_VISION: &str = "https://ns.webizen.org/q42/vision-observation";
246const CTX_HUMAN: &str = "https://ns.webizen.org/q42/human-attestation";
247
248#[inline]
249fn quin(s: u64, p: u64, o: u64, c: u64, m: u64) -> NQuin {
250 NQuin {
251 subject: s,
252 predicate: p,
253 object: o,
254 context: c,
255 metadata: m,
256 parity: s ^ p ^ o ^ c ^ m,
257 }
258}
259
260#[inline]
261fn pack_bbox(d: &NativeDetection) -> u64 {
262 (d.x_min_u16 as u64)
263 | ((d.y_min_u16 as u64) << 16)
264 | ((d.x_max_u16 as u64) << 32)
265 | ((d.y_max_u16 as u64) << 48)
266}
267
268pub fn compile_native_observation_quins(
271 media_hash: u64,
272 media_byte_len: u64,
273 model_hash: u64,
274 detections: &[NativeDetection],
275 out: &mut [NQuin],
276) -> Result<usize, VisionError> {
277 if out.is_empty() {
278 return Err(VisionError::Buffer("empty out".into()));
279 }
280 let mut w = 0usize;
281 out[w] = quin(
282 media_hash,
283 q_hash(P_MODEL_DIGEST),
284 model_hash,
285 q_hash(CTX_VISION),
286 media_byte_len,
287 );
288 w += 1;
289 let ctx = q_hash(CTX_VISION) ^ model_hash;
290 for d in detections {
291 if d.class_hash == 0 && d.score_u16 == 0 {
292 continue;
293 }
294 if w + 4 > out.len() {
295 break;
296 }
297 let meta_score =
298 (d.score_u16 as u64) | ((d.frame_index as u64) << 16) | ((d.flags as u64) << 48);
299 out[w] = quin(
300 media_hash,
301 q_hash(P_VISUAL_OBSERVATION),
302 d.instance_hash,
303 ctx,
304 meta_score,
305 );
306 w += 1;
307 out[w] = quin(
308 d.instance_hash,
309 q_hash(P_PROPOSES_CLASS),
310 d.class_hash,
311 ctx,
312 d.score_u16 as u64,
313 );
314 w += 1;
315 out[w] = quin(
316 d.instance_hash,
317 q_hash(P_HAS_BBOX),
318 pack_bbox(d),
319 ctx,
320 d.score_u16 as u64 | ((d.frame_index as u64) << 16),
321 );
322 w += 1;
323 out[w] = quin(
324 d.instance_hash,
325 q_hash(P_HAS_TRACK),
326 d.track_id as u64,
327 ctx,
328 d.frame_index as u64,
329 );
330 w += 1;
331 }
332 Ok(w)
333}
334
335pub fn human_reject_quin(human_did_hash: u64, instance_hash: u64, reason_hash: u64) -> NQuin {
337 quin(
338 human_did_hash,
339 q_hash(P_HUMAN_REJECTS),
340 instance_hash,
341 q_hash(CTX_HUMAN),
342 reason_hash,
343 )
344}
345
346pub fn human_correct_quin(human_did_hash: u64, instance_hash: u64, new_class_hash: u64) -> NQuin {
348 quin(
349 human_did_hash,
350 q_hash(P_HUMAN_CORRECTS),
351 new_class_hash,
352 q_hash(CTX_HUMAN) ^ instance_hash,
353 instance_hash,
354 )
355}
356
357pub fn append_native_observation_quins(
359 storage_root: &Path,
360 media_hash: u64,
361 media_byte_len: u64,
362 model_hash: u64,
363 detections: &[NativeDetection],
364) -> Result<usize, VisionError> {
365 let mut buf = [NQuin {
366 subject: 0,
367 predicate: 0,
368 object: 0,
369 context: 0,
370 metadata: 0,
371 parity: 0,
372 }; 256];
373 let n = compile_native_observation_quins(
374 media_hash,
375 media_byte_len,
376 model_hash,
377 detections,
378 &mut buf,
379 )?;
380 let wal_path = model_lifecycle::models_dir(storage_root).join("vision_native.wal");
381 if let Some(parent) = wal_path.parent() {
382 std::fs::create_dir_all(parent)?;
383 }
384 let mut wal = WriteAheadLog::open(&wal_path)
385 .map_err(|e| VisionError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
386 for q in buf.iter().take(n) {
387 wal.append_mutation(q)
388 .map_err(|e| VisionError::Wal(e.to_string()))?;
389 }
390 Ok(n)
391}
392
393pub fn append_human_attestation(storage_root: &Path, quin: &NQuin) -> Result<(), VisionError> {
395 let wal_path = model_lifecycle::models_dir(storage_root).join("vision_native.wal");
396 if let Some(parent) = wal_path.parent() {
397 std::fs::create_dir_all(parent)?;
398 }
399 let mut wal = WriteAheadLog::open(&wal_path)
400 .map_err(|e| VisionError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
401 wal.append_mutation(quin)
402 .map_err(|e| VisionError::Wal(e.to_string()))?;
403 Ok(())
404}
405
406#[cfg(test)]
407mod tests {
408 use super::*;
409
410 #[test]
411 fn native_compile_four_per_det_plus_digest() {
412 let d = NativeDetection {
413 class_hash: 1,
414 instance_hash: 2,
415 score_u16: 1000,
416 x_min_u16: 0,
417 y_min_u16: 0,
418 x_max_u16: 100,
419 y_max_u16: 100,
420 frame_index: 0,
421 track_id: 3,
422 flags: 0,
423 };
424 let mut out = [NQuin {
425 subject: 0,
426 predicate: 0,
427 object: 0,
428 context: 0,
429 metadata: 0,
430 parity: 0,
431 }; 16];
432 let n = compile_native_observation_quins(9, 64, 7, &[d], &mut out).unwrap();
433 assert_eq!(n, 5);
434 assert_eq!(out[0].predicate, q_hash(P_MODEL_DIGEST));
435 let rej = human_reject_quin(0xD1D, 2, 0);
436 assert_eq!(rej.predicate, q_hash(P_HUMAN_REJECTS));
437 assert!(out[..n]
439 .iter()
440 .any(|q| q.predicate == q_hash(P_PROPOSES_CLASS)));
441 }
442}