Skip to main content

qualia_client_core/
vision_ingest.rs

1//! Multimodal image ingest — binds assets to active mmproj + WAL (no Ollama).
2//!
3//! V6: also accepts **native detector** epistemic observation quins (no mmproj required)
4//! via `append_native_observation_quins` — machine claims with model digest + optional
5//! human reject/correct edges. Dense pixels never enter NQuins.
6
7use std::io::Read;
8use std::path::{Path, PathBuf};
9
10use qualia_core_db::{gguf_sharder::GGufSharder, q_hash, wal::WriteAheadLog, NQuin};
11use serde::Serialize;
12use sha2::{Digest, Sha256};
13
14use crate::model_lifecycle::{self, ActiveModelRecord, InstallManifest};
15
16#[derive(Debug)]
17pub enum VisionError {
18    NoActiveModel,
19    NotMultimodal,
20    MissingProjector,
21    InactiveLifecycle,
22    Io(std::io::Error),
23    Wal(String),
24    Json(serde_json::Error),
25    /// Caller buffer / detection packing error.
26    Buffer(String),
27}
28
29impl std::fmt::Display for VisionError {
30    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
31        match self {
32            VisionError::NoActiveModel => {
33                write!(
34                    f,
35                    "No active model — activate a multimodal model in LLM Hub"
36                )
37            }
38            VisionError::NotMultimodal => write!(f, "Active model is text-only; install a VLM"),
39            VisionError::MissingProjector => {
40                write!(f, "Active model is missing mmproj projector path")
41            }
42            VisionError::InactiveLifecycle => {
43                write!(
44                    f,
45                    "Model lifecycle is not Active — activate model before image ingest"
46                )
47            }
48            VisionError::Io(e) => write!(f, "IO error: {e}"),
49            VisionError::Wal(e) => write!(f, "WAL error: {e}"),
50            VisionError::Json(e) => write!(f, "JSON error: {e}"),
51            VisionError::Buffer(e) => write!(f, "Buffer error: {e}"),
52        }
53    }
54}
55
56impl From<std::io::Error> for VisionError {
57    fn from(e: std::io::Error) -> Self {
58        VisionError::Io(e)
59    }
60}
61
62impl From<serde_json::Error> for VisionError {
63    fn from(e: serde_json::Error) -> Self {
64        VisionError::Json(e)
65    }
66}
67
68#[derive(Debug, Clone, Serialize)]
69pub struct VisionIngestResult {
70    pub status: String,
71    pub file: String,
72    pub typology: String,
73    pub lexicon_id: String,
74    pub image_sha256: String,
75    pub model_id: String,
76    pub mmproj_path: String,
77    pub architecture: Option<String>,
78    pub facet: String,
79    pub wal_path: String,
80    pub vision_quins_appended: usize,
81}
82
83fn sha256_file(path: &Path) -> Result<String, std::io::Error> {
84    let mut file = std::fs::File::open(path)?;
85    let mut hasher = Sha256::new();
86    let mut buf = [0u8; 65_536];
87    loop {
88        let n = file.read(&mut buf)?;
89        if n == 0 {
90            break;
91        }
92        hasher.update(&buf[..n]);
93    }
94    Ok(hex::encode(hasher.finalize()))
95}
96
97fn facet_for_typology(typology: &str, image_hash: &str, arch: Option<&str>) -> String {
98    let arch_label = arch.unwrap_or("vlm");
99    match typology {
100        "Meme" => format!("{arch_label} meme tensor | irony-bound | sha256:{image_hash}"),
101        "Heraldry" => {
102            format!("{arch_label} heraldry charge tensor | tincture-bound | sha256:{image_hash}")
103        }
104        "Clinical" | "DICOM" => {
105            format!("{arch_label} clinical imaging facet | sha256:{image_hash}")
106        }
107        _ => format!("{arch_label} asset facet | typology:{typology} | sha256:{image_hash}"),
108    }
109}
110
111fn provenance_quin(image_path: &str, typology: &str, timestamp: u64) -> NQuin {
112    let subject = q_hash(&format!("vision:{}", image_path));
113    let predicate = q_hash("prov:wasDerivedFrom");
114    let object = q_hash(typology);
115    let context = q_hash("ctx:vision_ingest");
116    let metadata = timestamp & 0xFFFF_FFFF;
117    let parity = subject ^ predicate ^ object ^ context ^ metadata;
118    NQuin {
119        subject,
120        predicate,
121        object,
122        context,
123        metadata,
124        parity,
125    }
126}
127
128/// Resolves the active multimodal backend.
129pub fn resolve_active_multimodal(
130    storage_root: &Path,
131    active: &ActiveModelRecord,
132) -> Result<(InstallManifest, PathBuf), VisionError> {
133    if active.modality != "multimodal" {
134        return Err(VisionError::NotMultimodal);
135    }
136    if active.lifecycle_state != "Active" {
137        return Err(VisionError::InactiveLifecycle);
138    }
139    let manifest = model_lifecycle::load_install_manifest(storage_root, &active.model_id)
140        .ok_or(VisionError::NoActiveModel)?;
141    let mmproj = active
142        .mmproj_path
143        .as_deref()
144        .or(manifest.mmproj_path.as_deref())
145        .ok_or(VisionError::MissingProjector)?;
146    let mmproj_path = PathBuf::from(mmproj);
147    if !mmproj_path.is_file() {
148        return Err(VisionError::MissingProjector);
149    }
150    Ok((manifest, mmproj_path))
151}
152
153/// Ingests an image file.
154pub fn ingest_image_file(
155    storage_root: &Path,
156    active: &ActiveModelRecord,
157    file_path: &Path,
158    typology: &str,
159) -> Result<VisionIngestResult, VisionError> {
160    let (manifest, mmproj_path) = resolve_active_multimodal(storage_root, active)?;
161
162    if !file_path.is_file() {
163        return Err(VisionError::Io(std::io::Error::new(
164            std::io::ErrorKind::NotFound,
165            format!("Image not found: {}", file_path.display()),
166        )));
167    }
168
169    let image_sha256 = sha256_file(file_path)?;
170    let lexicon_id = format!("0x{:016X}", q_hash(&image_sha256) & 0xFFFF_FFFF_FFFF_FFFF);
171
172    let mmproj_str = mmproj_path.to_string_lossy().into_owned();
173    let vision_quins = GGufSharder::new(mmproj_str).generate_bidx_pointer_map();
174
175    let wal_path = model_lifecycle::models_dir(storage_root).join("vision_ingest.wal");
176    let mut wal = WriteAheadLog::open(&wal_path)
177        .map_err(|e| VisionError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
178
179    let timestamp = std::time::SystemTime::now()
180        .duration_since(std::time::UNIX_EPOCH)
181        .unwrap_or_default()
182        .as_secs();
183
184    let prov = provenance_quin(&file_path.to_string_lossy(), typology, timestamp);
185    wal.append_mutation(&prov)
186        .map_err(|e| VisionError::Wal(e.to_string()))?;
187
188    for q in &vision_quins {
189        wal.append_mutation(q)
190            .map_err(|e| VisionError::Wal(e.to_string()))?;
191    }
192
193    let facet = facet_for_typology(typology, &image_sha256, manifest.architecture.as_deref());
194
195    Ok(VisionIngestResult {
196        status: "success".to_string(),
197        file: file_path.to_string_lossy().into_owned(),
198        typology: typology.to_string(),
199        lexicon_id,
200        image_sha256,
201        model_id: active.model_id.clone(),
202        mmproj_path: mmproj_path.to_string_lossy().into_owned(),
203        architecture: manifest.architecture.clone(),
204        facet,
205        wal_path: wal_path.to_string_lossy().into_owned(),
206        vision_quins_appended: vision_quins.len(),
207    })
208}
209
210/// Ingests an image with the active record.
211pub fn ingest_image_with_active_record(
212    storage_root: &Path,
213    active: Option<ActiveModelRecord>,
214    file_path: &Path,
215    typology: &str,
216) -> Result<VisionIngestResult, VisionError> {
217    let active = active.ok_or(VisionError::NoActiveModel)?;
218    ingest_image_file(storage_root, &active, file_path, typology)
219}
220
221// ── V6 native detector / epistemic observation path (no mmproj) ─────────────
222
223/// Fixed-layout detection for WAL packing (mirrors `qualia_vision::Detection` fields).
224#[derive(Debug, Clone, Copy)]
225pub struct NativeDetection {
226    pub class_hash: u64,
227    pub instance_hash: u64,
228    pub score_u16: u16,
229    pub x_min_u16: u16,
230    pub y_min_u16: u16,
231    pub x_max_u16: u16,
232    pub y_max_u16: u16,
233    pub frame_index: u32,
234    pub track_id: u32,
235    pub flags: u32,
236}
237
238const P_VISUAL_OBSERVATION: &str = "https://ns.webizen.org/q42/VisualObservation";
239const P_PROPOSES_CLASS: &str = "https://ns.webizen.org/q42/proposesClass";
240const P_HAS_BBOX: &str = "https://ns.webizen.org/q42/hasBoundingBox";
241const P_HAS_TRACK: &str = "https://ns.webizen.org/q42/hasTrackId";
242const P_MODEL_DIGEST: &str = "https://ns.webizen.org/q42/modelDigest";
243const P_HUMAN_REJECTS: &str = "https://ns.webizen.org/q42/humanRejects";
244const P_HUMAN_CORRECTS: &str = "https://ns.webizen.org/q42/humanCorrectsClass";
245const CTX_VISION: &str = "https://ns.webizen.org/q42/vision-observation";
246const CTX_HUMAN: &str = "https://ns.webizen.org/q42/human-attestation";
247
248#[inline]
249fn quin(s: u64, p: u64, o: u64, c: u64, m: u64) -> NQuin {
250    NQuin {
251        subject: s,
252        predicate: p,
253        object: o,
254        context: c,
255        metadata: m,
256        parity: s ^ p ^ o ^ c ^ m,
257    }
258}
259
260#[inline]
261fn pack_bbox(d: &NativeDetection) -> u64 {
262    (d.x_min_u16 as u64)
263        | ((d.y_min_u16 as u64) << 16)
264        | ((d.x_max_u16 as u64) << 32)
265        | ((d.y_max_u16 as u64) << 48)
266}
267
268/// Compile native detections into epistemic NQuins (caller buffer).
269/// Writes model digest + per-det observation/class/bbox/track. Returns count.
270pub fn compile_native_observation_quins(
271    media_hash: u64,
272    media_byte_len: u64,
273    model_hash: u64,
274    detections: &[NativeDetection],
275    out: &mut [NQuin],
276) -> Result<usize, VisionError> {
277    if out.is_empty() {
278        return Err(VisionError::Buffer("empty out".into()));
279    }
280    let mut w = 0usize;
281    out[w] = quin(
282        media_hash,
283        q_hash(P_MODEL_DIGEST),
284        model_hash,
285        q_hash(CTX_VISION),
286        media_byte_len,
287    );
288    w += 1;
289    let ctx = q_hash(CTX_VISION) ^ model_hash;
290    for d in detections {
291        if d.class_hash == 0 && d.score_u16 == 0 {
292            continue;
293        }
294        if w + 4 > out.len() {
295            break;
296        }
297        let meta_score =
298            (d.score_u16 as u64) | ((d.frame_index as u64) << 16) | ((d.flags as u64) << 48);
299        out[w] = quin(
300            media_hash,
301            q_hash(P_VISUAL_OBSERVATION),
302            d.instance_hash,
303            ctx,
304            meta_score,
305        );
306        w += 1;
307        out[w] = quin(
308            d.instance_hash,
309            q_hash(P_PROPOSES_CLASS),
310            d.class_hash,
311            ctx,
312            d.score_u16 as u64,
313        );
314        w += 1;
315        out[w] = quin(
316            d.instance_hash,
317            q_hash(P_HAS_BBOX),
318            pack_bbox(d),
319            ctx,
320            d.score_u16 as u64 | ((d.frame_index as u64) << 16),
321        );
322        w += 1;
323        out[w] = quin(
324            d.instance_hash,
325            q_hash(P_HAS_TRACK),
326            d.track_id as u64,
327            ctx,
328            d.frame_index as u64,
329        );
330        w += 1;
331    }
332    Ok(w)
333}
334
335/// Human reject of a machine instance — does **not** erase machine quins.
336pub fn human_reject_quin(human_did_hash: u64, instance_hash: u64, reason_hash: u64) -> NQuin {
337    quin(
338        human_did_hash,
339        q_hash(P_HUMAN_REJECTS),
340        instance_hash,
341        q_hash(CTX_HUMAN),
342        reason_hash,
343    )
344}
345
346/// Human class correction — coexists with machine `proposesClass`.
347pub fn human_correct_quin(human_did_hash: u64, instance_hash: u64, new_class_hash: u64) -> NQuin {
348    quin(
349        human_did_hash,
350        q_hash(P_HUMAN_CORRECTS),
351        new_class_hash,
352        q_hash(CTX_HUMAN) ^ instance_hash,
353        instance_hash,
354    )
355}
356
357/// Append native observation quins to `vision_native.wal` (no active VLM required).
358pub fn append_native_observation_quins(
359    storage_root: &Path,
360    media_hash: u64,
361    media_byte_len: u64,
362    model_hash: u64,
363    detections: &[NativeDetection],
364) -> Result<usize, VisionError> {
365    let mut buf = [NQuin {
366        subject: 0,
367        predicate: 0,
368        object: 0,
369        context: 0,
370        metadata: 0,
371        parity: 0,
372    }; 256];
373    let n = compile_native_observation_quins(
374        media_hash,
375        media_byte_len,
376        model_hash,
377        detections,
378        &mut buf,
379    )?;
380    let wal_path = model_lifecycle::models_dir(storage_root).join("vision_native.wal");
381    if let Some(parent) = wal_path.parent() {
382        std::fs::create_dir_all(parent)?;
383    }
384    let mut wal = WriteAheadLog::open(&wal_path)
385        .map_err(|e| VisionError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
386    for q in buf.iter().take(n) {
387        wal.append_mutation(q)
388            .map_err(|e| VisionError::Wal(e.to_string()))?;
389    }
390    Ok(n)
391}
392
393/// Append a human reject/correct edge (provenance preserved).
394pub fn append_human_attestation(storage_root: &Path, quin: &NQuin) -> Result<(), VisionError> {
395    let wal_path = model_lifecycle::models_dir(storage_root).join("vision_native.wal");
396    if let Some(parent) = wal_path.parent() {
397        std::fs::create_dir_all(parent)?;
398    }
399    let mut wal = WriteAheadLog::open(&wal_path)
400        .map_err(|e| VisionError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
401    wal.append_mutation(quin)
402        .map_err(|e| VisionError::Wal(e.to_string()))?;
403    Ok(())
404}
405
406#[cfg(test)]
407mod tests {
408    use super::*;
409
410    #[test]
411    fn native_compile_four_per_det_plus_digest() {
412        let d = NativeDetection {
413            class_hash: 1,
414            instance_hash: 2,
415            score_u16: 1000,
416            x_min_u16: 0,
417            y_min_u16: 0,
418            x_max_u16: 100,
419            y_max_u16: 100,
420            frame_index: 0,
421            track_id: 3,
422            flags: 0,
423        };
424        let mut out = [NQuin {
425            subject: 0,
426            predicate: 0,
427            object: 0,
428            context: 0,
429            metadata: 0,
430            parity: 0,
431        }; 16];
432        let n = compile_native_observation_quins(9, 64, 7, &[d], &mut out).unwrap();
433        assert_eq!(n, 5);
434        assert_eq!(out[0].predicate, q_hash(P_MODEL_DIGEST));
435        let rej = human_reject_quin(0xD1D, 2, 0);
436        assert_eq!(rej.predicate, q_hash(P_HUMAN_REJECTS));
437        // Machine class proposal still in buffer.
438        assert!(out[..n]
439            .iter()
440            .any(|q| q.predicate == q_hash(P_PROPOSES_CLASS)));
441    }
442}