qualia_client_core/
vision_ingest.rs

1//! Multimodal image ingest — binds assets to active mmproj + WAL (no Ollama).
2//!
3//! V6: also accepts **native detector** epistemic observation quins (no mmproj required)
4//! via `append_native_observation_quins` — machine claims with model digest + optional
5//! human reject/correct edges. Dense pixels never enter NQuins.
6
7use std::io::Read;
8use std::path::{Path, PathBuf};
9
10use qualia_core_db::{gguf_sharder::GGufSharder, q_hash, wal::WriteAheadLog, NQuin};
11use serde::Serialize;
12use sha2::{Digest, Sha256};
13
14use crate::model_lifecycle::{self, ActiveModelRecord, InstallManifest};
15
16#[derive(Debug)]
17pub enum VisionError {
18    NoActiveModel,
19    NotMultimodal,
20    MissingProjector,
21    InactiveLifecycle,
22    Io(std::io::Error),
23    Wal(String),
24    Json(serde_json::Error),
25    /// Caller buffer / detection packing error.
26    Buffer(String),
27}
28
29impl std::fmt::Display for VisionError {
30    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
31        match self {
32            VisionError::NoActiveModel => {
33                write!(
34                    f,
35                    "No active model — activate a multimodal model in LLM Hub"
36                )
37            }
38            VisionError::NotMultimodal => write!(f, "Active model is text-only; install a VLM"),
39            VisionError::MissingProjector => {
40                write!(f, "Active model is missing mmproj projector path")
41            }
42            VisionError::InactiveLifecycle => {
43                write!(
44                    f,
45                    "Model lifecycle is not Active — activate model before image ingest"
46                )
47            }
48            VisionError::Io(e) => write!(f, "IO error: {e}"),
49            VisionError::Wal(e) => write!(f, "WAL error: {e}"),
50            VisionError::Json(e) => write!(f, "JSON error: {e}"),
51            VisionError::Buffer(e) => write!(f, "Buffer error: {e}"),
52        }
53    }
54}
55
56impl From<std::io::Error> for VisionError {
57    fn from(e: std::io::Error) -> Self {
58        VisionError::Io(e)
59    }
60}
61
62impl From<serde_json::Error> for VisionError {
63    fn from(e: serde_json::Error) -> Self {
64        VisionError::Json(e)
65    }
66}
67
68#[derive(Debug, Clone, Serialize)]
69pub struct VisionIngestResult {
70    pub status: String,
71    pub file: String,
72    pub typology: String,
73    pub lexicon_id: String,
74    pub image_sha256: String,
75    pub model_id: String,
76    pub mmproj_path: String,
77    pub architecture: Option<String>,
78    pub facet: String,
79    pub wal_path: String,
80    pub vision_quins_appended: usize,
81}
82
83fn sha256_file(path: &Path) -> Result<String, std::io::Error> {
84    let mut file = std::fs::File::open(path)?;
85    let mut hasher = Sha256::new();
86    let mut buf = [0u8; 65_536];
87    loop {
88        let n = file.read(&mut buf)?;
89        if n == 0 {
90            break;
91        }
92        hasher.update(&buf[..n]);
93    }
94    Ok(hex::encode(hasher.finalize()))
95}
96
97fn facet_for_typology(typology: &str, image_hash: &str, arch: Option<&str>) -> String {
98    let arch_label = arch.unwrap_or("vlm");
99    match typology {
100        "Meme" => format!("{arch_label} meme tensor | irony-bound | sha256:{image_hash}"),
101        "Heraldry" => {
102            format!("{arch_label} heraldry charge tensor | tincture-bound | sha256:{image_hash}")
103        }
104        "Clinical" | "DICOM" => {
105            format!("{arch_label} clinical imaging facet | sha256:{image_hash}")
106        }
107        _ => format!("{arch_label} asset facet | typology:{typology} | sha256:{image_hash}"),
108    }
109}
110
111fn provenance_quin(image_path: &str, typology: &str, timestamp: u64) -> NQuin {
112    let subject = q_hash(&format!("vision:{}", image_path));
113    let predicate = q_hash("prov:wasDerivedFrom");
114    let object = q_hash(typology);
115    let context = q_hash("ctx:vision_ingest");
116    let metadata = timestamp & 0xFFFF_FFFF;
117    let parity = subject ^ predicate ^ object ^ context ^ metadata;
118    NQuin {
119        subject,
120        predicate,
121        object,
122        context,
123        metadata,
124        parity,
125    }
126}
127
128pub fn resolve_active_multimodal(
129    storage_root: &Path,
130    active: &ActiveModelRecord,
131) -> Result<(InstallManifest, PathBuf), VisionError> {
132    if active.modality != "multimodal" {
133        return Err(VisionError::NotMultimodal);
134    }
135    if active.lifecycle_state != "Active" {
136        return Err(VisionError::InactiveLifecycle);
137    }
138    let manifest = model_lifecycle::load_install_manifest(storage_root, &active.model_id)
139        .ok_or(VisionError::NoActiveModel)?;
140    let mmproj = active
141        .mmproj_path
142        .as_deref()
143        .or(manifest.mmproj_path.as_deref())
144        .ok_or(VisionError::MissingProjector)?;
145    let mmproj_path = PathBuf::from(mmproj);
146    if !mmproj_path.is_file() {
147        return Err(VisionError::MissingProjector);
148    }
149    Ok((manifest, mmproj_path))
150}
151
152pub fn ingest_image_file(
153    storage_root: &Path,
154    active: &ActiveModelRecord,
155    file_path: &Path,
156    typology: &str,
157) -> Result<VisionIngestResult, VisionError> {
158    let (manifest, mmproj_path) = resolve_active_multimodal(storage_root, active)?;
159
160    if !file_path.is_file() {
161        return Err(VisionError::Io(std::io::Error::new(
162            std::io::ErrorKind::NotFound,
163            format!("Image not found: {}", file_path.display()),
164        )));
165    }
166
167    let image_sha256 = sha256_file(file_path)?;
168    let lexicon_id = format!("0x{:016X}", q_hash(&image_sha256) & 0xFFFF_FFFF_FFFF_FFFF);
169
170    let mmproj_str = mmproj_path.to_string_lossy().into_owned();
171    let vision_quins = GGufSharder::new(mmproj_str).generate_bidx_pointer_map();
172
173    let wal_path = model_lifecycle::models_dir(storage_root).join("vision_ingest.wal");
174    let mut wal = WriteAheadLog::open(&wal_path)
175        .map_err(|e| VisionError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
176
177    let timestamp = std::time::SystemTime::now()
178        .duration_since(std::time::UNIX_EPOCH)
179        .unwrap_or_default()
180        .as_secs();
181
182    let prov = provenance_quin(&file_path.to_string_lossy(), typology, timestamp);
183    wal.append_mutation(&prov)
184        .map_err(|e| VisionError::Wal(e.to_string()))?;
185
186    for q in &vision_quins {
187        wal.append_mutation(q)
188            .map_err(|e| VisionError::Wal(e.to_string()))?;
189    }
190
191    let facet = facet_for_typology(typology, &image_sha256, manifest.architecture.as_deref());
192
193    Ok(VisionIngestResult {
194        status: "success".to_string(),
195        file: file_path.to_string_lossy().into_owned(),
196        typology: typology.to_string(),
197        lexicon_id,
198        image_sha256,
199        model_id: active.model_id.clone(),
200        mmproj_path: mmproj_path.to_string_lossy().into_owned(),
201        architecture: manifest.architecture.clone(),
202        facet,
203        wal_path: wal_path.to_string_lossy().into_owned(),
204        vision_quins_appended: vision_quins.len(),
205    })
206}
207
208pub fn ingest_image_with_active_record(
209    storage_root: &Path,
210    active: Option<ActiveModelRecord>,
211    file_path: &Path,
212    typology: &str,
213) -> Result<VisionIngestResult, VisionError> {
214    let active = active.ok_or(VisionError::NoActiveModel)?;
215    ingest_image_file(storage_root, &active, file_path, typology)
216}
217
218// ── V6 native detector / epistemic observation path (no mmproj) ─────────────
219
220/// Fixed-layout detection for WAL packing (mirrors `qualia_vision::Detection` fields).
221#[derive(Debug, Clone, Copy)]
222pub struct NativeDetection {
223    pub class_hash: u64,
224    pub instance_hash: u64,
225    pub score_u16: u16,
226    pub x_min_u16: u16,
227    pub y_min_u16: u16,
228    pub x_max_u16: u16,
229    pub y_max_u16: u16,
230    pub frame_index: u32,
231    pub track_id: u32,
232    pub flags: u32,
233}
234
235const P_VISUAL_OBSERVATION: &str = "https://ns.webizen.org/q42/VisualObservation";
236const P_PROPOSES_CLASS: &str = "https://ns.webizen.org/q42/proposesClass";
237const P_HAS_BBOX: &str = "https://ns.webizen.org/q42/hasBoundingBox";
238const P_HAS_TRACK: &str = "https://ns.webizen.org/q42/hasTrackId";
239const P_MODEL_DIGEST: &str = "https://ns.webizen.org/q42/modelDigest";
240const P_HUMAN_REJECTS: &str = "https://ns.webizen.org/q42/humanRejects";
241const P_HUMAN_CORRECTS: &str = "https://ns.webizen.org/q42/humanCorrectsClass";
242const CTX_VISION: &str = "https://ns.webizen.org/q42/vision-observation";
243const CTX_HUMAN: &str = "https://ns.webizen.org/q42/human-attestation";
244
245#[inline]
246fn quin(s: u64, p: u64, o: u64, c: u64, m: u64) -> NQuin {
247    NQuin {
248        subject: s,
249        predicate: p,
250        object: o,
251        context: c,
252        metadata: m,
253        parity: s ^ p ^ o ^ c ^ m,
254    }
255}
256
257#[inline]
258fn pack_bbox(d: &NativeDetection) -> u64 {
259    (d.x_min_u16 as u64)
260        | ((d.y_min_u16 as u64) << 16)
261        | ((d.x_max_u16 as u64) << 32)
262        | ((d.y_max_u16 as u64) << 48)
263}
264
265/// Compile native detections into epistemic NQuins (caller buffer).
266/// Writes model digest + per-det observation/class/bbox/track. Returns count.
267pub fn compile_native_observation_quins(
268    media_hash: u64,
269    media_byte_len: u64,
270    model_hash: u64,
271    detections: &[NativeDetection],
272    out: &mut [NQuin],
273) -> Result<usize, VisionError> {
274    if out.is_empty() {
275        return Err(VisionError::Buffer("empty out".into()));
276    }
277    let mut w = 0usize;
278    out[w] = quin(
279        media_hash,
280        q_hash(P_MODEL_DIGEST),
281        model_hash,
282        q_hash(CTX_VISION),
283        media_byte_len,
284    );
285    w += 1;
286    let ctx = q_hash(CTX_VISION) ^ model_hash;
287    for d in detections {
288        if d.class_hash == 0 && d.score_u16 == 0 {
289            continue;
290        }
291        if w + 4 > out.len() {
292            break;
293        }
294        let meta_score =
295            (d.score_u16 as u64) | ((d.frame_index as u64) << 16) | ((d.flags as u64) << 48);
296        out[w] = quin(
297            media_hash,
298            q_hash(P_VISUAL_OBSERVATION),
299            d.instance_hash,
300            ctx,
301            meta_score,
302        );
303        w += 1;
304        out[w] = quin(
305            d.instance_hash,
306            q_hash(P_PROPOSES_CLASS),
307            d.class_hash,
308            ctx,
309            d.score_u16 as u64,
310        );
311        w += 1;
312        out[w] = quin(
313            d.instance_hash,
314            q_hash(P_HAS_BBOX),
315            pack_bbox(d),
316            ctx,
317            d.score_u16 as u64 | ((d.frame_index as u64) << 16),
318        );
319        w += 1;
320        out[w] = quin(
321            d.instance_hash,
322            q_hash(P_HAS_TRACK),
323            d.track_id as u64,
324            ctx,
325            d.frame_index as u64,
326        );
327        w += 1;
328    }
329    Ok(w)
330}
331
332/// Human reject of a machine instance — does **not** erase machine quins.
333pub fn human_reject_quin(human_did_hash: u64, instance_hash: u64, reason_hash: u64) -> NQuin {
334    quin(
335        human_did_hash,
336        q_hash(P_HUMAN_REJECTS),
337        instance_hash,
338        q_hash(CTX_HUMAN),
339        reason_hash,
340    )
341}
342
343/// Human class correction — coexists with machine `proposesClass`.
344pub fn human_correct_quin(human_did_hash: u64, instance_hash: u64, new_class_hash: u64) -> NQuin {
345    quin(
346        human_did_hash,
347        q_hash(P_HUMAN_CORRECTS),
348        new_class_hash,
349        q_hash(CTX_HUMAN) ^ instance_hash,
350        instance_hash,
351    )
352}
353
354/// Append native observation quins to `vision_native.wal` (no active VLM required).
355pub fn append_native_observation_quins(
356    storage_root: &Path,
357    media_hash: u64,
358    media_byte_len: u64,
359    model_hash: u64,
360    detections: &[NativeDetection],
361) -> Result<usize, VisionError> {
362    let mut buf = [NQuin {
363        subject: 0,
364        predicate: 0,
365        object: 0,
366        context: 0,
367        metadata: 0,
368        parity: 0,
369    }; 256];
370    let n = compile_native_observation_quins(
371        media_hash,
372        media_byte_len,
373        model_hash,
374        detections,
375        &mut buf,
376    )?;
377    let wal_path = model_lifecycle::models_dir(storage_root).join("vision_native.wal");
378    if let Some(parent) = wal_path.parent() {
379        std::fs::create_dir_all(parent)?;
380    }
381    let mut wal = WriteAheadLog::open(&wal_path)
382        .map_err(|e| VisionError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
383    for q in buf.iter().take(n) {
384        wal.append_mutation(q)
385            .map_err(|e| VisionError::Wal(e.to_string()))?;
386    }
387    Ok(n)
388}
389
390/// Append a human reject/correct edge (provenance preserved).
391pub fn append_human_attestation(storage_root: &Path, quin: &NQuin) -> Result<(), VisionError> {
392    let wal_path = model_lifecycle::models_dir(storage_root).join("vision_native.wal");
393    if let Some(parent) = wal_path.parent() {
394        std::fs::create_dir_all(parent)?;
395    }
396    let mut wal = WriteAheadLog::open(&wal_path)
397        .map_err(|e| VisionError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
398    wal.append_mutation(quin)
399        .map_err(|e| VisionError::Wal(e.to_string()))?;
400    Ok(())
401}
402
403#[cfg(test)]
404mod tests {
405    use super::*;
406
407    #[test]
408    fn native_compile_four_per_det_plus_digest() {
409        let d = NativeDetection {
410            class_hash: 1,
411            instance_hash: 2,
412            score_u16: 1000,
413            x_min_u16: 0,
414            y_min_u16: 0,
415            x_max_u16: 100,
416            y_max_u16: 100,
417            frame_index: 0,
418            track_id: 3,
419            flags: 0,
420        };
421        let mut out = [NQuin {
422            subject: 0,
423            predicate: 0,
424            object: 0,
425            context: 0,
426            metadata: 0,
427            parity: 0,
428        }; 16];
429        let n = compile_native_observation_quins(9, 64, 7, &[d], &mut out).unwrap();
430        assert_eq!(n, 5);
431        assert_eq!(out[0].predicate, q_hash(P_MODEL_DIGEST));
432        let rej = human_reject_quin(0xD1D, 2, 0);
433        assert_eq!(rej.predicate, q_hash(P_HUMAN_REJECTS));
434        // Machine class proposal still in buffer.
435        assert!(out[..n]
436            .iter()
437            .any(|q| q.predicate == q_hash(P_PROPOSES_CLASS)));
438    }
439}
qualia_client_core/vision_ingest.rs

qualia_client_core/
vision_ingest.rs
