Skip to main content

qualia_core_db/q42/
model_helper.rs

1//! Model metadata for a converted `.p64`, stored as a canonical unified `.q42` v3 volume.
2//!
3//! GGUF/safetensors remain import-only. A native model package consists of:
4//! - `.p64` — weights plus the compact Q42T tokenizer section;
5//! - `.q42` — behavioural metadata and provenance represented as NQuins with an
6//!   embedded Q42LEX lexicon.
7//!
8//! CBOR-LD encode/decode remains available as an interchange projection. It is not
9//! the on-disk `.q42` representation.
10
11use serde::{Deserialize, Serialize};
12use std::path::Path;
13
14/// Canonical file extension for the model metadata volume.
15pub const MODEL_HELPER_EXT: &str = "q42";
16/// Former raw self-describe-CBOR sidecar, accepted read-only during migration.
17pub const LEGACY_MODEL_HELPER_EXT: &str = "q42.cbor-ld";
18
19/// JSON-LD / CBOR-LD context IRI for this document type.
20pub const MODEL_HELPER_CONTEXT: &str = "https://webizen.org/ns/qualia/model-helper/v1";
21
22/// Behavioural + provenance metadata for a converted model package.
23#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
24pub struct ModelHelper {
25    #[serde(rename = "@context")]
26    pub context: String,
27    #[serde(rename = "@type")]
28    pub type_: String,
29    pub format: String,
30    pub source_gguf: String,
31    pub p64: String,
32    pub page_log2: u16,
33    pub layout: String,
34    pub converted_unix_ms: u64,
35    pub tokenizer: ModelHelperTokenizer,
36    pub notes: Vec<String>,
37}
38
39#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
40pub struct ModelHelperTokenizer {
41    pub bos_token_id: u32,
42    pub eos_token_id: u32,
43    pub add_bos_token: bool,
44    /// `ChatMl` / `Llama3` / `None` (unsupported families are not release candidates).
45    pub chat_family: String,
46    pub stop_token_ids: Vec<u32>,
47    pub stop_token_strings: Vec<String>,
48    pub vocab_len: u32,
49}
50
51impl ModelHelper {
52    /// Creates a default `ModelHelper`.
53    pub fn new(
54        source_gguf: impl Into<String>,
55        p64_path: impl Into<String>,
56        page_log2: u16,
57        layout: impl Into<String>,
58        tokenizer: ModelHelperTokenizer,
59    ) -> Self {
60        let converted_unix_ms = std::time::SystemTime::now()
61            .duration_since(std::time::UNIX_EPOCH)
62            .map(|d| d.as_millis() as u64)
63            .unwrap_or(0);
64        Self {
65            context: MODEL_HELPER_CONTEXT.to_string(),
66            type_: "QualiaModelHelper".to_string(),
67            format: "qualia.q42.model-helper.v1".to_string(),
68            source_gguf: source_gguf.into(),
69            p64: p64_path.into(),
70            page_log2,
71            layout: layout.into(),
72            converted_unix_ms,
73            tokenizer,
74            notes: vec![
75                "verbatim = GGML quant blocks preserved (same decode kernels as GGUF).".into(),
76                "Activate the .p64 path; keep GGUF as an import-only archive.".into(),
77                "Metadata is a canonical unified Q42 v3 volume.".into(),
78            ],
79        }
80    }
81
82    /// Interchange projection: encode as self-describe CBOR (tag 55799).
83    pub fn to_cbor_ld(&self) -> Result<Vec<u8>, String> {
84        let mut body = Vec::new();
85        ciborium::into_writer(self, &mut body).map_err(|e| format!("cbor encode: {e}"))?;
86        let mut out = Vec::with_capacity(body.len() + 3);
87        out.extend_from_slice(&[0xd9, 0xd9, 0xf7]);
88        out.extend_from_slice(&body);
89        Ok(out)
90    }
91
92    /// Interchange/migration projection: decode self-describe CBOR or a plain CBOR map.
93    pub fn from_cbor_ld(bytes: &[u8]) -> Result<Self, String> {
94        let payload = strip_self_describe_tag(bytes);
95        ciborium::from_reader(payload).map_err(|e| format!("cbor decode: {e}"))
96    }
97
98    /// Write `{stem}.q42` as a canonical unified Q42 v3 volume.
99    #[cfg(not(target_arch = "wasm32"))]
100    pub fn write_beside_p64(&self, p64_path: &Path) -> Result<std::path::PathBuf, String> {
101        use crate::q42_volume::UnifiedVolumeBuilder;
102
103        let helper_path = helper_path_for_p64(p64_path);
104        let (lex, mut quins) = self.to_q42_graph();
105        // Unified-volume BIDX ranges and FLAG_OBJECT_SORTED require object order.
106        quins.sort_unstable_by_key(|q| q.object);
107        let mut builder = UnifiedVolumeBuilder::with_lex_map(&lex)
108            .map_err(|e| format!("build Q42LEX for helper: {e:?}"))?;
109        builder
110            .push_block(0, &quins)
111            .map_err(|e| format!("build canonical Q42 helper: {e}"))?;
112        builder
113            .finish(&helper_path)
114            .map_err(|e| format!("write {}: {e}", helper_path.display()))?;
115        Ok(helper_path)
116    }
117
118    #[cfg(target_arch = "wasm32")]
119    /// Writes `beside-p64`.
120    pub fn write_beside_p64(&self, _p64_path: &Path) -> Result<std::path::PathBuf, String> {
121        Err("canonical Q42 model metadata is written by the native conversion tool".into())
122    }
123
124    /// Load the sibling canonical `.q42`; accept the former raw-CBOR name read-only.
125    #[cfg(not(target_arch = "wasm32"))]
126    pub fn load_beside_p64(p64_path: &Path) -> Result<Option<Self>, String> {
127        let helper_path = helper_path_for_p64(p64_path);
128        if helper_path.is_file() {
129            return Self::from_q42_path(&helper_path).map(Some);
130        }
131
132        let legacy_path = legacy_helper_path_for_p64(p64_path);
133        if !legacy_path.is_file() {
134            return Ok(None);
135        }
136        let bytes = std::fs::read(&legacy_path)
137            .map_err(|e| format!("read {}: {e}", legacy_path.display()))?;
138        Ok(Some(Self::from_cbor_ld(&bytes)?))
139    }
140
141    #[cfg(target_arch = "wasm32")]
142    /// Loads `beside-p64`.
143    pub fn load_beside_p64(_p64_path: &Path) -> Result<Option<Self>, String> {
144        Ok(None)
145    }
146
147    #[cfg(not(target_arch = "wasm32"))]
148    fn to_q42_graph(&self) -> (std::collections::HashMap<u64, String>, Vec<crate::NQuin>) {
149        use crate::frame_layout::{INLINE_TAG_BOOLEAN, INLINE_TAG_INTEGER, INLINE_VALUE_MASK};
150        use crate::q_hash;
151        use std::collections::HashMap;
152
153        let mut lex = HashMap::new();
154        let subject = q_hash(&self.p64);
155        let context = q_hash(MODEL_HELPER_CONTEXT);
156        insert_lex(&mut lex, context, MODEL_HELPER_CONTEXT);
157
158        let mut quins = Vec::new();
159        let mut string_edge = |predicate_iri: &str, value: &str, metadata: u64| {
160            let predicate = q_hash(predicate_iri);
161            let object = q_hash(value) & INLINE_VALUE_MASK;
162            insert_lex(&mut lex, predicate, predicate_iri);
163            insert_lex(&mut lex, object, value);
164            quins.push(make_quin(subject, predicate, object, context, metadata));
165        };
166
167        string_edge("rdf:type", "q42:QualiaModelHelper", 0);
168        string_edge("q42:format", &self.format, 0);
169        string_edge("q42:sourceGguf", &self.source_gguf, 0);
170        string_edge("q42:p64Asset", &self.p64, 0);
171        string_edge("q42:layout", &self.layout, 0);
172        string_edge("q42:chatFamily", &self.tokenizer.chat_family, 0);
173        for (i, value) in self.tokenizer.stop_token_strings.iter().enumerate() {
174            string_edge("q42:stopTokenString", value, i as u64);
175        }
176        for (i, value) in self.notes.iter().enumerate() {
177            string_edge("q42:note", value, i as u64);
178        }
179        drop(string_edge);
180
181        let mut integer_edge = |predicate_iri: &str, value: u64, metadata: u64| {
182            let predicate = q_hash(predicate_iri);
183            insert_lex(&mut lex, predicate, predicate_iri);
184            quins.push(make_quin(
185                subject,
186                predicate,
187                INLINE_TAG_INTEGER | (value & INLINE_VALUE_MASK),
188                context,
189                metadata,
190            ));
191        };
192        integer_edge("q42:pageLog2", self.page_log2 as u64, 0);
193        integer_edge("q42:convertedUnixMs", self.converted_unix_ms, 0);
194        integer_edge("q42:bosTokenId", self.tokenizer.bos_token_id as u64, 0);
195        integer_edge("q42:eosTokenId", self.tokenizer.eos_token_id as u64, 0);
196        integer_edge("q42:vocabLen", self.tokenizer.vocab_len as u64, 0);
197        for (i, value) in self.tokenizer.stop_token_ids.iter().enumerate() {
198            integer_edge("q42:stopTokenId", *value as u64, i as u64);
199        }
200        drop(integer_edge);
201
202        let predicate = q_hash("q42:addBosToken");
203        insert_lex(&mut lex, predicate, "q42:addBosToken");
204        quins.push(make_quin(
205            subject,
206            predicate,
207            INLINE_TAG_BOOLEAN | u64::from(self.tokenizer.add_bos_token),
208            context,
209            0,
210        ));
211
212        (lex, quins)
213    }
214
215    #[cfg(not(target_arch = "wasm32"))]
216    fn from_q42_path(path: &Path) -> Result<Self, String> {
217        use crate::frame_layout::{
218            INLINE_TAG_BOOLEAN, INLINE_TAG_INTEGER, INLINE_TAG_MASK, INLINE_VALUE_MASK,
219        };
220        use crate::q42_volume::Q42Volume;
221        use crate::q_hash;
222        use std::collections::BTreeMap;
223
224        let volume = Q42Volume::open(path)
225            .map_err(|e| format!("open canonical Q42 helper {}: {e}", path.display()))?;
226        volume
227            .header()
228            .verify_version()
229            .map_err(|e| format!("invalid Q42 helper {}: {e}", path.display()))?;
230        let lex = volume
231            .lex_view()
232            .map_err(|e| format!("invalid Q42LEX in {}: {e:?}", path.display()))?;
233        let quins = volume
234            .read_all_quins()
235            .map_err(|e| format!("read Q42 helper {}: {e}", path.display()))?;
236
237        let text = |object: u64| -> Result<String, String> {
238            if let Some(identity) = lex.lookup_webizen_identity(object) {
239                return Ok(identity.to_owned());
240            }
241            let mut term = String::new();
242            if lex.resolve_term_into(object, &mut term) {
243                Ok(term)
244            } else {
245                Err(format!(
246                    "Q42 helper has unresolved lexicon hash {object:#018x}"
247                ))
248            }
249        };
250        let integer = |object: u64| -> Result<u64, String> {
251            if object & INLINE_TAG_MASK != INLINE_TAG_INTEGER {
252                return Err(format!(
253                    "Q42 helper expected integer object, got {object:#018x}"
254                ));
255            }
256            Ok(object & INLINE_VALUE_MASK)
257        };
258
259        let mut format = None;
260        let mut source_gguf = None;
261        let mut p64 = None;
262        let mut page_log2 = None;
263        let mut layout = None;
264        let mut converted_unix_ms = None;
265        let mut bos_token_id = None;
266        let mut eos_token_id = None;
267        let mut add_bos_token = None;
268        let mut chat_family = None;
269        let mut vocab_len = None;
270        let mut stop_ids = BTreeMap::new();
271        let mut stop_strings = BTreeMap::new();
272        let mut notes = BTreeMap::new();
273        let mut saw_type = false;
274
275        for q in quins {
276            match q.predicate {
277                p if p == q_hash("rdf:type") => {
278                    saw_type = text(q.object)? == "q42:QualiaModelHelper";
279                }
280                p if p == q_hash("q42:format") => format = Some(text(q.object)?),
281                p if p == q_hash("q42:sourceGguf") => source_gguf = Some(text(q.object)?),
282                p if p == q_hash("q42:p64Asset") => p64 = Some(text(q.object)?),
283                p if p == q_hash("q42:layout") => layout = Some(text(q.object)?),
284                p if p == q_hash("q42:chatFamily") => chat_family = Some(text(q.object)?),
285                p if p == q_hash("q42:stopTokenString") => {
286                    stop_strings.insert(q.metadata, text(q.object)?);
287                }
288                p if p == q_hash("q42:note") => {
289                    notes.insert(q.metadata, text(q.object)?);
290                }
291                p if p == q_hash("q42:pageLog2") => page_log2 = Some(integer(q.object)? as u16),
292                p if p == q_hash("q42:convertedUnixMs") => {
293                    converted_unix_ms = Some(integer(q.object)?)
294                }
295                p if p == q_hash("q42:bosTokenId") => {
296                    bos_token_id = Some(integer(q.object)? as u32)
297                }
298                p if p == q_hash("q42:eosTokenId") => {
299                    eos_token_id = Some(integer(q.object)? as u32)
300                }
301                p if p == q_hash("q42:vocabLen") => vocab_len = Some(integer(q.object)? as u32),
302                p if p == q_hash("q42:stopTokenId") => {
303                    stop_ids.insert(q.metadata, integer(q.object)? as u32);
304                }
305                p if p == q_hash("q42:addBosToken") => {
306                    if q.object & INLINE_TAG_MASK != INLINE_TAG_BOOLEAN {
307                        return Err("Q42 helper addBosToken is not a boolean".into());
308                    }
309                    add_bos_token = Some(q.object & 1 != 0);
310                }
311                _ => {}
312            }
313        }
314
315        if !saw_type {
316            return Err("Q42 volume is not a QualiaModelHelper".into());
317        }
318        let required = |name: &str| format!("Q42 model helper missing {name}");
319        Ok(Self {
320            context: MODEL_HELPER_CONTEXT.to_string(),
321            type_: "QualiaModelHelper".to_string(),
322            format: format.ok_or_else(|| required("format"))?,
323            source_gguf: source_gguf.ok_or_else(|| required("sourceGguf"))?,
324            p64: p64.ok_or_else(|| required("p64Asset"))?,
325            page_log2: page_log2.ok_or_else(|| required("pageLog2"))?,
326            layout: layout.ok_or_else(|| required("layout"))?,
327            converted_unix_ms: converted_unix_ms.ok_or_else(|| required("convertedUnixMs"))?,
328            tokenizer: ModelHelperTokenizer {
329                bos_token_id: bos_token_id.ok_or_else(|| required("bosTokenId"))?,
330                eos_token_id: eos_token_id.ok_or_else(|| required("eosTokenId"))?,
331                add_bos_token: add_bos_token.ok_or_else(|| required("addBosToken"))?,
332                chat_family: chat_family.ok_or_else(|| required("chatFamily"))?,
333                stop_token_ids: stop_ids.into_values().collect(),
334                stop_token_strings: stop_strings.into_values().collect(),
335                vocab_len: vocab_len.ok_or_else(|| required("vocabLen"))?,
336            },
337            notes: notes.into_values().collect(),
338        })
339    }
340
341    /// Merge stop-token ids from this helper into a loaded tokenizer.
342    pub fn apply_stops_to_tokenizer(&self, tok: &mut crate::gguf_sharder::GgufTokenizer) {
343        tok.merge_stop_token_ids(&self.tokenizer.stop_token_ids);
344    }
345}
346
347#[cfg(not(target_arch = "wasm32"))]
348fn insert_lex(lex: &mut std::collections::HashMap<u64, String>, hash: u64, value: &str) {
349    lex.insert(hash, value.to_string());
350}
351
352#[cfg(not(target_arch = "wasm32"))]
353fn make_quin(
354    subject: u64,
355    predicate: u64,
356    object: u64,
357    context: u64,
358    metadata: u64,
359) -> crate::NQuin {
360    crate::NQuin {
361        subject,
362        predicate,
363        object,
364        context,
365        metadata,
366        // Five-field ECC: same fold as NQuin::calculate_parity (metadata included).
367        parity: crate::NQuin::calculate_parity(subject, predicate, object, context, metadata),
368    }
369}
370
371/// `{dir}/{stem}.q42` for a given `.p64` path (preserves layout suffixes in the stem).
372pub fn helper_path_for_p64(p64_path: &Path) -> std::path::PathBuf {
373    helper_path_with_ext(p64_path, MODEL_HELPER_EXT)
374}
375
376/// `ModelHelper::legacy_helper_path_for_p64`.
377pub fn legacy_helper_path_for_p64(p64_path: &Path) -> std::path::PathBuf {
378    helper_path_with_ext(p64_path, LEGACY_MODEL_HELPER_EXT)
379}
380
381fn helper_path_with_ext(p64_path: &Path, ext: &str) -> std::path::PathBuf {
382    let parent = p64_path.parent().unwrap_or_else(|| Path::new("."));
383    let stem = p64_path
384        .file_stem()
385        .and_then(|s| s.to_str())
386        .unwrap_or("model");
387    parent.join(format!("{stem}.{ext}"))
388}
389
390fn strip_self_describe_tag(bytes: &[u8]) -> &[u8] {
391    if bytes.starts_with(&[0xd9, 0xd9, 0xf7]) {
392        &bytes[3..]
393    } else {
394        bytes
395    }
396}
397
398/// Magic sniff for the legacy CBOR-LD interchange projection.
399pub fn has_model_helper_magic(bytes: &[u8]) -> bool {
400    bytes.starts_with(&[0xd9, 0xd9, 0xf7])
401}
402
403#[cfg(test)]
404mod tests {
405    use super::*;
406
407    fn sample() -> ModelHelper {
408        ModelHelper::new(
409            "x.gguf",
410            "x.p64",
411            14,
412            "Verbatim",
413            ModelHelperTokenizer {
414                bos_token_id: 1,
415                eos_token_id: 2,
416                add_bos_token: true,
417                chat_family: "Llama3".into(),
418                stop_token_ids: vec![2, 128_009],
419                stop_token_strings: vec!["</s>".into(), "<|eot_id|>".into()],
420                vocab_len: 128_256,
421            },
422        )
423    }
424
425    #[test]
426    fn cbor_ld_interchange_round_trip() {
427        let h = sample();
428        let bytes = h.to_cbor_ld().expect("encode");
429        assert!(has_model_helper_magic(&bytes));
430        assert_eq!(ModelHelper::from_cbor_ld(&bytes).unwrap(), h);
431    }
432
433    #[cfg(not(target_arch = "wasm32"))]
434    #[test]
435    fn canonical_q42_round_trip_preserves_model_metadata() {
436        use crate::q42_volume::{Q42Volume, Q42_MAGIC, Q42_VERSION_V4};
437
438        let dir = tempfile::tempdir().unwrap();
439        let p64 = dir.path().join("x.p64");
440        std::fs::write(&p64, b"p64\0").unwrap();
441        let h = sample();
442        let path = h.write_beside_p64(&p64).unwrap();
443        assert_eq!(path.extension().and_then(|x| x.to_str()), Some("q42"));
444        assert!(std::fs::read(&path).unwrap().starts_with(&Q42_MAGIC));
445        let volume = Q42Volume::open(&path).unwrap();
446        assert_eq!({ volume.header().version }, Q42_VERSION_V4);
447        volume
448            .verify_all_blocks()
449            .expect("canonical helper must pass five-field ECC + BIDX");
450        assert!(volume.header().flags & crate::q42_volume::FLAG_FIELD_POSTINGS != 0);
451        assert!(volume.header().flags & crate::q42_volume::FLAG_FIELD_RANGES != 0);
452        assert_eq!(ModelHelper::load_beside_p64(&p64).unwrap(), Some(h));
453    }
454
455    #[test]
456    fn helper_path_is_plain_q42() {
457        let p = Path::new(r"C:\LLM_Models\P64\smollm2.f16.p64");
458        assert!(helper_path_for_p64(p)
459            .to_string_lossy()
460            .ends_with("smollm2.f16.q42"));
461    }
462}