qualia_core_db/q42/
model_helper.rs

1//! Model metadata for a converted `.p64`, stored as a canonical unified `.q42` v3 volume.
2//!
3//! GGUF/safetensors remain import-only. A native model package consists of:
4//! - `.p64` — weights plus the compact Q42T tokenizer section;
5//! - `.q42` — behavioural metadata and provenance represented as NQuins with an
6//!   embedded Q42LEX lexicon.
7//!
8//! CBOR-LD encode/decode remains available as an interchange projection. It is not
9//! the on-disk `.q42` representation.
10
11use serde::{Deserialize, Serialize};
12use std::path::Path;
13
14/// Canonical file extension for the model metadata volume.
15pub const MODEL_HELPER_EXT: &str = "q42";
16/// Former raw self-describe-CBOR sidecar, accepted read-only during migration.
17pub const LEGACY_MODEL_HELPER_EXT: &str = "q42.cbor-ld";
18
19/// JSON-LD / CBOR-LD context IRI for this document type.
20pub const MODEL_HELPER_CONTEXT: &str = "https://webizen.org/ns/qualia/model-helper/v1";
21
22/// Behavioural + provenance metadata for a converted model package.
23#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
24pub struct ModelHelper {
25    #[serde(rename = "@context")]
26    pub context: String,
27    #[serde(rename = "@type")]
28    pub type_: String,
29    pub format: String,
30    pub source_gguf: String,
31    pub p64: String,
32    pub page_log2: u16,
33    pub layout: String,
34    pub converted_unix_ms: u64,
35    pub tokenizer: ModelHelperTokenizer,
36    pub notes: Vec<String>,
37}
38
39#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
40pub struct ModelHelperTokenizer {
41    pub bos_token_id: u32,
42    pub eos_token_id: u32,
43    pub add_bos_token: bool,
44    /// `ChatMl` / `Llama3` / `None` (unsupported families are not release candidates).
45    pub chat_family: String,
46    pub stop_token_ids: Vec<u32>,
47    pub stop_token_strings: Vec<String>,
48    pub vocab_len: u32,
49}
50
51impl ModelHelper {
52    pub fn new(
53        source_gguf: impl Into<String>,
54        p64_path: impl Into<String>,
55        page_log2: u16,
56        layout: impl Into<String>,
57        tokenizer: ModelHelperTokenizer,
58    ) -> Self {
59        let converted_unix_ms = std::time::SystemTime::now()
60            .duration_since(std::time::UNIX_EPOCH)
61            .map(|d| d.as_millis() as u64)
62            .unwrap_or(0);
63        Self {
64            context: MODEL_HELPER_CONTEXT.to_string(),
65            type_: "QualiaModelHelper".to_string(),
66            format: "qualia.q42.model-helper.v1".to_string(),
67            source_gguf: source_gguf.into(),
68            p64: p64_path.into(),
69            page_log2,
70            layout: layout.into(),
71            converted_unix_ms,
72            tokenizer,
73            notes: vec![
74                "verbatim = GGML quant blocks preserved (same decode kernels as GGUF).".into(),
75                "Activate the .p64 path; keep GGUF as an import-only archive.".into(),
76                "Metadata is a canonical unified Q42 v3 volume.".into(),
77            ],
78        }
79    }
80
81    /// Interchange projection: encode as self-describe CBOR (tag 55799).
82    pub fn to_cbor_ld(&self) -> Result<Vec<u8>, String> {
83        let mut body = Vec::new();
84        ciborium::into_writer(self, &mut body).map_err(|e| format!("cbor encode: {e}"))?;
85        let mut out = Vec::with_capacity(body.len() + 3);
86        out.extend_from_slice(&[0xd9, 0xd9, 0xf7]);
87        out.extend_from_slice(&body);
88        Ok(out)
89    }
90
91    /// Interchange/migration projection: decode self-describe CBOR or a plain CBOR map.
92    pub fn from_cbor_ld(bytes: &[u8]) -> Result<Self, String> {
93        let payload = strip_self_describe_tag(bytes);
94        ciborium::from_reader(payload).map_err(|e| format!("cbor decode: {e}"))
95    }
96
97    /// Write `{stem}.q42` as a canonical unified Q42 v3 volume.
98    #[cfg(not(target_arch = "wasm32"))]
99    pub fn write_beside_p64(&self, p64_path: &Path) -> Result<std::path::PathBuf, String> {
100        use crate::q42_volume::UnifiedVolumeBuilder;
101
102        let helper_path = helper_path_for_p64(p64_path);
103        let (lex, mut quins) = self.to_q42_graph();
104        // Unified-volume BIDX ranges and FLAG_OBJECT_SORTED require object order.
105        quins.sort_unstable_by_key(|q| q.object);
106        let mut builder = UnifiedVolumeBuilder::with_lex_map(&lex)
107            .map_err(|e| format!("build Q42LEX for helper: {e:?}"))?;
108        builder
109            .push_block(0, &quins)
110            .map_err(|e| format!("build canonical Q42 helper: {e}"))?;
111        builder
112            .finish(&helper_path)
113            .map_err(|e| format!("write {}: {e}", helper_path.display()))?;
114        Ok(helper_path)
115    }
116
117    #[cfg(target_arch = "wasm32")]
118    pub fn write_beside_p64(&self, _p64_path: &Path) -> Result<std::path::PathBuf, String> {
119        Err("canonical Q42 model metadata is written by the native conversion tool".into())
120    }
121
122    /// Load the sibling canonical `.q42`; accept the former raw-CBOR name read-only.
123    #[cfg(not(target_arch = "wasm32"))]
124    pub fn load_beside_p64(p64_path: &Path) -> Result<Option<Self>, String> {
125        let helper_path = helper_path_for_p64(p64_path);
126        if helper_path.is_file() {
127            return Self::from_q42_path(&helper_path).map(Some);
128        }
129
130        let legacy_path = legacy_helper_path_for_p64(p64_path);
131        if !legacy_path.is_file() {
132            return Ok(None);
133        }
134        let bytes = std::fs::read(&legacy_path)
135            .map_err(|e| format!("read {}: {e}", legacy_path.display()))?;
136        Ok(Some(Self::from_cbor_ld(&bytes)?))
137    }
138
139    #[cfg(target_arch = "wasm32")]
140    pub fn load_beside_p64(_p64_path: &Path) -> Result<Option<Self>, String> {
141        Ok(None)
142    }
143
144    #[cfg(not(target_arch = "wasm32"))]
145    fn to_q42_graph(&self) -> (std::collections::HashMap<u64, String>, Vec<crate::NQuin>) {
146        use crate::frame_layout::{INLINE_TAG_BOOLEAN, INLINE_TAG_INTEGER, INLINE_VALUE_MASK};
147        use crate::q_hash;
148        use std::collections::HashMap;
149
150        let mut lex = HashMap::new();
151        let subject = q_hash(&self.p64);
152        let context = q_hash(MODEL_HELPER_CONTEXT);
153        insert_lex(&mut lex, context, MODEL_HELPER_CONTEXT);
154
155        let mut quins = Vec::new();
156        let mut string_edge = |predicate_iri: &str, value: &str, metadata: u64| {
157            let predicate = q_hash(predicate_iri);
158            let object = q_hash(value) & INLINE_VALUE_MASK;
159            insert_lex(&mut lex, predicate, predicate_iri);
160            insert_lex(&mut lex, object, value);
161            quins.push(make_quin(subject, predicate, object, context, metadata));
162        };
163
164        string_edge("rdf:type", "q42:QualiaModelHelper", 0);
165        string_edge("q42:format", &self.format, 0);
166        string_edge("q42:sourceGguf", &self.source_gguf, 0);
167        string_edge("q42:p64Asset", &self.p64, 0);
168        string_edge("q42:layout", &self.layout, 0);
169        string_edge("q42:chatFamily", &self.tokenizer.chat_family, 0);
170        for (i, value) in self.tokenizer.stop_token_strings.iter().enumerate() {
171            string_edge("q42:stopTokenString", value, i as u64);
172        }
173        for (i, value) in self.notes.iter().enumerate() {
174            string_edge("q42:note", value, i as u64);
175        }
176        drop(string_edge);
177
178        let mut integer_edge = |predicate_iri: &str, value: u64, metadata: u64| {
179            let predicate = q_hash(predicate_iri);
180            insert_lex(&mut lex, predicate, predicate_iri);
181            quins.push(make_quin(
182                subject,
183                predicate,
184                INLINE_TAG_INTEGER | (value & INLINE_VALUE_MASK),
185                context,
186                metadata,
187            ));
188        };
189        integer_edge("q42:pageLog2", self.page_log2 as u64, 0);
190        integer_edge("q42:convertedUnixMs", self.converted_unix_ms, 0);
191        integer_edge("q42:bosTokenId", self.tokenizer.bos_token_id as u64, 0);
192        integer_edge("q42:eosTokenId", self.tokenizer.eos_token_id as u64, 0);
193        integer_edge("q42:vocabLen", self.tokenizer.vocab_len as u64, 0);
194        for (i, value) in self.tokenizer.stop_token_ids.iter().enumerate() {
195            integer_edge("q42:stopTokenId", *value as u64, i as u64);
196        }
197        drop(integer_edge);
198
199        let predicate = q_hash("q42:addBosToken");
200        insert_lex(&mut lex, predicate, "q42:addBosToken");
201        quins.push(make_quin(
202            subject,
203            predicate,
204            INLINE_TAG_BOOLEAN | u64::from(self.tokenizer.add_bos_token),
205            context,
206            0,
207        ));
208
209        (lex, quins)
210    }
211
212    #[cfg(not(target_arch = "wasm32"))]
213    fn from_q42_path(path: &Path) -> Result<Self, String> {
214        use crate::frame_layout::{
215            INLINE_TAG_BOOLEAN, INLINE_TAG_INTEGER, INLINE_TAG_MASK, INLINE_VALUE_MASK,
216        };
217        use crate::q42_volume::Q42Volume;
218        use crate::q_hash;
219        use std::collections::BTreeMap;
220
221        let volume = Q42Volume::open(path)
222            .map_err(|e| format!("open canonical Q42 helper {}: {e}", path.display()))?;
223        volume
224            .header()
225            .verify_version()
226            .map_err(|e| format!("invalid Q42 helper {}: {e}", path.display()))?;
227        let lex = volume
228            .lex_view()
229            .map_err(|e| format!("invalid Q42LEX in {}: {e:?}", path.display()))?;
230        let quins = volume
231            .read_all_quins()
232            .map_err(|e| format!("read Q42 helper {}: {e}", path.display()))?;
233
234        let text = |object: u64| -> Result<String, String> {
235            lex.lookup_webizen_identity(object)
236                .or_else(|| lex.lookup_hash(object))
237                .map(str::to_owned)
238                .ok_or_else(|| format!("Q42 helper has unresolved lexicon hash {object:#018x}"))
239        };
240        let integer = |object: u64| -> Result<u64, String> {
241            if object & INLINE_TAG_MASK != INLINE_TAG_INTEGER {
242                return Err(format!(
243                    "Q42 helper expected integer object, got {object:#018x}"
244                ));
245            }
246            Ok(object & INLINE_VALUE_MASK)
247        };
248
249        let mut format = None;
250        let mut source_gguf = None;
251        let mut p64 = None;
252        let mut page_log2 = None;
253        let mut layout = None;
254        let mut converted_unix_ms = None;
255        let mut bos_token_id = None;
256        let mut eos_token_id = None;
257        let mut add_bos_token = None;
258        let mut chat_family = None;
259        let mut vocab_len = None;
260        let mut stop_ids = BTreeMap::new();
261        let mut stop_strings = BTreeMap::new();
262        let mut notes = BTreeMap::new();
263        let mut saw_type = false;
264
265        for q in quins {
266            match q.predicate {
267                p if p == q_hash("rdf:type") => {
268                    saw_type = text(q.object)? == "q42:QualiaModelHelper";
269                }
270                p if p == q_hash("q42:format") => format = Some(text(q.object)?),
271                p if p == q_hash("q42:sourceGguf") => source_gguf = Some(text(q.object)?),
272                p if p == q_hash("q42:p64Asset") => p64 = Some(text(q.object)?),
273                p if p == q_hash("q42:layout") => layout = Some(text(q.object)?),
274                p if p == q_hash("q42:chatFamily") => chat_family = Some(text(q.object)?),
275                p if p == q_hash("q42:stopTokenString") => {
276                    stop_strings.insert(q.metadata, text(q.object)?);
277                }
278                p if p == q_hash("q42:note") => {
279                    notes.insert(q.metadata, text(q.object)?);
280                }
281                p if p == q_hash("q42:pageLog2") => page_log2 = Some(integer(q.object)? as u16),
282                p if p == q_hash("q42:convertedUnixMs") => {
283                    converted_unix_ms = Some(integer(q.object)?)
284                }
285                p if p == q_hash("q42:bosTokenId") => {
286                    bos_token_id = Some(integer(q.object)? as u32)
287                }
288                p if p == q_hash("q42:eosTokenId") => {
289                    eos_token_id = Some(integer(q.object)? as u32)
290                }
291                p if p == q_hash("q42:vocabLen") => vocab_len = Some(integer(q.object)? as u32),
292                p if p == q_hash("q42:stopTokenId") => {
293                    stop_ids.insert(q.metadata, integer(q.object)? as u32);
294                }
295                p if p == q_hash("q42:addBosToken") => {
296                    if q.object & INLINE_TAG_MASK != INLINE_TAG_BOOLEAN {
297                        return Err("Q42 helper addBosToken is not a boolean".into());
298                    }
299                    add_bos_token = Some(q.object & 1 != 0);
300                }
301                _ => {}
302            }
303        }
304
305        if !saw_type {
306            return Err("Q42 volume is not a QualiaModelHelper".into());
307        }
308        let required = |name: &str| format!("Q42 model helper missing {name}");
309        Ok(Self {
310            context: MODEL_HELPER_CONTEXT.to_string(),
311            type_: "QualiaModelHelper".to_string(),
312            format: format.ok_or_else(|| required("format"))?,
313            source_gguf: source_gguf.ok_or_else(|| required("sourceGguf"))?,
314            p64: p64.ok_or_else(|| required("p64Asset"))?,
315            page_log2: page_log2.ok_or_else(|| required("pageLog2"))?,
316            layout: layout.ok_or_else(|| required("layout"))?,
317            converted_unix_ms: converted_unix_ms.ok_or_else(|| required("convertedUnixMs"))?,
318            tokenizer: ModelHelperTokenizer {
319                bos_token_id: bos_token_id.ok_or_else(|| required("bosTokenId"))?,
320                eos_token_id: eos_token_id.ok_or_else(|| required("eosTokenId"))?,
321                add_bos_token: add_bos_token.ok_or_else(|| required("addBosToken"))?,
322                chat_family: chat_family.ok_or_else(|| required("chatFamily"))?,
323                stop_token_ids: stop_ids.into_values().collect(),
324                stop_token_strings: stop_strings.into_values().collect(),
325                vocab_len: vocab_len.ok_or_else(|| required("vocabLen"))?,
326            },
327            notes: notes.into_values().collect(),
328        })
329    }
330
331    /// Merge stop-token ids from this helper into a loaded tokenizer.
332    #[cfg(any(not(target_arch = "wasm32"), feature = "wasm-llm"))]
333    pub fn apply_stops_to_tokenizer(&self, tok: &mut crate::gguf_sharder::GgufTokenizer) {
334        tok.merge_stop_token_ids(&self.tokenizer.stop_token_ids);
335    }
336}
337
338#[cfg(not(target_arch = "wasm32"))]
339fn insert_lex(lex: &mut std::collections::HashMap<u64, String>, hash: u64, value: &str) {
340    lex.insert(hash, value.to_string());
341}
342
343#[cfg(not(target_arch = "wasm32"))]
344fn make_quin(
345    subject: u64,
346    predicate: u64,
347    object: u64,
348    context: u64,
349    metadata: u64,
350) -> crate::NQuin {
351    crate::NQuin {
352        subject,
353        predicate,
354        object,
355        context,
356        metadata,
357        // Five-field ECC: same fold as NQuin::calculate_parity (metadata included).
358        parity: crate::NQuin::calculate_parity(subject, predicate, object, context, metadata),
359    }
360}
361
362/// `{dir}/{stem}.q42` for a given `.p64` path (preserves layout suffixes in the stem).
363pub fn helper_path_for_p64(p64_path: &Path) -> std::path::PathBuf {
364    helper_path_with_ext(p64_path, MODEL_HELPER_EXT)
365}
366
367pub fn legacy_helper_path_for_p64(p64_path: &Path) -> std::path::PathBuf {
368    helper_path_with_ext(p64_path, LEGACY_MODEL_HELPER_EXT)
369}
370
371fn helper_path_with_ext(p64_path: &Path, ext: &str) -> std::path::PathBuf {
372    let parent = p64_path.parent().unwrap_or_else(|| Path::new("."));
373    let stem = p64_path
374        .file_stem()
375        .and_then(|s| s.to_str())
376        .unwrap_or("model");
377    parent.join(format!("{stem}.{ext}"))
378}
379
380fn strip_self_describe_tag(bytes: &[u8]) -> &[u8] {
381    if bytes.starts_with(&[0xd9, 0xd9, 0xf7]) {
382        &bytes[3..]
383    } else {
384        bytes
385    }
386}
387
388/// Magic sniff for the legacy CBOR-LD interchange projection.
389pub fn has_model_helper_magic(bytes: &[u8]) -> bool {
390    bytes.starts_with(&[0xd9, 0xd9, 0xf7])
391}
392
393#[cfg(test)]
394mod tests {
395    use super::*;
396
397    fn sample() -> ModelHelper {
398        ModelHelper::new(
399            "x.gguf",
400            "x.p64",
401            14,
402            "Verbatim",
403            ModelHelperTokenizer {
404                bos_token_id: 1,
405                eos_token_id: 2,
406                add_bos_token: true,
407                chat_family: "Llama3".into(),
408                stop_token_ids: vec![2, 128_009],
409                stop_token_strings: vec!["</s>".into(), "<|eot_id|>".into()],
410                vocab_len: 128_256,
411            },
412        )
413    }
414
415    #[test]
416    fn cbor_ld_interchange_round_trip() {
417        let h = sample();
418        let bytes = h.to_cbor_ld().expect("encode");
419        assert!(has_model_helper_magic(&bytes));
420        assert_eq!(ModelHelper::from_cbor_ld(&bytes).unwrap(), h);
421    }
422
423    #[cfg(not(target_arch = "wasm32"))]
424    #[test]
425    fn canonical_q42_round_trip_preserves_model_metadata() {
426        use crate::q42_volume::{Q42Volume, Q42_MAGIC, Q42_VERSION_V3};
427
428        let dir = tempfile::tempdir().unwrap();
429        let p64 = dir.path().join("x.p64");
430        std::fs::write(&p64, b"p64\0").unwrap();
431        let h = sample();
432        let path = h.write_beside_p64(&p64).unwrap();
433        assert_eq!(path.extension().and_then(|x| x.to_str()), Some("q42"));
434        assert!(std::fs::read(&path).unwrap().starts_with(&Q42_MAGIC));
435        let volume = Q42Volume::open(&path).unwrap();
436        assert_eq!({ volume.header().version }, Q42_VERSION_V3);
437        volume
438            .verify_all_blocks()
439            .expect("canonical helper must pass five-field ECC + BIDX");
440        assert!(volume.header().flags & crate::q42_volume::FLAG_FIELD_POSTINGS != 0);
441        assert!(volume.header().flags & crate::q42_volume::FLAG_FIELD_RANGES != 0);
442        assert_eq!(ModelHelper::load_beside_p64(&p64).unwrap(), Some(h));
443    }
444
445    #[test]
446    fn helper_path_is_plain_q42() {
447        let p = Path::new(r"C:\LLM_Models\P64\smollm2.f16.p64");
448        assert!(helper_path_for_p64(p)
449            .to_string_lossy()
450            .ends_with("smollm2.f16.q42"));
451    }
452}
qualia_core_db/q42/model_helper.rs

qualia_core_db/q42/
model_helper.rs
