1use serde::{Deserialize, Serialize};
12use std::path::Path;
13
14pub const MODEL_HELPER_EXT: &str = "q42";
16pub const LEGACY_MODEL_HELPER_EXT: &str = "q42.cbor-ld";
18
19pub const MODEL_HELPER_CONTEXT: &str = "https://webizen.org/ns/qualia/model-helper/v1";
21
22#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
24pub struct ModelHelper {
25 #[serde(rename = "@context")]
26 pub context: String,
27 #[serde(rename = "@type")]
28 pub type_: String,
29 pub format: String,
30 pub source_gguf: String,
31 pub p64: String,
32 pub page_log2: u16,
33 pub layout: String,
34 pub converted_unix_ms: u64,
35 pub tokenizer: ModelHelperTokenizer,
36 pub notes: Vec<String>,
37}
38
39#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
40pub struct ModelHelperTokenizer {
41 pub bos_token_id: u32,
42 pub eos_token_id: u32,
43 pub add_bos_token: bool,
44 pub chat_family: String,
46 pub stop_token_ids: Vec<u32>,
47 pub stop_token_strings: Vec<String>,
48 pub vocab_len: u32,
49}
50
51impl ModelHelper {
52 pub fn new(
54 source_gguf: impl Into<String>,
55 p64_path: impl Into<String>,
56 page_log2: u16,
57 layout: impl Into<String>,
58 tokenizer: ModelHelperTokenizer,
59 ) -> Self {
60 let converted_unix_ms = std::time::SystemTime::now()
61 .duration_since(std::time::UNIX_EPOCH)
62 .map(|d| d.as_millis() as u64)
63 .unwrap_or(0);
64 Self {
65 context: MODEL_HELPER_CONTEXT.to_string(),
66 type_: "QualiaModelHelper".to_string(),
67 format: "qualia.q42.model-helper.v1".to_string(),
68 source_gguf: source_gguf.into(),
69 p64: p64_path.into(),
70 page_log2,
71 layout: layout.into(),
72 converted_unix_ms,
73 tokenizer,
74 notes: vec![
75 "verbatim = GGML quant blocks preserved (same decode kernels as GGUF).".into(),
76 "Activate the .p64 path; keep GGUF as an import-only archive.".into(),
77 "Metadata is a canonical unified Q42 v3 volume.".into(),
78 ],
79 }
80 }
81
82 pub fn to_cbor_ld(&self) -> Result<Vec<u8>, String> {
84 let mut body = Vec::new();
85 ciborium::into_writer(self, &mut body).map_err(|e| format!("cbor encode: {e}"))?;
86 let mut out = Vec::with_capacity(body.len() + 3);
87 out.extend_from_slice(&[0xd9, 0xd9, 0xf7]);
88 out.extend_from_slice(&body);
89 Ok(out)
90 }
91
92 pub fn from_cbor_ld(bytes: &[u8]) -> Result<Self, String> {
94 let payload = strip_self_describe_tag(bytes);
95 ciborium::from_reader(payload).map_err(|e| format!("cbor decode: {e}"))
96 }
97
98 #[cfg(not(target_arch = "wasm32"))]
100 pub fn write_beside_p64(&self, p64_path: &Path) -> Result<std::path::PathBuf, String> {
101 use crate::q42_volume::UnifiedVolumeBuilder;
102
103 let helper_path = helper_path_for_p64(p64_path);
104 let (lex, mut quins) = self.to_q42_graph();
105 quins.sort_unstable_by_key(|q| q.object);
107 let mut builder = UnifiedVolumeBuilder::with_lex_map(&lex)
108 .map_err(|e| format!("build Q42LEX for helper: {e:?}"))?;
109 builder
110 .push_block(0, &quins)
111 .map_err(|e| format!("build canonical Q42 helper: {e}"))?;
112 builder
113 .finish(&helper_path)
114 .map_err(|e| format!("write {}: {e}", helper_path.display()))?;
115 Ok(helper_path)
116 }
117
118 #[cfg(target_arch = "wasm32")]
119 pub fn write_beside_p64(&self, _p64_path: &Path) -> Result<std::path::PathBuf, String> {
121 Err("canonical Q42 model metadata is written by the native conversion tool".into())
122 }
123
124 #[cfg(not(target_arch = "wasm32"))]
126 pub fn load_beside_p64(p64_path: &Path) -> Result<Option<Self>, String> {
127 let helper_path = helper_path_for_p64(p64_path);
128 if helper_path.is_file() {
129 return Self::from_q42_path(&helper_path).map(Some);
130 }
131
132 let legacy_path = legacy_helper_path_for_p64(p64_path);
133 if !legacy_path.is_file() {
134 return Ok(None);
135 }
136 let bytes = std::fs::read(&legacy_path)
137 .map_err(|e| format!("read {}: {e}", legacy_path.display()))?;
138 Ok(Some(Self::from_cbor_ld(&bytes)?))
139 }
140
141 #[cfg(target_arch = "wasm32")]
142 pub fn load_beside_p64(_p64_path: &Path) -> Result<Option<Self>, String> {
144 Ok(None)
145 }
146
147 #[cfg(not(target_arch = "wasm32"))]
148 fn to_q42_graph(&self) -> (std::collections::HashMap<u64, String>, Vec<crate::NQuin>) {
149 use crate::frame_layout::{INLINE_TAG_BOOLEAN, INLINE_TAG_INTEGER, INLINE_VALUE_MASK};
150 use crate::q_hash;
151 use std::collections::HashMap;
152
153 let mut lex = HashMap::new();
154 let subject = q_hash(&self.p64);
155 let context = q_hash(MODEL_HELPER_CONTEXT);
156 insert_lex(&mut lex, context, MODEL_HELPER_CONTEXT);
157
158 let mut quins = Vec::new();
159 let mut string_edge = |predicate_iri: &str, value: &str, metadata: u64| {
160 let predicate = q_hash(predicate_iri);
161 let object = q_hash(value) & INLINE_VALUE_MASK;
162 insert_lex(&mut lex, predicate, predicate_iri);
163 insert_lex(&mut lex, object, value);
164 quins.push(make_quin(subject, predicate, object, context, metadata));
165 };
166
167 string_edge("rdf:type", "q42:QualiaModelHelper", 0);
168 string_edge("q42:format", &self.format, 0);
169 string_edge("q42:sourceGguf", &self.source_gguf, 0);
170 string_edge("q42:p64Asset", &self.p64, 0);
171 string_edge("q42:layout", &self.layout, 0);
172 string_edge("q42:chatFamily", &self.tokenizer.chat_family, 0);
173 for (i, value) in self.tokenizer.stop_token_strings.iter().enumerate() {
174 string_edge("q42:stopTokenString", value, i as u64);
175 }
176 for (i, value) in self.notes.iter().enumerate() {
177 string_edge("q42:note", value, i as u64);
178 }
179 drop(string_edge);
180
181 let mut integer_edge = |predicate_iri: &str, value: u64, metadata: u64| {
182 let predicate = q_hash(predicate_iri);
183 insert_lex(&mut lex, predicate, predicate_iri);
184 quins.push(make_quin(
185 subject,
186 predicate,
187 INLINE_TAG_INTEGER | (value & INLINE_VALUE_MASK),
188 context,
189 metadata,
190 ));
191 };
192 integer_edge("q42:pageLog2", self.page_log2 as u64, 0);
193 integer_edge("q42:convertedUnixMs", self.converted_unix_ms, 0);
194 integer_edge("q42:bosTokenId", self.tokenizer.bos_token_id as u64, 0);
195 integer_edge("q42:eosTokenId", self.tokenizer.eos_token_id as u64, 0);
196 integer_edge("q42:vocabLen", self.tokenizer.vocab_len as u64, 0);
197 for (i, value) in self.tokenizer.stop_token_ids.iter().enumerate() {
198 integer_edge("q42:stopTokenId", *value as u64, i as u64);
199 }
200 drop(integer_edge);
201
202 let predicate = q_hash("q42:addBosToken");
203 insert_lex(&mut lex, predicate, "q42:addBosToken");
204 quins.push(make_quin(
205 subject,
206 predicate,
207 INLINE_TAG_BOOLEAN | u64::from(self.tokenizer.add_bos_token),
208 context,
209 0,
210 ));
211
212 (lex, quins)
213 }
214
215 #[cfg(not(target_arch = "wasm32"))]
216 fn from_q42_path(path: &Path) -> Result<Self, String> {
217 use crate::frame_layout::{
218 INLINE_TAG_BOOLEAN, INLINE_TAG_INTEGER, INLINE_TAG_MASK, INLINE_VALUE_MASK,
219 };
220 use crate::q42_volume::Q42Volume;
221 use crate::q_hash;
222 use std::collections::BTreeMap;
223
224 let volume = Q42Volume::open(path)
225 .map_err(|e| format!("open canonical Q42 helper {}: {e}", path.display()))?;
226 volume
227 .header()
228 .verify_version()
229 .map_err(|e| format!("invalid Q42 helper {}: {e}", path.display()))?;
230 let lex = volume
231 .lex_view()
232 .map_err(|e| format!("invalid Q42LEX in {}: {e:?}", path.display()))?;
233 let quins = volume
234 .read_all_quins()
235 .map_err(|e| format!("read Q42 helper {}: {e}", path.display()))?;
236
237 let text = |object: u64| -> Result<String, String> {
238 if let Some(identity) = lex.lookup_webizen_identity(object) {
239 return Ok(identity.to_owned());
240 }
241 let mut term = String::new();
242 if lex.resolve_term_into(object, &mut term) {
243 Ok(term)
244 } else {
245 Err(format!(
246 "Q42 helper has unresolved lexicon hash {object:#018x}"
247 ))
248 }
249 };
250 let integer = |object: u64| -> Result<u64, String> {
251 if object & INLINE_TAG_MASK != INLINE_TAG_INTEGER {
252 return Err(format!(
253 "Q42 helper expected integer object, got {object:#018x}"
254 ));
255 }
256 Ok(object & INLINE_VALUE_MASK)
257 };
258
259 let mut format = None;
260 let mut source_gguf = None;
261 let mut p64 = None;
262 let mut page_log2 = None;
263 let mut layout = None;
264 let mut converted_unix_ms = None;
265 let mut bos_token_id = None;
266 let mut eos_token_id = None;
267 let mut add_bos_token = None;
268 let mut chat_family = None;
269 let mut vocab_len = None;
270 let mut stop_ids = BTreeMap::new();
271 let mut stop_strings = BTreeMap::new();
272 let mut notes = BTreeMap::new();
273 let mut saw_type = false;
274
275 for q in quins {
276 match q.predicate {
277 p if p == q_hash("rdf:type") => {
278 saw_type = text(q.object)? == "q42:QualiaModelHelper";
279 }
280 p if p == q_hash("q42:format") => format = Some(text(q.object)?),
281 p if p == q_hash("q42:sourceGguf") => source_gguf = Some(text(q.object)?),
282 p if p == q_hash("q42:p64Asset") => p64 = Some(text(q.object)?),
283 p if p == q_hash("q42:layout") => layout = Some(text(q.object)?),
284 p if p == q_hash("q42:chatFamily") => chat_family = Some(text(q.object)?),
285 p if p == q_hash("q42:stopTokenString") => {
286 stop_strings.insert(q.metadata, text(q.object)?);
287 }
288 p if p == q_hash("q42:note") => {
289 notes.insert(q.metadata, text(q.object)?);
290 }
291 p if p == q_hash("q42:pageLog2") => page_log2 = Some(integer(q.object)? as u16),
292 p if p == q_hash("q42:convertedUnixMs") => {
293 converted_unix_ms = Some(integer(q.object)?)
294 }
295 p if p == q_hash("q42:bosTokenId") => {
296 bos_token_id = Some(integer(q.object)? as u32)
297 }
298 p if p == q_hash("q42:eosTokenId") => {
299 eos_token_id = Some(integer(q.object)? as u32)
300 }
301 p if p == q_hash("q42:vocabLen") => vocab_len = Some(integer(q.object)? as u32),
302 p if p == q_hash("q42:stopTokenId") => {
303 stop_ids.insert(q.metadata, integer(q.object)? as u32);
304 }
305 p if p == q_hash("q42:addBosToken") => {
306 if q.object & INLINE_TAG_MASK != INLINE_TAG_BOOLEAN {
307 return Err("Q42 helper addBosToken is not a boolean".into());
308 }
309 add_bos_token = Some(q.object & 1 != 0);
310 }
311 _ => {}
312 }
313 }
314
315 if !saw_type {
316 return Err("Q42 volume is not a QualiaModelHelper".into());
317 }
318 let required = |name: &str| format!("Q42 model helper missing {name}");
319 Ok(Self {
320 context: MODEL_HELPER_CONTEXT.to_string(),
321 type_: "QualiaModelHelper".to_string(),
322 format: format.ok_or_else(|| required("format"))?,
323 source_gguf: source_gguf.ok_or_else(|| required("sourceGguf"))?,
324 p64: p64.ok_or_else(|| required("p64Asset"))?,
325 page_log2: page_log2.ok_or_else(|| required("pageLog2"))?,
326 layout: layout.ok_or_else(|| required("layout"))?,
327 converted_unix_ms: converted_unix_ms.ok_or_else(|| required("convertedUnixMs"))?,
328 tokenizer: ModelHelperTokenizer {
329 bos_token_id: bos_token_id.ok_or_else(|| required("bosTokenId"))?,
330 eos_token_id: eos_token_id.ok_or_else(|| required("eosTokenId"))?,
331 add_bos_token: add_bos_token.ok_or_else(|| required("addBosToken"))?,
332 chat_family: chat_family.ok_or_else(|| required("chatFamily"))?,
333 stop_token_ids: stop_ids.into_values().collect(),
334 stop_token_strings: stop_strings.into_values().collect(),
335 vocab_len: vocab_len.ok_or_else(|| required("vocabLen"))?,
336 },
337 notes: notes.into_values().collect(),
338 })
339 }
340
341 pub fn apply_stops_to_tokenizer(&self, tok: &mut crate::gguf_sharder::GgufTokenizer) {
343 tok.merge_stop_token_ids(&self.tokenizer.stop_token_ids);
344 }
345}
346
347#[cfg(not(target_arch = "wasm32"))]
348fn insert_lex(lex: &mut std::collections::HashMap<u64, String>, hash: u64, value: &str) {
349 lex.insert(hash, value.to_string());
350}
351
352#[cfg(not(target_arch = "wasm32"))]
353fn make_quin(
354 subject: u64,
355 predicate: u64,
356 object: u64,
357 context: u64,
358 metadata: u64,
359) -> crate::NQuin {
360 crate::NQuin {
361 subject,
362 predicate,
363 object,
364 context,
365 metadata,
366 parity: crate::NQuin::calculate_parity(subject, predicate, object, context, metadata),
368 }
369}
370
371pub fn helper_path_for_p64(p64_path: &Path) -> std::path::PathBuf {
373 helper_path_with_ext(p64_path, MODEL_HELPER_EXT)
374}
375
376pub fn legacy_helper_path_for_p64(p64_path: &Path) -> std::path::PathBuf {
378 helper_path_with_ext(p64_path, LEGACY_MODEL_HELPER_EXT)
379}
380
381fn helper_path_with_ext(p64_path: &Path, ext: &str) -> std::path::PathBuf {
382 let parent = p64_path.parent().unwrap_or_else(|| Path::new("."));
383 let stem = p64_path
384 .file_stem()
385 .and_then(|s| s.to_str())
386 .unwrap_or("model");
387 parent.join(format!("{stem}.{ext}"))
388}
389
390fn strip_self_describe_tag(bytes: &[u8]) -> &[u8] {
391 if bytes.starts_with(&[0xd9, 0xd9, 0xf7]) {
392 &bytes[3..]
393 } else {
394 bytes
395 }
396}
397
398pub fn has_model_helper_magic(bytes: &[u8]) -> bool {
400 bytes.starts_with(&[0xd9, 0xd9, 0xf7])
401}
402
403#[cfg(test)]
404mod tests {
405 use super::*;
406
407 fn sample() -> ModelHelper {
408 ModelHelper::new(
409 "x.gguf",
410 "x.p64",
411 14,
412 "Verbatim",
413 ModelHelperTokenizer {
414 bos_token_id: 1,
415 eos_token_id: 2,
416 add_bos_token: true,
417 chat_family: "Llama3".into(),
418 stop_token_ids: vec![2, 128_009],
419 stop_token_strings: vec!["</s>".into(), "<|eot_id|>".into()],
420 vocab_len: 128_256,
421 },
422 )
423 }
424
425 #[test]
426 fn cbor_ld_interchange_round_trip() {
427 let h = sample();
428 let bytes = h.to_cbor_ld().expect("encode");
429 assert!(has_model_helper_magic(&bytes));
430 assert_eq!(ModelHelper::from_cbor_ld(&bytes).unwrap(), h);
431 }
432
433 #[cfg(not(target_arch = "wasm32"))]
434 #[test]
435 fn canonical_q42_round_trip_preserves_model_metadata() {
436 use crate::q42_volume::{Q42Volume, Q42_MAGIC, Q42_VERSION_V4};
437
438 let dir = tempfile::tempdir().unwrap();
439 let p64 = dir.path().join("x.p64");
440 std::fs::write(&p64, b"p64\0").unwrap();
441 let h = sample();
442 let path = h.write_beside_p64(&p64).unwrap();
443 assert_eq!(path.extension().and_then(|x| x.to_str()), Some("q42"));
444 assert!(std::fs::read(&path).unwrap().starts_with(&Q42_MAGIC));
445 let volume = Q42Volume::open(&path).unwrap();
446 assert_eq!({ volume.header().version }, Q42_VERSION_V4);
447 volume
448 .verify_all_blocks()
449 .expect("canonical helper must pass five-field ECC + BIDX");
450 assert!(volume.header().flags & crate::q42_volume::FLAG_FIELD_POSTINGS != 0);
451 assert!(volume.header().flags & crate::q42_volume::FLAG_FIELD_RANGES != 0);
452 assert_eq!(ModelHelper::load_beside_p64(&p64).unwrap(), Some(h));
453 }
454
455 #[test]
456 fn helper_path_is_plain_q42() {
457 let p = Path::new(r"C:\LLM_Models\P64\smollm2.f16.p64");
458 assert!(helper_path_for_p64(p)
459 .to_string_lossy()
460 .ends_with("smollm2.f16.q42"));
461 }
462}