Skip to main content

qualia_core_db/q42/species_corpus/
codec.rs

1//! Native `.q42` encoding of a [`SpeciesCorpus`] and its CBOR-LD interchange projection.
2//!
3//! On disk the corpus is a unified Q42 volume: NQuins plus an embedded Q42LEX lexicon, the same
4//! representation as [`crate::q42::model_helper`]. The root corpus node carries header fields;
5//! every source, assertion and gap is its own node linked from the root (metadata = position).
6//! Record fields become edges whose predicate IRI is the field path under the corpus context;
7//! strings and exact numeric lexical forms resolve through the lexicon so decoding is lossless.
8//! Non-negative numbers that fit also get an inline `xsd:decimal` mirror edge (`#decimal`) for
9//! native numeric queries.
10
11use std::collections::{BTreeMap, HashMap};
12
13use serde_json::{Map, Number, Value};
14
15#[cfg(not(target_arch = "wasm32"))]
16use super::model::CorpusConflict;
17use super::model::{SpeciesCorpus, SPECIES_CORPUS_CONTEXT};
18
19/// A corpus written both as a native `.q42` volume and as its CBOR-LD projection.
20#[cfg(not(target_arch = "wasm32"))]
21pub struct SealedSpeciesCorpus {
22    pub q42: Vec<u8>,
23    pub cbor_ld: Vec<u8>,
24    pub q42_sha256: String,
25    pub conflicts: Vec<CorpusConflict>,
26}
27use crate::query::resolver::{INLINE_TAG_BOOLEAN, INLINE_TAG_DECIMAL, INLINE_TAG_MASK, INLINE_VALUE_MASK};
28use crate::{q_hash, NQuin};
29
30pub const MAX_CORPUS_Q42_BYTES: usize = 256 * 1024 * 1024;
31const RECORD_KINDS: [&str; 3] = ["sources", "assertions", "gaps"];
32const RDF_TYPE: &str = "rdf:type";
33const TYPE_IRI: &str = "q42:QualiaSpeciesCorpus";
34
35const LEAF_STRING: u64 = 0;
36const LEAF_NUMBER: u64 = 1;
37const LEAF_BOOL: u64 = 2;
38const LEAF_NULL: u64 = 3;
39const LEAF_DECIMAL_MIRROR: u64 = 4;
40const LEAF_LINK: u64 = 5;
41const LEAF_TYPE: u64 = 6;
42const LEAF_EMPTY_OBJECT: u64 = 7;
43const LEAF_EMPTY_ARRAY: u64 = 8;
44/// Leaf type occupies metadata bits [0..3]; position index occupies bits [8..32].
45const LEAF_MASK: u64 = 0xF;
46const INDEX_SHIFT: u32 = 8;
47const MAX_INDEX: usize = 0x00FF_FFFF;
48
49fn node_id(iri: &str) -> u64 {
50    q_hash(iri) & INLINE_VALUE_MASK
51}
52
53fn context_id() -> u64 {
54    q_hash(SPECIES_CORPUS_CONTEXT) & 0x00FF_FFFF_FFFF_FFFF
55}
56
57fn predicate_iri(kind: &str, path: &str) -> String {
58    format!("{SPECIES_CORPUS_CONTEXT}#{kind}/{path}")
59}
60
61struct Graph {
62    lex: HashMap<u64, String>,
63    quins: Vec<NQuin>,
64    ctx: u64,
65}
66
67impl Graph {
68    fn intern(&mut self, hash: u64, term: &str) -> Result<u64, String> {
69        match self.lex.get(&hash) {
70            Some(existing) if existing != term => Err(format!(
71                "lexicon hash collision between '{existing}' and '{term}'"
72            )),
73            Some(_) => Ok(hash),
74            None => {
75                self.lex.insert(hash, term.to_string());
76                Ok(hash)
77            }
78        }
79    }
80
81    fn edge(&mut self, s: u64, p_iri: &str, o: u64, meta: u64) -> Result<(), String> {
82        let p = self.intern(q_hash(p_iri), p_iri)?;
83        self.quins.push(NQuin {
84            subject: s,
85            predicate: p,
86            object: o,
87            context: self.ctx,
88            metadata: meta,
89            parity: NQuin::calculate_parity(s, p, o, self.ctx, meta),
90        });
91        Ok(())
92    }
93
94    fn leaf(&mut self, s: u64, kind: &str, path: &str, v: &Value) -> Result<(), String> {
95        let p = predicate_iri(kind, path);
96        match v {
97            Value::String(t) => {
98                let o = self.intern(node_id(t), t)?;
99                self.edge(s, &p, o, LEAF_STRING)
100            }
101            Value::Number(n) => {
102                let text = n.to_string();
103                let o = self.intern(node_id(&text), &text)?;
104                self.edge(s, &p, o, LEAF_NUMBER)?;
105                if let Some(f) = n.as_f64() {
106                    let scaled = f * 1_000_000.0;
107                    if f.is_finite() && f >= 0.0 && scaled <= INLINE_VALUE_MASK as f64 {
108                        let o = INLINE_TAG_DECIMAL | (scaled.round() as u64 & INLINE_VALUE_MASK);
109                        self.edge(s, &format!("{p}#decimal"), o, LEAF_DECIMAL_MIRROR)?;
110                    }
111                }
112                Ok(())
113            }
114            Value::Bool(b) => self.edge(s, &p, INLINE_TAG_BOOLEAN | u64::from(*b), LEAF_BOOL),
115            Value::Null => self.edge(s, &p, 0, LEAF_NULL),
116            _ => Err(format!("non-leaf value at '{path}'")),
117        }
118    }
119
120    fn marker(&mut self, s: u64, kind: &str, path: &str, leaf_type: u64) -> Result<(), String> {
121        if path.is_empty() {
122            return Err("empty container has no field path".into());
123        }
124        self.edge(s, &predicate_iri(kind, path), 0, leaf_type)
125    }
126
127    fn walk(&mut self, s: u64, kind: &str, path: &str, v: &Value) -> Result<(), String> {
128        match v {
129            Value::Object(m) if m.is_empty() => self.marker(s, kind, path, LEAF_EMPTY_OBJECT),
130            Value::Array(items) if items.is_empty() => self.marker(s, kind, path, LEAF_EMPTY_ARRAY),
131            Value::Object(m) => {
132                for (k, child) in m {
133                    if k.contains(['/', '[', ']', '#']) {
134                        return Err(format!("field name '{k}' contains a reserved character"));
135                    }
136                    let p = if path.is_empty() { k.clone() } else { format!("{path}/{k}") };
137                    self.walk(s, kind, &p, child)?;
138                }
139                Ok(())
140            }
141            Value::Array(items) => {
142                for (i, child) in items.iter().enumerate() {
143                    self.walk(s, kind, &format!("{path}[{i}]"), child)?;
144                }
145                Ok(())
146            }
147            leaf => self.leaf(s, kind, path, leaf),
148        }
149    }
150}
151
152/// Lowercase SHA-256 hex of volume bytes. This is the digest a species spec cites.
153pub fn volume_sha256(bytes: &[u8]) -> String {
154    use sha2::{Digest, Sha256};
155    let digest = Sha256::digest(bytes);
156    let mut out = String::with_capacity(64);
157    for b in digest {
158        use std::fmt::Write;
159        let _ = write!(&mut out, "{b:02x}");
160    }
161    out
162}
163
164impl SpeciesCorpus {
165    /// Lexicon and quins for the native `.q42` representation, quins sorted by object.
166    pub fn to_q42_graph(&self) -> Result<(HashMap<u64, String>, Vec<NQuin>), String> {
167        self.validate()?;
168        let doc = serde_json::to_value(self).map_err(|e| format!("corpus serialization: {e}"))?;
169        let Value::Object(mut root) = doc else {
170            return Err("corpus did not serialize to a record".into());
171        };
172        let mut g = Graph {
173            lex: HashMap::new(),
174            quins: Vec::new(),
175            ctx: context_id(),
176        };
177        g.intern(context_id(), SPECIES_CORPUS_CONTEXT)?;
178        let root_id = g.intern(node_id(&self.corpus_iri), &self.corpus_iri)?;
179        let type_obj = g.intern(node_id(TYPE_IRI), TYPE_IRI)?;
180        g.edge(root_id, RDF_TYPE, type_obj, LEAF_TYPE)?;
181
182        for kind in RECORD_KINDS {
183            let Some(Value::Array(items)) = root.remove(kind) else {
184                continue;
185            };
186            if items.len() > MAX_INDEX {
187                return Err(format!("too many {kind} records"));
188            }
189            for (i, item) in items.iter().enumerate() {
190                let iri = format!("{}#{kind}/{i}", self.corpus_iri);
191                let node = g.intern(node_id(&iri), &iri)?;
192                g.edge(
193                    root_id,
194                    &predicate_iri("corpus", kind),
195                    node,
196                    LEAF_LINK | ((i as u64) << INDEX_SHIFT),
197                )?;
198                g.walk(node, kind, "", item)?;
199            }
200        }
201        g.walk(root_id, "corpus", "", &Value::Object(root))?;
202        g.quins.sort_unstable_by_key(|q| (q.object, q.subject, q.predicate));
203        Ok((g.lex, g.quins))
204    }
205
206    /// Reconstructs and validates a [`SpeciesCorpus`] from an in-memory Q42 lexicon map and NQuin slice.
207    pub fn from_q42_graph(lex: &HashMap<u64, String>, quins: &[NQuin]) -> Result<Self, String> {
208        decode_graph(quins, |h| {
209            lex.get(&h)
210                .cloned()
211                .ok_or_else(|| format!("species corpus has unresolved lexicon hash {h:#018x}"))
212        })
213    }
214
215    /// Encodes the corpus as a unified `.q42` volume.
216    #[cfg(not(target_arch = "wasm32"))]
217    pub fn to_q42_bytes(&self) -> Result<Vec<u8>, String> {
218        use crate::q42_volume::UnifiedVolumeBuilder;
219        use crate::QUINS_PER_BLOCK;
220        let (lex, quins) = self.to_q42_graph()?;
221        let mut builder = UnifiedVolumeBuilder::with_lex_map(&lex)
222            .map_err(|e| format!("species corpus Q42LEX: {e:?}"))?;
223        for (seq, chunk) in quins.chunks(QUINS_PER_BLOCK).enumerate() {
224            builder
225                .push_block(seq as u64, chunk)
226                .map_err(|e| format!("species corpus block {seq}: {e}"))?;
227        }
228        let bytes = builder.finish_to_bytes();
229        if bytes.len() > MAX_CORPUS_Q42_BYTES {
230            return Err(format!("species corpus exceeds {MAX_CORPUS_Q42_BYTES} bytes"));
231        }
232        Ok(bytes)
233    }
234
235    /// Decodes and validates a corpus from unified `.q42` volume bytes.
236    #[cfg(not(target_arch = "wasm32"))]
237    pub fn from_q42_bytes(bytes: &[u8]) -> Result<Self, String> {
238        if bytes.len() > MAX_CORPUS_Q42_BYTES {
239            return Err(format!("species corpus exceeds {MAX_CORPUS_Q42_BYTES} bytes"));
240        }
241        let dir = tempfile::TempDir::new().map_err(|e| format!("species corpus scratch: {e}"))?;
242        let path = dir.path().join("corpus.q42");
243        std::fs::write(&path, bytes).map_err(|e| format!("species corpus scratch write: {e}"))?;
244        Self::from_q42_path(&path)
245    }
246
247    /// Opens, verifies and decodes a corpus `.q42` volume.
248    #[cfg(not(target_arch = "wasm32"))]
249    pub fn from_q42_path(path: &std::path::Path) -> Result<Self, String> {
250        use crate::q42_volume::Q42Volume;
251        let volume = Q42Volume::open(path).map_err(|e| format!("open species corpus: {e}"))?;
252        volume
253            .header()
254            .verify_version()
255            .map_err(|e| format!("invalid species corpus volume: {e}"))?;
256        volume
257            .verify_all_blocks()
258            .map_err(|e| format!("species corpus block verification: {e}"))?;
259        let lex = volume
260            .lex_view()
261            .map_err(|e| format!("invalid species corpus Q42LEX: {e:?}"))?;
262        let quins = volume
263            .read_all_quins()
264            .map_err(|e| format!("read species corpus: {e}"))?;
265        let resolve = |h: u64| -> Result<String, String> {
266            let mut t = String::new();
267            if lex.resolve_term_into(h, &mut t) {
268                Ok(t)
269            } else {
270                Err(format!("species corpus has unresolved lexicon hash {h:#018x}"))
271            }
272        };
273        decode_graph(&quins, resolve)
274    }
275
276    /// Native `.q42` bytes, the CBOR-LD projection, the volume digest, and derived conflicts.
277    /// Conflicts stay in the volume as the disagreeing assertions; this also lists them.
278    #[cfg(not(target_arch = "wasm32"))]
279    pub fn seal(&self) -> Result<SealedSpeciesCorpus, String> {
280        self.validate()?;
281        let q42 = self.to_q42_bytes()?;
282        let cbor_ld = self.to_cbor_ld()?;
283        Ok(SealedSpeciesCorpus {
284            q42_sha256: volume_sha256(&q42),
285            q42,
286            cbor_ld,
287            conflicts: self.conflicts(),
288        })
289    }
290
291    /// CBOR-LD interchange projection (self-describe tag 55799).
292    pub fn to_cbor_ld(&self) -> Result<Vec<u8>, String> {
293        self.validate()?;
294        let mut out = vec![0xd9, 0xd9, 0xf7];
295        ciborium::into_writer(self, &mut out).map_err(|e| format!("cbor encode: {e}"))?;
296        if out.len() > MAX_CORPUS_Q42_BYTES {
297            return Err(format!("species corpus exceeds {MAX_CORPUS_Q42_BYTES} bytes"));
298        }
299        Ok(out)
300    }
301
302    /// Decodes and validates the CBOR-LD interchange projection.
303    pub fn from_cbor_ld(bytes: &[u8]) -> Result<Self, String> {
304        if bytes.len() > MAX_CORPUS_Q42_BYTES {
305            return Err(format!("species corpus exceeds {MAX_CORPUS_Q42_BYTES} bytes"));
306        }
307        let body = bytes.strip_prefix(&[0xd9, 0xd9, 0xf7]).unwrap_or(bytes);
308        let corpus: Self = ciborium::from_reader(body).map_err(|e| format!("cbor decode: {e}"))?;
309        corpus.validate()?;
310        Ok(corpus)
311    }
312}
313
314enum Seg {
315    Key(String),
316    Index(usize),
317}
318
319fn parse_path(path: &str) -> Result<Vec<Seg>, String> {
320    let mut out = Vec::new();
321    for part in path.split('/') {
322        let (key, rest) = part.split_at(part.find('[').unwrap_or(part.len()));
323        if key.is_empty() {
324            return Err(format!("malformed field path '{path}'"));
325        }
326        out.push(Seg::Key(key.to_string()));
327        let mut rest = rest;
328        while let Some(r) = rest.strip_prefix('[') {
329            let end = r.find(']').ok_or_else(|| format!("malformed field path '{path}'"))?;
330            let i: usize = r[..end]
331                .parse()
332                .map_err(|_| format!("malformed index in '{path}'"))?;
333            if i > MAX_INDEX {
334                return Err(format!("index too large in '{path}'"));
335            }
336            out.push(Seg::Index(i));
337            rest = &r[end + 1..];
338        }
339        if !rest.is_empty() {
340            return Err(format!("malformed field path '{path}'"));
341        }
342    }
343    Ok(out)
344}
345
346/// Sparse arrays are filled with `Null` during insertion and checked afterwards.
347fn insert(root: &mut Value, segs: &[Seg], leaf: Value) -> Result<(), String> {
348    let mut cur = root;
349    for (n, seg) in segs.iter().enumerate() {
350        let last = n + 1 == segs.len();
351        let next_is_index = matches!(segs.get(n + 1), Some(Seg::Index(_)));
352        let fresh = || if next_is_index { Value::Array(Vec::new()) } else { Value::Object(Map::new()) };
353        cur = match seg {
354            Seg::Key(k) => {
355                let m = cur.as_object_mut().ok_or("field path conflicts with a non-record value")?;
356                if last {
357                    if m.insert(k.clone(), leaf).is_some() {
358                        return Err(format!("duplicate field '{k}'"));
359                    }
360                    return Ok(());
361                }
362                m.entry(k.clone()).or_insert_with(fresh)
363            }
364            Seg::Index(i) => {
365                let a = cur.as_array_mut().ok_or("field path conflicts with a non-list value")?;
366                if a.len() <= *i {
367                    a.resize(*i + 1, Value::Null);
368                }
369                if last {
370                    a[*i] = leaf;
371                    return Ok(());
372                }
373                if a[*i].is_null() {
374                    a[*i] = fresh();
375                }
376                &mut a[*i]
377            }
378        };
379    }
380    Err("empty field path".into())
381}
382
383fn decode_graph(
384    quins: &[NQuin],
385    resolve: impl Fn(u64) -> Result<String, String>,
386) -> Result<SpeciesCorpus, String> {
387    let ctx = context_id();
388    let mut root_id = None;
389    for q in quins {
390        if q.parity != NQuin::calculate_parity(q.subject, q.predicate, q.object, q.context, q.metadata) {
391            return Err("species corpus quin parity mismatch".into());
392        }
393        if q.context != ctx {
394            return Err("species corpus quin has a foreign context".into());
395        }
396        if q.metadata & LEAF_MASK == LEAF_TYPE {
397            if resolve(q.object)? != TYPE_IRI || root_id.replace(q.subject).is_some() {
398                return Err("species corpus must have exactly one corpus root".into());
399            }
400        }
401    }
402    let root_id = root_id.ok_or("Q42 volume is not a species corpus")?;
403
404    let mut by_subject: BTreeMap<u64, Vec<&NQuin>> = BTreeMap::new();
405    for q in quins {
406        by_subject.entry(q.subject).or_default().push(q);
407    }
408
409    let mut doc = Value::Object(Map::new());
410    let mut records: BTreeMap<&str, BTreeMap<u64, u64>> = BTreeMap::new();
411    let corpus_prefix = format!("{SPECIES_CORPUS_CONTEXT}#corpus/");
412    for q in by_subject.get(&root_id).map(Vec::as_slice).unwrap_or(&[]) {
413        if q.metadata & LEAF_MASK == LEAF_LINK {
414            let p = resolve(q.predicate)?;
415            let kind = p
416                .strip_prefix(&corpus_prefix)
417                .and_then(|k| RECORD_KINDS.iter().find(|r| **r == k))
418                .ok_or_else(|| format!("unknown corpus record link '{p}'"))?;
419            let index = (q.metadata >> INDEX_SHIFT) & MAX_INDEX as u64;
420            if records.entry(kind).or_default().insert(index, q.object).is_some() {
421                return Err(format!("duplicate {kind} position {index}"));
422            }
423        }
424    }
425    decode_fields(&mut doc, "corpus", by_subject.get(&root_id), &resolve)?;
426
427    for kind in RECORD_KINDS {
428        let mut items = Vec::new();
429        for (expected, (index, node)) in records.get(kind).into_iter().flatten().enumerate() {
430            if *index != expected as u64 {
431                return Err(format!("{kind} positions are not contiguous"));
432            }
433            let mut item = Value::Object(Map::new());
434            decode_fields(&mut item, kind, by_subject.get(node), &resolve)?;
435            items.push(item);
436        }
437        doc.as_object_mut().unwrap().insert(kind.to_string(), Value::Array(items));
438    }
439    if contains_null_hole(&doc) {
440        return Err("species corpus has a missing list element".into());
441    }
442    let corpus: SpeciesCorpus =
443        serde_json::from_value(doc).map_err(|e| format!("species corpus decode: {e}"))?;
444    corpus.validate()?;
445    Ok(corpus)
446}
447
448fn decode_fields(
449    target: &mut Value,
450    kind: &str,
451    quins: Option<&Vec<&NQuin>>,
452    resolve: &impl Fn(u64) -> Result<String, String>,
453) -> Result<(), String> {
454    let prefix = format!("{SPECIES_CORPUS_CONTEXT}#{kind}/");
455    for q in quins.map(Vec::as_slice).unwrap_or(&[]) {
456        let leaf_type = q.metadata & LEAF_MASK;
457        if matches!(leaf_type, LEAF_DECIMAL_MIRROR | LEAF_LINK | LEAF_TYPE) {
458            continue;
459        }
460        let p = resolve(q.predicate)?;
461        let path = p
462            .strip_prefix(&prefix)
463            .ok_or_else(|| format!("field predicate '{p}' is outside {kind}"))?;
464        let leaf = match leaf_type {
465            LEAF_EMPTY_OBJECT => Value::Object(Map::new()),
466            LEAF_EMPTY_ARRAY => Value::Array(Vec::new()),
467            LEAF_STRING => Value::String(resolve(q.object)?),
468            LEAF_NUMBER => {
469                let text = resolve(q.object)?;
470                Value::Number(
471                    text.parse::<Number>()
472                        .map_err(|_| format!("invalid number '{text}' at '{path}'"))?,
473                )
474            }
475            LEAF_BOOL => {
476                if q.object & INLINE_TAG_MASK != INLINE_TAG_BOOLEAN {
477                    return Err(format!("expected boolean at '{path}'"));
478                }
479                Value::Bool(q.object & 1 != 0)
480            }
481            LEAF_NULL => Value::Null,
482            other => return Err(format!("unknown leaf type {other} at '{path}'")),
483        };
484        insert(target, &parse_path(path)?, leaf)?;
485    }
486    Ok(())
487}
488
489fn contains_null_hole(v: &Value) -> bool {
490    match v {
491        Value::Array(a) => a.iter().any(|x| x.is_null() || contains_null_hole(x)),
492        Value::Object(m) => m.values().any(contains_null_hole),
493        _ => false,
494    }
495}