1use std::collections::{BTreeMap, HashMap};
12
13use serde_json::{Map, Number, Value};
14
15#[cfg(not(target_arch = "wasm32"))]
16use super::model::CorpusConflict;
17use super::model::{SpeciesCorpus, SPECIES_CORPUS_CONTEXT};
18
19#[cfg(not(target_arch = "wasm32"))]
21pub struct SealedSpeciesCorpus {
22 pub q42: Vec<u8>,
23 pub cbor_ld: Vec<u8>,
24 pub q42_sha256: String,
25 pub conflicts: Vec<CorpusConflict>,
26}
27use crate::query::resolver::{INLINE_TAG_BOOLEAN, INLINE_TAG_DECIMAL, INLINE_TAG_MASK, INLINE_VALUE_MASK};
28use crate::{q_hash, NQuin};
29
30pub const MAX_CORPUS_Q42_BYTES: usize = 256 * 1024 * 1024;
31const RECORD_KINDS: [&str; 3] = ["sources", "assertions", "gaps"];
32const RDF_TYPE: &str = "rdf:type";
33const TYPE_IRI: &str = "q42:QualiaSpeciesCorpus";
34
35const LEAF_STRING: u64 = 0;
36const LEAF_NUMBER: u64 = 1;
37const LEAF_BOOL: u64 = 2;
38const LEAF_NULL: u64 = 3;
39const LEAF_DECIMAL_MIRROR: u64 = 4;
40const LEAF_LINK: u64 = 5;
41const LEAF_TYPE: u64 = 6;
42const LEAF_EMPTY_OBJECT: u64 = 7;
43const LEAF_EMPTY_ARRAY: u64 = 8;
44const LEAF_MASK: u64 = 0xF;
46const INDEX_SHIFT: u32 = 8;
47const MAX_INDEX: usize = 0x00FF_FFFF;
48
49fn node_id(iri: &str) -> u64 {
50 q_hash(iri) & INLINE_VALUE_MASK
51}
52
53fn context_id() -> u64 {
54 q_hash(SPECIES_CORPUS_CONTEXT) & 0x00FF_FFFF_FFFF_FFFF
55}
56
57fn predicate_iri(kind: &str, path: &str) -> String {
58 format!("{SPECIES_CORPUS_CONTEXT}#{kind}/{path}")
59}
60
61struct Graph {
62 lex: HashMap<u64, String>,
63 quins: Vec<NQuin>,
64 ctx: u64,
65}
66
67impl Graph {
68 fn intern(&mut self, hash: u64, term: &str) -> Result<u64, String> {
69 match self.lex.get(&hash) {
70 Some(existing) if existing != term => Err(format!(
71 "lexicon hash collision between '{existing}' and '{term}'"
72 )),
73 Some(_) => Ok(hash),
74 None => {
75 self.lex.insert(hash, term.to_string());
76 Ok(hash)
77 }
78 }
79 }
80
81 fn edge(&mut self, s: u64, p_iri: &str, o: u64, meta: u64) -> Result<(), String> {
82 let p = self.intern(q_hash(p_iri), p_iri)?;
83 self.quins.push(NQuin {
84 subject: s,
85 predicate: p,
86 object: o,
87 context: self.ctx,
88 metadata: meta,
89 parity: NQuin::calculate_parity(s, p, o, self.ctx, meta),
90 });
91 Ok(())
92 }
93
94 fn leaf(&mut self, s: u64, kind: &str, path: &str, v: &Value) -> Result<(), String> {
95 let p = predicate_iri(kind, path);
96 match v {
97 Value::String(t) => {
98 let o = self.intern(node_id(t), t)?;
99 self.edge(s, &p, o, LEAF_STRING)
100 }
101 Value::Number(n) => {
102 let text = n.to_string();
103 let o = self.intern(node_id(&text), &text)?;
104 self.edge(s, &p, o, LEAF_NUMBER)?;
105 if let Some(f) = n.as_f64() {
106 let scaled = f * 1_000_000.0;
107 if f.is_finite() && f >= 0.0 && scaled <= INLINE_VALUE_MASK as f64 {
108 let o = INLINE_TAG_DECIMAL | (scaled.round() as u64 & INLINE_VALUE_MASK);
109 self.edge(s, &format!("{p}#decimal"), o, LEAF_DECIMAL_MIRROR)?;
110 }
111 }
112 Ok(())
113 }
114 Value::Bool(b) => self.edge(s, &p, INLINE_TAG_BOOLEAN | u64::from(*b), LEAF_BOOL),
115 Value::Null => self.edge(s, &p, 0, LEAF_NULL),
116 _ => Err(format!("non-leaf value at '{path}'")),
117 }
118 }
119
120 fn marker(&mut self, s: u64, kind: &str, path: &str, leaf_type: u64) -> Result<(), String> {
121 if path.is_empty() {
122 return Err("empty container has no field path".into());
123 }
124 self.edge(s, &predicate_iri(kind, path), 0, leaf_type)
125 }
126
127 fn walk(&mut self, s: u64, kind: &str, path: &str, v: &Value) -> Result<(), String> {
128 match v {
129 Value::Object(m) if m.is_empty() => self.marker(s, kind, path, LEAF_EMPTY_OBJECT),
130 Value::Array(items) if items.is_empty() => self.marker(s, kind, path, LEAF_EMPTY_ARRAY),
131 Value::Object(m) => {
132 for (k, child) in m {
133 if k.contains(['/', '[', ']', '#']) {
134 return Err(format!("field name '{k}' contains a reserved character"));
135 }
136 let p = if path.is_empty() { k.clone() } else { format!("{path}/{k}") };
137 self.walk(s, kind, &p, child)?;
138 }
139 Ok(())
140 }
141 Value::Array(items) => {
142 for (i, child) in items.iter().enumerate() {
143 self.walk(s, kind, &format!("{path}[{i}]"), child)?;
144 }
145 Ok(())
146 }
147 leaf => self.leaf(s, kind, path, leaf),
148 }
149 }
150}
151
152pub fn volume_sha256(bytes: &[u8]) -> String {
154 use sha2::{Digest, Sha256};
155 let digest = Sha256::digest(bytes);
156 let mut out = String::with_capacity(64);
157 for b in digest {
158 use std::fmt::Write;
159 let _ = write!(&mut out, "{b:02x}");
160 }
161 out
162}
163
164impl SpeciesCorpus {
165 pub fn to_q42_graph(&self) -> Result<(HashMap<u64, String>, Vec<NQuin>), String> {
167 self.validate()?;
168 let doc = serde_json::to_value(self).map_err(|e| format!("corpus serialization: {e}"))?;
169 let Value::Object(mut root) = doc else {
170 return Err("corpus did not serialize to a record".into());
171 };
172 let mut g = Graph {
173 lex: HashMap::new(),
174 quins: Vec::new(),
175 ctx: context_id(),
176 };
177 g.intern(context_id(), SPECIES_CORPUS_CONTEXT)?;
178 let root_id = g.intern(node_id(&self.corpus_iri), &self.corpus_iri)?;
179 let type_obj = g.intern(node_id(TYPE_IRI), TYPE_IRI)?;
180 g.edge(root_id, RDF_TYPE, type_obj, LEAF_TYPE)?;
181
182 for kind in RECORD_KINDS {
183 let Some(Value::Array(items)) = root.remove(kind) else {
184 continue;
185 };
186 if items.len() > MAX_INDEX {
187 return Err(format!("too many {kind} records"));
188 }
189 for (i, item) in items.iter().enumerate() {
190 let iri = format!("{}#{kind}/{i}", self.corpus_iri);
191 let node = g.intern(node_id(&iri), &iri)?;
192 g.edge(
193 root_id,
194 &predicate_iri("corpus", kind),
195 node,
196 LEAF_LINK | ((i as u64) << INDEX_SHIFT),
197 )?;
198 g.walk(node, kind, "", item)?;
199 }
200 }
201 g.walk(root_id, "corpus", "", &Value::Object(root))?;
202 g.quins.sort_unstable_by_key(|q| (q.object, q.subject, q.predicate));
203 Ok((g.lex, g.quins))
204 }
205
206 pub fn from_q42_graph(lex: &HashMap<u64, String>, quins: &[NQuin]) -> Result<Self, String> {
208 decode_graph(quins, |h| {
209 lex.get(&h)
210 .cloned()
211 .ok_or_else(|| format!("species corpus has unresolved lexicon hash {h:#018x}"))
212 })
213 }
214
215 #[cfg(not(target_arch = "wasm32"))]
217 pub fn to_q42_bytes(&self) -> Result<Vec<u8>, String> {
218 use crate::q42_volume::UnifiedVolumeBuilder;
219 use crate::QUINS_PER_BLOCK;
220 let (lex, quins) = self.to_q42_graph()?;
221 let mut builder = UnifiedVolumeBuilder::with_lex_map(&lex)
222 .map_err(|e| format!("species corpus Q42LEX: {e:?}"))?;
223 for (seq, chunk) in quins.chunks(QUINS_PER_BLOCK).enumerate() {
224 builder
225 .push_block(seq as u64, chunk)
226 .map_err(|e| format!("species corpus block {seq}: {e}"))?;
227 }
228 let bytes = builder.finish_to_bytes();
229 if bytes.len() > MAX_CORPUS_Q42_BYTES {
230 return Err(format!("species corpus exceeds {MAX_CORPUS_Q42_BYTES} bytes"));
231 }
232 Ok(bytes)
233 }
234
235 #[cfg(not(target_arch = "wasm32"))]
237 pub fn from_q42_bytes(bytes: &[u8]) -> Result<Self, String> {
238 if bytes.len() > MAX_CORPUS_Q42_BYTES {
239 return Err(format!("species corpus exceeds {MAX_CORPUS_Q42_BYTES} bytes"));
240 }
241 let dir = tempfile::TempDir::new().map_err(|e| format!("species corpus scratch: {e}"))?;
242 let path = dir.path().join("corpus.q42");
243 std::fs::write(&path, bytes).map_err(|e| format!("species corpus scratch write: {e}"))?;
244 Self::from_q42_path(&path)
245 }
246
247 #[cfg(not(target_arch = "wasm32"))]
249 pub fn from_q42_path(path: &std::path::Path) -> Result<Self, String> {
250 use crate::q42_volume::Q42Volume;
251 let volume = Q42Volume::open(path).map_err(|e| format!("open species corpus: {e}"))?;
252 volume
253 .header()
254 .verify_version()
255 .map_err(|e| format!("invalid species corpus volume: {e}"))?;
256 volume
257 .verify_all_blocks()
258 .map_err(|e| format!("species corpus block verification: {e}"))?;
259 let lex = volume
260 .lex_view()
261 .map_err(|e| format!("invalid species corpus Q42LEX: {e:?}"))?;
262 let quins = volume
263 .read_all_quins()
264 .map_err(|e| format!("read species corpus: {e}"))?;
265 let resolve = |h: u64| -> Result<String, String> {
266 let mut t = String::new();
267 if lex.resolve_term_into(h, &mut t) {
268 Ok(t)
269 } else {
270 Err(format!("species corpus has unresolved lexicon hash {h:#018x}"))
271 }
272 };
273 decode_graph(&quins, resolve)
274 }
275
276 #[cfg(not(target_arch = "wasm32"))]
279 pub fn seal(&self) -> Result<SealedSpeciesCorpus, String> {
280 self.validate()?;
281 let q42 = self.to_q42_bytes()?;
282 let cbor_ld = self.to_cbor_ld()?;
283 Ok(SealedSpeciesCorpus {
284 q42_sha256: volume_sha256(&q42),
285 q42,
286 cbor_ld,
287 conflicts: self.conflicts(),
288 })
289 }
290
291 pub fn to_cbor_ld(&self) -> Result<Vec<u8>, String> {
293 self.validate()?;
294 let mut out = vec![0xd9, 0xd9, 0xf7];
295 ciborium::into_writer(self, &mut out).map_err(|e| format!("cbor encode: {e}"))?;
296 if out.len() > MAX_CORPUS_Q42_BYTES {
297 return Err(format!("species corpus exceeds {MAX_CORPUS_Q42_BYTES} bytes"));
298 }
299 Ok(out)
300 }
301
302 pub fn from_cbor_ld(bytes: &[u8]) -> Result<Self, String> {
304 if bytes.len() > MAX_CORPUS_Q42_BYTES {
305 return Err(format!("species corpus exceeds {MAX_CORPUS_Q42_BYTES} bytes"));
306 }
307 let body = bytes.strip_prefix(&[0xd9, 0xd9, 0xf7]).unwrap_or(bytes);
308 let corpus: Self = ciborium::from_reader(body).map_err(|e| format!("cbor decode: {e}"))?;
309 corpus.validate()?;
310 Ok(corpus)
311 }
312}
313
314enum Seg {
315 Key(String),
316 Index(usize),
317}
318
319fn parse_path(path: &str) -> Result<Vec<Seg>, String> {
320 let mut out = Vec::new();
321 for part in path.split('/') {
322 let (key, rest) = part.split_at(part.find('[').unwrap_or(part.len()));
323 if key.is_empty() {
324 return Err(format!("malformed field path '{path}'"));
325 }
326 out.push(Seg::Key(key.to_string()));
327 let mut rest = rest;
328 while let Some(r) = rest.strip_prefix('[') {
329 let end = r.find(']').ok_or_else(|| format!("malformed field path '{path}'"))?;
330 let i: usize = r[..end]
331 .parse()
332 .map_err(|_| format!("malformed index in '{path}'"))?;
333 if i > MAX_INDEX {
334 return Err(format!("index too large in '{path}'"));
335 }
336 out.push(Seg::Index(i));
337 rest = &r[end + 1..];
338 }
339 if !rest.is_empty() {
340 return Err(format!("malformed field path '{path}'"));
341 }
342 }
343 Ok(out)
344}
345
346fn insert(root: &mut Value, segs: &[Seg], leaf: Value) -> Result<(), String> {
348 let mut cur = root;
349 for (n, seg) in segs.iter().enumerate() {
350 let last = n + 1 == segs.len();
351 let next_is_index = matches!(segs.get(n + 1), Some(Seg::Index(_)));
352 let fresh = || if next_is_index { Value::Array(Vec::new()) } else { Value::Object(Map::new()) };
353 cur = match seg {
354 Seg::Key(k) => {
355 let m = cur.as_object_mut().ok_or("field path conflicts with a non-record value")?;
356 if last {
357 if m.insert(k.clone(), leaf).is_some() {
358 return Err(format!("duplicate field '{k}'"));
359 }
360 return Ok(());
361 }
362 m.entry(k.clone()).or_insert_with(fresh)
363 }
364 Seg::Index(i) => {
365 let a = cur.as_array_mut().ok_or("field path conflicts with a non-list value")?;
366 if a.len() <= *i {
367 a.resize(*i + 1, Value::Null);
368 }
369 if last {
370 a[*i] = leaf;
371 return Ok(());
372 }
373 if a[*i].is_null() {
374 a[*i] = fresh();
375 }
376 &mut a[*i]
377 }
378 };
379 }
380 Err("empty field path".into())
381}
382
383fn decode_graph(
384 quins: &[NQuin],
385 resolve: impl Fn(u64) -> Result<String, String>,
386) -> Result<SpeciesCorpus, String> {
387 let ctx = context_id();
388 let mut root_id = None;
389 for q in quins {
390 if q.parity != NQuin::calculate_parity(q.subject, q.predicate, q.object, q.context, q.metadata) {
391 return Err("species corpus quin parity mismatch".into());
392 }
393 if q.context != ctx {
394 return Err("species corpus quin has a foreign context".into());
395 }
396 if q.metadata & LEAF_MASK == LEAF_TYPE {
397 if resolve(q.object)? != TYPE_IRI || root_id.replace(q.subject).is_some() {
398 return Err("species corpus must have exactly one corpus root".into());
399 }
400 }
401 }
402 let root_id = root_id.ok_or("Q42 volume is not a species corpus")?;
403
404 let mut by_subject: BTreeMap<u64, Vec<&NQuin>> = BTreeMap::new();
405 for q in quins {
406 by_subject.entry(q.subject).or_default().push(q);
407 }
408
409 let mut doc = Value::Object(Map::new());
410 let mut records: BTreeMap<&str, BTreeMap<u64, u64>> = BTreeMap::new();
411 let corpus_prefix = format!("{SPECIES_CORPUS_CONTEXT}#corpus/");
412 for q in by_subject.get(&root_id).map(Vec::as_slice).unwrap_or(&[]) {
413 if q.metadata & LEAF_MASK == LEAF_LINK {
414 let p = resolve(q.predicate)?;
415 let kind = p
416 .strip_prefix(&corpus_prefix)
417 .and_then(|k| RECORD_KINDS.iter().find(|r| **r == k))
418 .ok_or_else(|| format!("unknown corpus record link '{p}'"))?;
419 let index = (q.metadata >> INDEX_SHIFT) & MAX_INDEX as u64;
420 if records.entry(kind).or_default().insert(index, q.object).is_some() {
421 return Err(format!("duplicate {kind} position {index}"));
422 }
423 }
424 }
425 decode_fields(&mut doc, "corpus", by_subject.get(&root_id), &resolve)?;
426
427 for kind in RECORD_KINDS {
428 let mut items = Vec::new();
429 for (expected, (index, node)) in records.get(kind).into_iter().flatten().enumerate() {
430 if *index != expected as u64 {
431 return Err(format!("{kind} positions are not contiguous"));
432 }
433 let mut item = Value::Object(Map::new());
434 decode_fields(&mut item, kind, by_subject.get(node), &resolve)?;
435 items.push(item);
436 }
437 doc.as_object_mut().unwrap().insert(kind.to_string(), Value::Array(items));
438 }
439 if contains_null_hole(&doc) {
440 return Err("species corpus has a missing list element".into());
441 }
442 let corpus: SpeciesCorpus =
443 serde_json::from_value(doc).map_err(|e| format!("species corpus decode: {e}"))?;
444 corpus.validate()?;
445 Ok(corpus)
446}
447
448fn decode_fields(
449 target: &mut Value,
450 kind: &str,
451 quins: Option<&Vec<&NQuin>>,
452 resolve: &impl Fn(u64) -> Result<String, String>,
453) -> Result<(), String> {
454 let prefix = format!("{SPECIES_CORPUS_CONTEXT}#{kind}/");
455 for q in quins.map(Vec::as_slice).unwrap_or(&[]) {
456 let leaf_type = q.metadata & LEAF_MASK;
457 if matches!(leaf_type, LEAF_DECIMAL_MIRROR | LEAF_LINK | LEAF_TYPE) {
458 continue;
459 }
460 let p = resolve(q.predicate)?;
461 let path = p
462 .strip_prefix(&prefix)
463 .ok_or_else(|| format!("field predicate '{p}' is outside {kind}"))?;
464 let leaf = match leaf_type {
465 LEAF_EMPTY_OBJECT => Value::Object(Map::new()),
466 LEAF_EMPTY_ARRAY => Value::Array(Vec::new()),
467 LEAF_STRING => Value::String(resolve(q.object)?),
468 LEAF_NUMBER => {
469 let text = resolve(q.object)?;
470 Value::Number(
471 text.parse::<Number>()
472 .map_err(|_| format!("invalid number '{text}' at '{path}'"))?,
473 )
474 }
475 LEAF_BOOL => {
476 if q.object & INLINE_TAG_MASK != INLINE_TAG_BOOLEAN {
477 return Err(format!("expected boolean at '{path}'"));
478 }
479 Value::Bool(q.object & 1 != 0)
480 }
481 LEAF_NULL => Value::Null,
482 other => return Err(format!("unknown leaf type {other} at '{path}'")),
483 };
484 insert(target, &parse_path(path)?, leaf)?;
485 }
486 Ok(())
487}
488
489fn contains_null_hole(v: &Value) -> bool {
490 match v {
491 Value::Array(a) => a.iter().any(|x| x.is_null() || contains_null_hole(x)),
492 Value::Object(m) => m.values().any(contains_null_hole),
493 _ => false,
494 }
495}