Skip to main content

qualia_core_db/q42/
q42_volume.rs

1//! Unified `.q42` v3 volume — lexicon, block index, and LZ4-compressed SuperBlocks in one file.
2//!
3//! Layout (all little-endian):
4//! ```text
5//! [0..256)              Q42VolumeHeader
6//! [lex_offset ..]       Q42LEX blob (uncompressed)
7//! [bidx_offset ..]      BIDX blob (uncompressed)
8//! [block_dir_offset ..] block_count × BlockDirectoryEntry (16 bytes each)
9//! [data_offset ..]      concatenated LZ4 block payloads (lz4_flex prepend_size)
10//! ```
11//!
12//! Legacy v1 sidecars (`.q42.lex`, `.q42.bidx`) and separate `.c.q42` transport files
13//! are deprecated; new ingest writes v3 only.
14
15use std::collections::HashMap;
16#[cfg(not(target_arch = "wasm32"))]
17use std::fs::{File, OpenOptions};
18use std::io::{self, Write};
19#[cfg(not(target_arch = "wasm32"))]
20use std::io::{BufWriter, Read};
21#[cfg(not(target_arch = "wasm32"))]
22use std::path::Path;
23
24#[cfg(not(target_arch = "wasm32"))]
25use memmap2::{Mmap, MmapOptions};
26
27#[cfg(not(target_arch = "wasm32"))]
28use crate::q42_lex::Q42LexMmap;
29use crate::q42_lex::{LexError, LexiconEntry, LEX_MAGIC};
30use crate::{NQuin, QUINS_PER_BLOCK};
31
32#[path = "volume/mod.rs"]
33mod volume;
34
35// Byte-format / transport-neutral surface — compiles on wasm32 so verified
36// `.q42` pack entries decode in the browser through `Q42RangeVolume` over a
37// `Q42RangeSource`.
38pub use volume::{
39    decode_and_verify_car, deny_public_publication, encode_block_postings, encode_postings_section,
40    encode_raw_car, extract_entity_bytes, inclusive_entity_bytes, measure_bloom_false_positives,
41    provenance_path_for, quin_requires_sanctuary, validate_exact_range_response,
42    verify_car_bytes_as_q42_source, verify_source_sha256, BidxBlockRange, BidxMatchPage,
43    BlockFieldPostings, CidSha256, ClassificationCounts, IngestAccessPolicy,
44    IngestProvenanceRecord, OpfsCallbackRangeSource, OpfsSliceRangeSource, PublicationIntent,
45    Q42ByteRange, Q42LexiconRangeFactory, Q42LexiconSegment, Q42ObjectMatchPage,
46    Q42ObjectSearchCursor, Q42PublicationClass, Q42PublicationVerdict, Q42QueryMode,
47    Q42RangeQueryCursor, Q42RangeQueryPage, Q42RangeQueryPattern, Q42RangeQueryPlan,
48    Q42RangeQueryStrategy, Q42RangeSource, Q42RangeVolume, Q42RangeVolumeSet, Q42SegmentMatchPage,
49    Q42SegmentMatchRange, Q42SegmentRangeFactory, Q42Transport, Q42VolumeManifest,
50    Q42VolumeSegment, Q42VolumeSetQueryCursor, Q42VolumeSetQueryPage, VerifiedCarBlock,
51    VerifiedCarRangeSource, FIELD_POSTINGS_MAGIC, MAX_VOLUME_MANIFEST_BYTES,
52    RESIDENT_QUERY_MAX_BYTES, WasmSliceQ42Reader,
53};
54// Filesystem / memmap / HTTP / publish surface — native only.
55#[cfg(not(target_arch = "wasm32"))]
56pub use volume::{
57    admit_logical_size, append_segment_to_root, classify_q42_path, classify_q42_volume,
58    classify_q42_volume_set, compact_volume_set, compose_magnet, root_relative_path, sha1_hex_file,
59    verify_local_car_as_q42_source, verify_volume_set_from_root, CheckStatus, Completeness,
60    LocalFileRangeSource, NetworkCursor, Q42BlockCursor, Q42BlockMeta, Q42InspectReport, Q42Magnet,
61    Q42RolloverPublisher, Q42SectionReport, Q42VerifyReceipt, Q42VerifySetReport, Q42VolumeSet,
62    Q42VolumeSetMagnets, ScanBudget, StreamingQ42VolumeWriter, VerifyCheck, VerifyLevel,
63    DEFAULT_SEGMENT_MAX_BYTES,
64};
65#[cfg(not(target_arch = "wasm32"))]
66pub use volume::{
67    ipfs_gateway_range_source, ipns_gateway_range_source, HttpRangeSource,
68    IpfsGatewaySegmentFactory,
69};
70#[cfg(not(target_arch = "wasm32"))]
71pub use volume::{write_sorted_quins_volume, write_sorted_quins_volume_with_author};
72pub use crate::bundle::cbor_ld_manifest::PayloadCatalogueEntry;
73#[path = "volume/payload_catalogue.rs"]
74pub mod payload_catalogue;
75pub use payload_catalogue::{
76    decode_payload_catalogue_section, decode_payload_catalogue_with_bodies,
77    encode_payload_catalogue_section, encode_payload_catalogue_with_bodies,
78    header_payload_catalogue_range, set_header_payload_catalogue_range,
79    validate_payload_catalogue_entry, PayloadCatalogueBlob,
80};
81
82/// Public publication-policy gate surface (`Q42-00`).
83pub mod publication {
84    #[cfg(not(target_arch = "wasm32"))]
85    pub use super::volume::{classify_q42_path, classify_q42_volume, classify_q42_volume_set};
86    pub use super::volume::{
87        check_public_distribution_allowed, deny_public_publication, provenance_path_for,
88        quin_requires_sanctuary, CheckPublicDistribution, ClassificationCounts,
89        IngestAccessPolicy, IngestProvenanceRecord, PublicationIntent, Q42PublicationClass,
90        Q42PublicationVerdict, Q42Transport,
91    };
92}
93pub use publication::check_public_distribution_allowed;
94
95pub const Q42_MAGIC: [u8; 4] = [0x51, 0x34, 0x32, 0x00]; // "Q42\0"
96pub const Q42_VERSION_V3: u16 = 3;
97/// Current write generation. v4 volumes carry the namespaced paged Q42LEX
98/// (`q42_lex_ns`, ADR 0015) from `encode_lex`; the header layout itself is
99/// unchanged from v3. Readers accept v3 and v4 — existing pre-release data
100/// stays loadable until regenerated.
101pub const Q42_VERSION_V4: u16 = 4;
102pub const HEADER_SIZE: usize = 256;
103pub const SUPERBLOCK_SIZE: usize = 40_960;
104/// Conservative caller-buffer bound for an LZ4 `prepend_size` encoding of one
105/// Q42 SuperBlock.  The bound includes its four-byte decoded-size prefix.
106pub const MAX_COMPRESSED_SUPERBLOCK_SIZE: usize = SUPERBLOCK_SIZE + (SUPERBLOCK_SIZE / 255) + 36;
107pub const SUPERBLOCK_HEADER: usize = 160;
108pub const QUIN_SIZE: usize = 48;
109pub const BIDX_MAGIC: [u8; 4] = *b"BIDX";
110pub const FLAG_BLOCKS_LZ4: u16 = 0x0001;
111pub const FLAG_OBJECT_SORTED: u16 = 0x0002;
112pub const FLAG_VOLUME_ROOT: u16 = 0x0004;
113/// A per-SuperBlock subject/predicate/context range index is present in front
114/// matter.  It supplements (but does not replace) the object-sorted BIDX.
115pub const FLAG_FIELD_RANGES: u16 = 0x0008;
116/// Compact per-block S/P/C postings (or measured Bloom) are present.
117pub const FLAG_FIELD_POSTINGS: u16 = 0x0010;
118/// Affirmative Permissive Commons catalog. Required (with a clean Quin scan)
119/// before a public magnet / HTTP web-seed / IPFS pin is emitted.
120pub const FLAG_PERMISSIVE_COMMONS: u16 = 0x0020;
121/// Affirmative Sanctuary / Selfhood volume. Public hash addressing is denied.
122/// Writers also set this when any Quin is restricted, classified, medical,
123/// legal, fiduciary, or bilateral.
124pub const FLAG_SANCTUARY: u16 = 0x0040;
125/// Reserved: optional payload catalogue for oversized lexical / prose blobs
126/// (hash-keyed; not for media — use `.hmc`). Stub only until ADR writer lands.
127pub const FLAG_PAYLOAD_CATALOGUE: u16 = 0x0080;
128pub const FIELD_RANGE_INDEX_MAGIC: [u8; 4] = *b"FIDX";
129pub const FIELD_RANGE_INDEX_HEADER_BYTES: usize = 16;
130pub const FIELD_RANGE_INDEX_ENTRY_BYTES: usize = 48;
131
132/// Exhaustive cold-path verification receipt for one physical Q42 segment.
133#[derive(Clone, Copy, Debug, Eq, PartialEq)]
134pub struct Q42VerificationReceipt {
135    pub blocks_verified: u64,
136    pub quins_verified: u64,
137}
138
139/// Q42 volume header — 256 bytes, `repr(C, packed)`.
140/// v3 builds hard-reject files with `version < 3` — run `q42 migrate meta` first.
141#[repr(C, packed)]
142#[derive(Clone, Copy, Debug)]
143pub struct Q42VolumeHeader {
144    pub magic: [u8; 4],
145    pub version: u16,
146    pub flags: u16,
147    pub lex_offset: u64,
148    pub lex_length: u64,
149    pub bidx_offset: u64,
150    pub bidx_length: u64,
151    pub block_dir_offset: u64,
152    pub block_dir_length: u64,
153    pub data_offset: u64,
154    pub data_length: u64,
155    pub block_count: u64,
156    pub block_size: u32,
157    pub quins_per_block: u32,
158    // v3 extension fields (carved from former _reserved[0..88]):
159    pub temporal_index_offset: u64,
160    pub temporal_index_length: u64,
161    pub merkle_root: [u8; 32], // SHA3-256 of DAG root; all-zero = no history
162    pub assertion_timestamp: u64, // ms since Unix epoch when volume was last written
163    pub dag_root_offset: u64,  // offset into file of DagNode store; 0 = absent
164    pub dag_root_length: u64,  // byte length of DagNode store section
165
166    // Governance / Identity Bifurcation
167    pub natural_person_did_offset: u64, // Offset to human-reality declarative/consent DAG
168    pub software_agent_did_offset: u64, // Offset to agent-reality logic/policy DAG
169
170    pub _reserved: [u8; 80], // remaining reserved (88 named + 72 v3 ext + 16 gov + 80 = 256 bytes)
171}
172
173const _: () = assert!(
174    std::mem::size_of::<Q42VolumeHeader>() == 256,
175    "Q42VolumeHeader must be exactly 256 bytes — matches HEADER_SIZE constant"
176);
177
178impl Q42VolumeHeader {
179    /// Reject v2 files. Call before any read/write on a mapped header.
180    pub fn verify_version(&self) -> Result<(), String> {
181        // Copy fields out of the packed struct before comparing to avoid unaligned refs.
182        let magic = self.magic;
183        let version = { self.version };
184        if magic != Q42_MAGIC {
185            return Err(format!("bad magic {magic:?}"));
186        }
187        if version != Q42_VERSION_V3 && version != Q42_VERSION_V4 {
188            return Err(format!(
189                "Q42 file is version {version}; v3/v4 required (run q42 migrate meta for v2)"
190            ));
191        }
192        Ok(())
193    }
194
195    /// Embedded volume-manifest (offset, length) decoded from the
196    /// `_reserved` field; `None` unless `FLAG_VOLUME_ROOT` is set.
197    fn volume_manifest_range(&self) -> Option<(u64, u64)> {
198        if self.flags & FLAG_VOLUME_ROOT == 0 {
199            return None;
200        }
201        let reserved = self._reserved;
202        Some((
203            u64::from_le_bytes(reserved[0..8].try_into().unwrap()),
204            u64::from_le_bytes(reserved[8..16].try_into().unwrap()),
205        ))
206    }
207
208    /// Field-range-index (offset, length) from `_reserved`; `None`
209    /// unless `FLAG_FIELD_RANGES` is set.
210    fn field_range_index_range(&self) -> Option<(u64, u64)> {
211        if self.flags & FLAG_FIELD_RANGES == 0 {
212            return None;
213        }
214        let reserved = self._reserved;
215        Some((
216            u64::from_le_bytes(reserved[16..24].try_into().unwrap()),
217            u64::from_le_bytes(reserved[24..32].try_into().unwrap()),
218        ))
219    }
220
221    /// Field-postings (offset, length) from `_reserved`; `None` unless
222    /// `FLAG_FIELD_POSTINGS` is set.
223    fn field_postings_range(&self) -> Option<(u64, u64)> {
224        if self.flags & FLAG_FIELD_POSTINGS == 0 {
225            return None;
226        }
227        let reserved = self._reserved;
228        Some((
229            u64::from_le_bytes(reserved[32..40].try_into().unwrap()),
230            u64::from_le_bytes(reserved[40..48].try_into().unwrap()),
231        ))
232    }
233
234    /// Payload catalogue `(offset, length)` from `_reserved[48..64]`; `None`
235    /// unless `FLAG_PAYLOAD_CATALOGUE` (`0x0080`) is set.
236    pub fn payload_catalogue_range(&self) -> Option<(u64, u64)> {
237        header_payload_catalogue_range(self)
238    }
239
240    /// Build a minimal valid v4 header with all extension fields zeroed.
241    pub fn new_v4(
242        lex_offset: u64,
243        lex_length: u64,
244        bidx_offset: u64,
245        bidx_length: u64,
246        block_dir_offset: u64,
247        block_dir_length: u64,
248        data_offset: u64,
249        data_length: u64,
250        block_count: u64,
251        block_size: u32,
252        quins_per_block: u32,
253    ) -> Self {
254        let assertion_timestamp = std::time::SystemTime::now()
255            .duration_since(std::time::UNIX_EPOCH)
256            .map(|d| d.as_millis() as u64)
257            .unwrap_or(0);
258        Self {
259            magic: Q42_MAGIC,
260            version: Q42_VERSION_V4,
261            flags: FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED,
262            lex_offset,
263            lex_length,
264            bidx_offset,
265            bidx_length,
266            block_dir_offset,
267            block_dir_length,
268            data_offset,
269            data_length,
270            block_count,
271            block_size,
272            quins_per_block,
273            temporal_index_offset: 0,
274            temporal_index_length: 0,
275            merkle_root: [0u8; 32],
276            assertion_timestamp,
277            dag_root_offset: 0,
278            dag_root_length: 0,
279            natural_person_did_offset: 0,
280            software_agent_did_offset: 0,
281            _reserved: [0u8; 80],
282        }
283    }
284}
285
286#[repr(C, packed)]
287#[derive(Clone, Copy, Debug)]
288pub struct BlockDirectoryEntry {
289    pub rel_offset: u64,
290    pub comp_len: u32,
291    pub uncomp_len: u32,
292}
293
294impl BlockDirectoryEntry {
295    pub const SIZE: usize = 16;
296
297    /// Serialize the 16-byte entry (`rel_offset`, `comp_len`,
298    /// `uncomp_len`, little-endian) to `out`.
299    pub fn write_to(&self, out: &mut impl Write) -> io::Result<()> {
300        out.write_all(&self.rel_offset.to_le_bytes())?;
301        out.write_all(&self.comp_len.to_le_bytes())?;
302        out.write_all(&self.uncomp_len.to_le_bytes())?;
303        Ok(())
304    }
305
306    /// Deserialize a 16-byte little-endian directory entry.
307    fn from_bytes(buf: &[u8; 16]) -> Self {
308        Self {
309            rel_offset: u64::from_le_bytes(buf[0..8].try_into().unwrap()),
310            comp_len: u32::from_le_bytes(buf[8..12].try_into().unwrap()),
311            uncomp_len: u32::from_le_bytes(buf[12..16].try_into().unwrap()),
312        }
313    }
314}
315
316/// Upgrade a v1/v2 `.q42` file to v3 by rewriting the version field and zeroing
317/// the new extension fields in the 256-byte header.  Data blocks are untouched.
318#[cfg(not(target_arch = "wasm32"))]
319pub fn migrate_v2_to_v3(path: &Path) -> io::Result<()> {
320    use std::io::{Seek, SeekFrom};
321    let mut f = OpenOptions::new().read(true).write(true).open(path)?;
322    let mut header = [0u8; HEADER_SIZE];
323    f.read_exact(&mut header)?;
324    if &header[0..4] != &Q42_MAGIC {
325        return Err(io::Error::new(
326            io::ErrorKind::InvalidData,
327            "not a Q42 volume",
328        ));
329    }
330    let version = u16::from_le_bytes([header[4], header[5]]);
331    if version >= Q42_VERSION_V3 as u16 {
332        return Ok(()); // v3/v4 headers already share the unified layout
333    }
334    // Bump version to the current generation.
335    header[4..6].copy_from_slice(&(Q42_VERSION_V4 as u16).to_le_bytes());
336    // v2 only knew LZ4 + object-sorted. Later flags (root, FIDX, PIDX) must not
337    // survive a header bump with their reserved offsets zeroed.
338    let flags = u16::from_le_bytes([header[6], header[7]]) & (FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED);
339    header[6..8].copy_from_slice(&flags.to_le_bytes());
340    // Zero out the v3 extension fields (bytes 88..256 within the header).
341    header[88..256].fill(0);
342    f.seek(SeekFrom::Start(0))?;
343    f.write_all(&header)?;
344    f.flush()
345}
346
347/// Returns true if `path` begins with a unified volume header.
348#[cfg(not(target_arch = "wasm32"))]
349pub fn is_unified_volume(path: &Path) -> io::Result<bool> {
350    let mut f = File::open(path)?;
351    let mut magic = [0u8; 4];
352    f.read_exact(&mut magic)?;
353    Ok(magic == Q42_MAGIC)
354}
355
356/// Encode Q42LEX bytes from a hash → string map (namespaced paged **v4**:
357/// each page carries its own namespace dictionary and qualifying terms are
358/// stored as `(ns_id, local)` references — see `q42_lex_ns` and ADR 0015).
359pub fn encode_lex(lex: &HashMap<u64, String>) -> Result<Vec<u8>, LexError> {
360    crate::q42_lex_ns::serialize_namespaced_paged_lexicon(
361        lex,
362        crate::q42_lex::DEFAULT_LEX_PAGE_ENTRIES,
363    )
364}
365
366/// Encode Q42LEX bytes from a hash → LexiconEntry map (supports embedded triples).
367pub fn encode_lex_with_entries(lex: &HashMap<u64, LexiconEntry>) -> Result<Vec<u8>, LexError> {
368    let mut entries: Vec<(u64, &LexiconEntry)> = lex.iter().map(|(&h, e)| (h, e)).collect();
369    entries.sort_unstable_by_key(|&(h, _)| h);
370
371    let entry_count = entries.len() as u64;
372    let strings_offset = 32 + entry_count * 16;
373
374    let mut string_blob: Vec<u8> = Vec::new();
375    let mut index = Vec::with_capacity(entries.len() * 16);
376    for (hash, entry) in &entries {
377        let str_off = string_blob.len() as u64;
378        match entry {
379            LexiconEntry::String(text) => {
380                // Write type tag
381                string_blob.push(0x01);
382                let b = text.as_bytes();
383                let len = u16::try_from(b.len()).map_err(|_| LexError::TermTooLong)?;
384                string_blob.extend_from_slice(&len.to_le_bytes());
385                string_blob.extend_from_slice(&b[..len as usize]);
386            }
387            LexiconEntry::EmbeddedTriple(triple) => {
388                // Write type tag
389                string_blob.push(0x02);
390                for &id in triple {
391                    string_blob.extend_from_slice(&id.to_le_bytes());
392                }
393            }
394            LexiconEntry::Webizen(webid) => {
395                // Write type tag
396                string_blob.push(0x03);
397                let b = webid.as_bytes();
398                let len = u16::try_from(b.len()).map_err(|_| LexError::TermTooLong)?;
399                string_blob.extend_from_slice(&len.to_le_bytes());
400                string_blob.extend_from_slice(&b[..len as usize]);
401            }
402        }
403        index.extend_from_slice(&hash.to_le_bytes());
404        index.extend_from_slice(&str_off.to_le_bytes());
405    }
406
407    let mut out = Vec::with_capacity(strings_offset as usize + string_blob.len());
408    out.extend_from_slice(&LEX_MAGIC);
409    out.extend_from_slice(&entry_count.to_le_bytes());
410    out.extend_from_slice(&strings_offset.to_le_bytes());
411    out.extend_from_slice(&1u64.to_le_bytes());
412    out.extend_from_slice(&index);
413    out.extend_from_slice(&string_blob);
414    Ok(out)
415}
416
417/// Encode BIDX bytes from per-block min/max object hashes.
418pub fn encode_bidx(ranges: &[(u64, u64)]) -> Vec<u8> {
419    let block_count = ranges.len() as u32;
420    let mut out = Vec::with_capacity(16 + ranges.len() * 16);
421    out.extend_from_slice(&BIDX_MAGIC);
422    out.extend_from_slice(&1u32.to_le_bytes());
423    out.extend_from_slice(&block_count.to_le_bytes());
424    out.extend_from_slice(&0u32.to_le_bytes());
425    for (min, max) in ranges {
426        out.extend_from_slice(&min.to_le_bytes());
427        out.extend_from_slice(&max.to_le_bytes());
428    }
429    out
430}
431
432/// Encode conservative per-block ranges for the non-object triple fields.
433/// Every range encloses all values in its block, so a range miss is safe to
434/// skip while a hit still requires normal Quin-level matching.
435pub fn encode_field_range_index(ranges: &[(u64, u64, u64, u64, u64, u64)]) -> Vec<u8> {
436    let mut out = Vec::with_capacity(
437        FIELD_RANGE_INDEX_HEADER_BYTES + ranges.len() * FIELD_RANGE_INDEX_ENTRY_BYTES,
438    );
439    out.extend_from_slice(&FIELD_RANGE_INDEX_MAGIC);
440    out.extend_from_slice(&1u32.to_le_bytes());
441    out.extend_from_slice(&(ranges.len() as u32).to_le_bytes());
442    out.extend_from_slice(&0u32.to_le_bytes());
443    for range in ranges {
444        for value in [range.0, range.1, range.2, range.3, range.4, range.5] {
445            out.extend_from_slice(&value.to_le_bytes());
446        }
447    }
448    out
449}
450
451/// Encode Q42LEX bytes from a hash → LexiconEntry map (supports embedded triples).
452pub fn encode_superblock(seq_id: u64, quins: &[NQuin]) -> [u8; SUPERBLOCK_SIZE] {
453    debug_assert!(quins.len() <= QUINS_PER_BLOCK);
454    let mut block = [0u8; SUPERBLOCK_SIZE];
455    block[0..8].copy_from_slice(&seq_id.to_le_bytes());
456    block[16..24].copy_from_slice(&(quins.len() as u64).to_le_bytes());
457    let zero = [0u8; QUIN_SIZE];
458    let mut off = SUPERBLOCK_HEADER;
459    for q in quins {
460        block[off..off + QUIN_SIZE].copy_from_slice(bytemuck::bytes_of(q));
461        off += QUIN_SIZE;
462    }
463    for _ in quins.len()..QUINS_PER_BLOCK {
464        block[off..off + QUIN_SIZE].copy_from_slice(&zero);
465        off += QUIN_SIZE;
466    }
467    block
468}
469
470/// Live Quins in a decompressed SuperBlock (160-byte header + 48-byte slots).
471pub fn decode_superblock_quins(block: &[u8]) -> io::Result<Vec<NQuin>> {
472    if block.len() < SUPERBLOCK_HEADER {
473        return Err(io::Error::new(
474            io::ErrorKind::InvalidData,
475            "SuperBlock shorter than its 160-byte header",
476        ));
477    }
478    let live = u64::from_le_bytes(block[16..24].try_into().unwrap()) as usize;
479    if live > QUINS_PER_BLOCK {
480        return Err(io::Error::new(
481            io::ErrorKind::InvalidData,
482            "SuperBlock live Quin count exceeds capacity",
483        ));
484    }
485    let mut out = Vec::with_capacity(live);
486    for index in 0..live {
487        let offset = SUPERBLOCK_HEADER + index * QUIN_SIZE;
488        let end = offset + QUIN_SIZE;
489        if end > block.len() {
490            return Err(io::Error::new(
491                io::ErrorKind::InvalidData,
492                "SuperBlock truncated inside a Quin slot",
493            ));
494        }
495        out.push(bytemuck::pod_read_unaligned(&block[offset..end]));
496    }
497    Ok(out)
498}
499
500/// Serialize a v3/v4 volume header to its fixed `HEADER_SIZE` byte
501/// layout (core fields 0–88, v3 extensions 88–176, reserved tail).
502pub fn header_to_bytes(h: &Q42VolumeHeader) -> [u8; HEADER_SIZE] {
503    let mut buf = [0u8; HEADER_SIZE];
504    // Core fields (0..88)
505    buf[0..4].copy_from_slice(&h.magic);
506    buf[4..6].copy_from_slice(&h.version.to_le_bytes());
507    buf[6..8].copy_from_slice(&h.flags.to_le_bytes());
508    buf[8..16].copy_from_slice(&h.lex_offset.to_le_bytes());
509    buf[16..24].copy_from_slice(&h.lex_length.to_le_bytes());
510    buf[24..32].copy_from_slice(&h.bidx_offset.to_le_bytes());
511    buf[32..40].copy_from_slice(&h.bidx_length.to_le_bytes());
512    buf[40..48].copy_from_slice(&h.block_dir_offset.to_le_bytes());
513    buf[48..56].copy_from_slice(&h.block_dir_length.to_le_bytes());
514    buf[56..64].copy_from_slice(&h.data_offset.to_le_bytes());
515    buf[64..72].copy_from_slice(&h.data_length.to_le_bytes());
516    buf[72..80].copy_from_slice(&h.block_count.to_le_bytes());
517    buf[80..84].copy_from_slice(&h.block_size.to_le_bytes());
518    buf[84..88].copy_from_slice(&h.quins_per_block.to_le_bytes());
519    // v3 extension fields (88..160)
520    buf[88..96].copy_from_slice(&h.temporal_index_offset.to_le_bytes());
521    buf[96..104].copy_from_slice(&h.temporal_index_length.to_le_bytes());
522    buf[104..136].copy_from_slice(&h.merkle_root);
523    buf[136..144].copy_from_slice(&h.assertion_timestamp.to_le_bytes());
524    buf[144..152].copy_from_slice(&h.dag_root_offset.to_le_bytes());
525    buf[152..160].copy_from_slice(&h.dag_root_length.to_le_bytes());
526    buf[160..168].copy_from_slice(&h.natural_person_did_offset.to_le_bytes());
527    buf[168..176].copy_from_slice(&h.software_agent_did_offset.to_le_bytes());
528    buf[176..256].copy_from_slice(&h._reserved);
529    buf
530}
531
532/// Deserialize and validate a header — rejects bad magic and
533/// non-v3/v4 versions.
534fn header_from_bytes(buf: &[u8; HEADER_SIZE]) -> io::Result<Q42VolumeHeader> {
535    if buf[0..4] != Q42_MAGIC {
536        return Err(io::Error::new(
537            io::ErrorKind::InvalidData,
538            "invalid Q42 magic",
539        ));
540    }
541    let version = u16::from_le_bytes(buf[4..6].try_into().unwrap());
542    if version != Q42_VERSION_V3 && version != Q42_VERSION_V4 {
543        return Err(io::Error::new(
544            io::ErrorKind::InvalidData,
545            format!("Q42 file is version {version}; v3/v4 required"),
546        ));
547    }
548    Ok(Q42VolumeHeader {
549        magic: Q42_MAGIC,
550        version,
551        flags: u16::from_le_bytes(buf[6..8].try_into().unwrap()),
552        lex_offset: u64::from_le_bytes(buf[8..16].try_into().unwrap()),
553        lex_length: u64::from_le_bytes(buf[16..24].try_into().unwrap()),
554        bidx_offset: u64::from_le_bytes(buf[24..32].try_into().unwrap()),
555        bidx_length: u64::from_le_bytes(buf[32..40].try_into().unwrap()),
556        block_dir_offset: u64::from_le_bytes(buf[40..48].try_into().unwrap()),
557        block_dir_length: u64::from_le_bytes(buf[48..56].try_into().unwrap()),
558        data_offset: u64::from_le_bytes(buf[56..64].try_into().unwrap()),
559        data_length: u64::from_le_bytes(buf[64..72].try_into().unwrap()),
560        block_count: u64::from_le_bytes(buf[72..80].try_into().unwrap()),
561        block_size: u32::from_le_bytes(buf[80..84].try_into().unwrap()),
562        quins_per_block: u32::from_le_bytes(buf[84..88].try_into().unwrap()),
563        temporal_index_offset: u64::from_le_bytes(buf[88..96].try_into().unwrap()),
564        temporal_index_length: u64::from_le_bytes(buf[96..104].try_into().unwrap()),
565        merkle_root: buf[104..136].try_into().unwrap(),
566        assertion_timestamp: u64::from_le_bytes(buf[136..144].try_into().unwrap()),
567        dag_root_offset: u64::from_le_bytes(buf[144..152].try_into().unwrap()),
568        dag_root_length: u64::from_le_bytes(buf[152..160].try_into().unwrap()),
569        natural_person_did_offset: u64::from_le_bytes(buf[160..168].try_into().unwrap()),
570        software_agent_did_offset: u64::from_le_bytes(buf[168..176].try_into().unwrap()),
571        _reserved: buf[176..256].try_into().unwrap(),
572    })
573}
574
575/// Write a unified v3 `.q42` volume.
576#[cfg(not(target_arch = "wasm32"))]
577pub fn write_unified_volume(
578    path: &Path,
579    lex: &HashMap<u64, String>,
580    block_ranges: &[(u64, u64)],
581    blocks: &[Vec<NQuin>],
582) -> io::Result<()> {
583    if blocks.len() != block_ranges.len() {
584        return Err(io::Error::new(
585            io::ErrorKind::InvalidInput,
586            "block count mismatch",
587        ));
588    }
589    let mut writer = StreamingQ42VolumeWriter::new(lex)?;
590    for ((seq, quins), declared_range) in blocks.iter().enumerate().zip(block_ranges) {
591        let actual_range = object_range(quins)?;
592        if actual_range != *declared_range {
593            return Err(io::Error::new(
594                io::ErrorKind::InvalidInput,
595                format!(
596                    "declared BIDX range {:?} does not match block {seq} object range {:?}",
597                    declared_range, actual_range
598                ),
599            ));
600        }
601        writer.push_block(seq as u64, quins)?;
602    }
603    writer.finish(path)
604}
605
606/// Write a unified v3 .q42 volume with embedded triple support.
607///
608/// Accepts `HashMap<u64, LexiconEntry>` to support SPARQL-Star embedded triples.
609#[cfg(not(target_arch = "wasm32"))]
610pub fn write_unified_volume_with_entries(
611    path: &Path,
612    lex: &HashMap<u64, LexiconEntry>,
613    block_ranges: &[(u64, u64)],
614    blocks: &[Vec<NQuin>],
615) -> io::Result<()> {
616    if blocks.len() != block_ranges.len() {
617        return Err(io::Error::new(
618            io::ErrorKind::InvalidInput,
619            "block count mismatch",
620        ));
621    }
622    let mut builder = UnifiedVolumeBuilder::with_lex_entries(lex).map_err(lex_error_to_io)?;
623    for ((seq, quins), declared_range) in blocks.iter().enumerate().zip(block_ranges) {
624        let actual_range = object_range(quins)?;
625        if actual_range != *declared_range {
626            return Err(io::Error::new(
627                io::ErrorKind::InvalidInput,
628                format!(
629                    "declared BIDX range {:?} does not match block {seq} object range {:?}",
630                    declared_range, actual_range
631                ),
632            ));
633        }
634        builder.push_block(seq as u64, quins)?;
635    }
636    builder.finish(path)
637}
638
639/// Publish a root Q42 segment whose front matter catalogs immutable child
640/// segments. The root is intentionally data-empty; query code opens the
641/// catalog snapshot through [`Q42VolumeSet`].
642#[cfg(not(target_arch = "wasm32"))]
643pub fn write_volume_root(path: &Path, manifest: &Q42VolumeManifest) -> io::Result<()> {
644    UnifiedVolumeBuilder::with_empty_lex()
645        .with_volume_manifest(manifest)?
646        .finish(path)
647}
648
649/// Same as [`write_volume_root`], but marks the catalog as Permissive Commons.
650/// Use for public ontologies / knowledge graphs, never for personal volumes.
651#[cfg(not(target_arch = "wasm32"))]
652pub fn write_volume_root_for_commons(path: &Path, manifest: &Q42VolumeManifest) -> io::Result<()> {
653    UnifiedVolumeBuilder::with_empty_lex()
654        .with_permissive_commons()
655        .with_volume_manifest(manifest)?
656        .finish(path)
657}
658
659/// Publish a logical-volume root with a lossless shared Q42LEX in its front
660/// matter. Child data segments may keep empty local lexicons because all term
661/// resolution for the snapshot is recoverable from this immutable root.
662#[cfg(not(target_arch = "wasm32"))]
663pub fn write_volume_root_with_lex(
664    path: &Path,
665    lex: &HashMap<u64, String>,
666    manifest: &Q42VolumeManifest,
667) -> io::Result<()> {
668    UnifiedVolumeBuilder::with_lex_map(lex)
669        .map_err(lex_error_to_io)?
670        .with_volume_manifest(manifest)?
671        .finish(path)
672}
673
674#[cfg(not(target_arch = "wasm32"))]
675/// Map a `LexError` to `io::Error` (`InvalidInput`, message includes
676/// the lex error debug form).
677fn lex_error_to_io(error: LexError) -> io::Error {
678    io::Error::new(
679        io::ErrorKind::InvalidInput,
680        format!("invalid Q42LEX: {error:?}"),
681    )
682}
683
684/// Return the object-hash interval for one canonical SuperBlock.
685///
686/// The v3 BIDX is only sound when every block is internally object-sorted and
687/// the blocks themselves form one non-decreasing stream. Rejecting malformed
688/// input here prevents a writer from advertising an index it cannot honour.
689fn object_range(quins: &[NQuin]) -> io::Result<(u64, u64)> {
690    let Some(first) = quins.first() else {
691        return Err(io::Error::new(
692            io::ErrorKind::InvalidInput,
693            "a Q42 SuperBlock must contain at least one Quin",
694        ));
695    };
696    if quins.len() > QUINS_PER_BLOCK {
697        return Err(io::Error::new(
698            io::ErrorKind::InvalidInput,
699            format!(
700                "Q42 SuperBlock contains {} Quins, exceeding capacity {QUINS_PER_BLOCK}",
701                quins.len()
702            ),
703        ));
704    }
705
706    let mut previous = first.object;
707    for quin in &quins[1..] {
708        if quin.object < previous {
709            return Err(io::Error::new(
710                io::ErrorKind::InvalidInput,
711                "Q42 SuperBlock Quins must be sorted by object hash",
712            ));
713        }
714        previous = quin.object;
715    }
716    Ok((first.object, previous))
717}
718
719/// Incremental builder for large external-sort merges (one SuperBlock at a time).
720pub struct UnifiedVolumeBuilder {
721    lex_bytes: Vec<u8>,
722    volume_manifest: Option<Vec<u8>>,
723    block_ranges: Vec<(u64, u64)>,
724    field_ranges: Vec<(u64, u64, u64, u64, u64, u64)>,
725    dir_entries: Vec<BlockDirectoryEntry>,
726    field_postings: Vec<BlockFieldPostings>,
727    data_blob: Vec<u8>,
728    /// Merkle-DAG commit history — populated as each SuperBlock is pushed.
729    dag_store: crate::git_bridge::DagStore,
730    /// DID hash of the agent performing this ingest (0 = system/anonymous).
731    author_did: u64,
732    /// Hash of the last committed DagNode; all-zero until first push.
733    last_dag_hash: [u8; 32],
734    /// Last object hash admitted to the object-sorted stream.
735    last_object_hash: Option<u64>,
736    publication_commons: bool,
737    publication_sanctuary: bool,
738    sanctuary_quin_count: u64,
739    payload_catalogue_blobs: Vec<PayloadCatalogueBlob>,
740    payload_catalogue_bytes: Option<Vec<u8>>,
741}
742
743impl UnifiedVolumeBuilder {
744    /// Builder whose Q42LEX section is encoded from `lex`
745    /// (hash → term string).
746    pub fn with_lex_map(lex: &HashMap<u64, String>) -> Result<Self, LexError> {
747        Ok(Self {
748            lex_bytes: encode_lex(lex)?,
749            volume_manifest: None,
750            block_ranges: Vec::new(),
751            field_ranges: Vec::new(),
752            dir_entries: Vec::new(),
753            field_postings: Vec::new(),
754            data_blob: Vec::new(),
755            dag_store: crate::git_bridge::DagStore::new(),
756            author_did: 0,
757            last_dag_hash: [0u8; 32],
758            last_object_hash: None,
759            publication_commons: false,
760            publication_sanctuary: false,
761            sanctuary_quin_count: 0,
762            payload_catalogue_blobs: Vec::new(),
763            payload_catalogue_bytes: None,
764        })
765    }
766
767    /// Create a builder with a lexicon that supports embedded triples (LexiconEntry).
768    pub fn with_lex_entries(lex: &HashMap<u64, LexiconEntry>) -> Result<Self, LexError> {
769        Ok(Self {
770            lex_bytes: encode_lex_with_entries(lex)?,
771            volume_manifest: None,
772            block_ranges: Vec::new(),
773            field_ranges: Vec::new(),
774            dir_entries: Vec::new(),
775            field_postings: Vec::new(),
776            data_blob: Vec::new(),
777            dag_store: crate::git_bridge::DagStore::new(),
778            author_did: 0,
779            last_dag_hash: [0u8; 32],
780            last_object_hash: None,
781            publication_commons: false,
782            publication_sanctuary: false,
783            sanctuary_quin_count: 0,
784            payload_catalogue_blobs: Vec::new(),
785            payload_catalogue_bytes: None,
786        })
787    }
788
789    /// Append a UTF-8 prose/descriptor payload (`ADR 0014`) to this volume's `PCAT` section.
790    /// Rejects media MIME types (`image/*`, `audio/*`, `video/*`, `.p64`, `.10d`).
791    pub fn add_payload_catalogue_utf8(
792        &mut self,
793        payload_key: &str,
794        mime_type: &str,
795        language: Option<&str>,
796        role: &str,
797        utf8_body: &str,
798    ) -> io::Result<PayloadCatalogueEntry> {
799        let blob = PayloadCatalogueBlob::new_utf8(
800            payload_key,
801            mime_type,
802            language.map(str::to_string),
803            role,
804            utf8_body,
805        )
806        .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
807        let entry = blob.entry.clone();
808        self.payload_catalogue_blobs.push(blob);
809        let encoded = encode_payload_catalogue_with_bodies(&self.payload_catalogue_blobs)
810            .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
811        self.payload_catalogue_bytes = Some(encoded);
812        Ok(entry)
813    }
814
815    /// Attach a slice of [`PayloadCatalogueBlob`] records (metadata + UTF-8 body arena).
816    pub fn with_payload_catalogue_blobs(mut self, blobs: &[PayloadCatalogueBlob]) -> io::Result<Self> {
817        if blobs.is_empty() {
818            self.payload_catalogue_blobs.clear();
819            self.payload_catalogue_bytes = None;
820            return Ok(self);
821        }
822        let encoded = encode_payload_catalogue_with_bodies(blobs)
823            .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
824        self.payload_catalogue_blobs = blobs.to_vec();
825        self.payload_catalogue_bytes = Some(encoded);
826        Ok(self)
827    }
828
829    /// Attach metadata-only [`PayloadCatalogueEntry`] descriptors.
830    pub fn with_payload_catalogue_entries(
831        mut self,
832        entries: &[PayloadCatalogueEntry],
833    ) -> io::Result<Self> {
834        if entries.is_empty() {
835            self.payload_catalogue_blobs.clear();
836            self.payload_catalogue_bytes = None;
837            return Ok(self);
838        }
839        let encoded = encode_payload_catalogue_section(entries)
840            .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
841        self.payload_catalogue_blobs.clear();
842        self.payload_catalogue_bytes = Some(encoded);
843        Ok(self)
844    }
845
846    /// Builder with an empty (but structurally valid) Q42LEX section.
847    pub fn with_empty_lex() -> Self {
848        Self::with_lex_map(&HashMap::new()).expect("an empty Q42LEX is valid")
849    }
850
851    /// Embed an immutable logical-volume descriptor in this root Q42 file's
852    /// front matter. Child segments remain ordinary self-contained Q42 files.
853    pub fn with_volume_manifest(mut self, manifest: &Q42VolumeManifest) -> io::Result<Self> {
854        self.volume_manifest = Some(manifest.encode()?);
855        Ok(self)
856    }
857
858    /// Set the author DID for DAG commit nodes (optional; defaults to 0 = system).
859    pub fn with_author_did(mut self, did: u64) -> Self {
860        self.author_did = did;
861        self
862    }
863
864    /// Affirm this tiny embedded volume is a Permissive Commons catalog.
865    pub fn with_permissive_commons(mut self) -> Self {
866        self.publication_commons = true;
867        self
868    }
869
870    /// Affirm this volume is Sanctuary / Selfhood.
871    pub fn with_sanctuary(mut self) -> Self {
872        self.publication_sanctuary = true;
873        self
874    }
875
876    /// Append one SuperBlock: enforces global object-hash sort order,
877    /// accumulates block/field ranges, postings and the LZ4'd data
878    /// blob, and commits the block to the Merkle-DAG
879    /// (genesis on first block).
880    pub fn push_block(&mut self, seq_id: u64, quins: &[NQuin]) -> io::Result<()> {
881        let (min_hash, max_hash) = object_range(quins)?;
882        if let Some(previous) = self.last_object_hash {
883            if min_hash < previous {
884                return Err(io::Error::new(
885                    io::ErrorKind::InvalidInput,
886                    format!(
887                        "block {seq_id} starts at object hash {min_hash:#018X}, before prior block maximum {previous:#018X}"
888                    ),
889                ));
890            }
891        }
892        self.block_ranges.push((min_hash, max_hash));
893        let mut subject_min = quins[0].subject;
894        let mut subject_max = quins[0].subject;
895        let mut predicate_min = quins[0].predicate;
896        let mut predicate_max = quins[0].predicate;
897        let mut context_min = quins[0].context;
898        let mut context_max = quins[0].context;
899        for quin in &quins[1..] {
900            subject_min = subject_min.min(quin.subject);
901            subject_max = subject_max.max(quin.subject);
902            predicate_min = predicate_min.min(quin.predicate);
903            predicate_max = predicate_max.max(quin.predicate);
904            context_min = context_min.min(quin.context);
905            context_max = context_max.max(quin.context);
906        }
907        self.sanctuary_quin_count += quins
908            .iter()
909            .filter(|quin| volume::quin_requires_sanctuary(quin))
910            .count() as u64;
911        self.field_ranges.push((
912            subject_min,
913            subject_max,
914            predicate_min,
915            predicate_max,
916            context_min,
917            context_max,
918        ));
919        self.field_postings
920            .push(BlockFieldPostings::from_quins(quins));
921        let raw = encode_superblock(seq_id, quins);
922        let compressed = lz4_flex::compress_prepend_size(&raw);
923        self.dir_entries.push(BlockDirectoryEntry {
924            rel_offset: self.data_blob.len() as u64,
925            comp_len: compressed.len() as u32,
926            uncomp_len: SUPERBLOCK_SIZE as u32,
927        });
928        self.data_blob.extend_from_slice(&compressed);
929
930        // Commit this SuperBlock to the Merkle-DAG.
931        let ts = std::time::SystemTime::now()
932            .duration_since(std::time::UNIX_EPOCH)
933            .map(|d| d.as_millis() as u64)
934            .unwrap_or(0);
935        let msg = format!("ingest block {seq_id}");
936        self.last_dag_hash = if self.last_dag_hash == [0u8; 32] {
937            self.dag_store
938                .genesis_node(quins, self.author_did, ts, &msg)
939        } else {
940            self.dag_store
941                .commit_node(self.last_dag_hash, quins, self.author_did, ts, &msg)
942        };
943        self.last_object_hash = Some(max_hash);
944        Ok(())
945    }
946
947    /// SuperBlocks pushed so far.
948    pub fn block_count(&self) -> u64 {
949        self.block_ranges.len() as u64
950    }
951
952    /// Assemble and write the complete `.q42` file to `path`
953    /// (native only — needs filesystem access).
954    #[cfg(not(target_arch = "wasm32"))]
955    pub fn finish(self, path: &Path) -> io::Result<()> {
956        let bytes = self.finish_to_bytes();
957        let out = OpenOptions::new()
958            .create(true)
959            .write(true)
960            .truncate(true)
961            .open(path)?;
962        let mut w = BufWriter::new(out);
963        w.write_all(&bytes)?;
964        w.flush()?;
965        Ok(())
966    }
967
968    /// Serialise the unified v3 `.q42` volume to an in-memory byte buffer.
969    ///
970    /// Byte-for-byte the same volume [`finish`](Self::finish) writes to disk, but
971    /// returned as bytes for callers that embed a `.q42` *inside another container*
972    /// — e.g. a [`crate::bundle`] `.hmc` pack — or ship it over the wire without
973    /// touching the filesystem. `finish` is exactly this plus a file write.
974    pub fn finish_to_bytes(self) -> Vec<u8> {
975        let bidx_bytes = encode_bidx(&self.block_ranges);
976        let field_range_bytes = encode_field_range_index(&self.field_ranges);
977        let postings_bytes = encode_postings_section(&self.field_postings);
978        let block_count = self.block_ranges.len() as u64;
979        let manifest_bytes = self.volume_manifest.as_deref().unwrap_or(&[]);
980
981        let lex_offset = HEADER_SIZE as u64;
982        let manifest_offset = lex_offset + self.lex_bytes.len() as u64;
983        let bidx_offset = manifest_offset + manifest_bytes.len() as u64;
984        let field_range_offset = bidx_offset + bidx_bytes.len() as u64;
985        let postings_offset = field_range_offset + field_range_bytes.len() as u64;
986        let block_dir_offset = postings_offset + postings_bytes.len() as u64;
987        let block_dir_length = block_count * BlockDirectoryEntry::SIZE as u64;
988        let data_offset = block_dir_offset + block_dir_length;
989
990        let assertion_timestamp = std::time::SystemTime::now()
991            .duration_since(std::time::UNIX_EPOCH)
992            .map(|d| d.as_millis() as u64)
993            .unwrap_or(0);
994
995        // Serialize the Merkle-DAG and compute layout.
996        let dag_bytes = self.dag_store.serialize();
997        let dag_root_offset = if dag_bytes.is_empty() {
998            0
999        } else {
1000            data_offset + self.data_blob.len() as u64
1001        };
1002        let dag_root_length = dag_bytes.len() as u64;
1003
1004        // merkle_root = SHA-256 of last committed DagNode hash (all-zero if no blocks).
1005        let merkle_root = if self.last_dag_hash == [0u8; 32] {
1006            [0u8; 32]
1007        } else {
1008            // Re-hash the tip hash so the header field is a hash-of-hash, not the raw node hash.
1009            use sha2::{Digest, Sha256};
1010            let mut h = Sha256::new();
1011            h.update(self.last_dag_hash);
1012            h.finalize().into()
1013        };
1014
1015        let mut reserved = [0u8; 80];
1016        let mut flags = FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED;
1017        if self.publication_sanctuary || self.sanctuary_quin_count > 0 {
1018            flags |= FLAG_SANCTUARY;
1019        } else if self.publication_commons {
1020            flags |= FLAG_PERMISSIVE_COMMONS;
1021        }
1022        if !self.field_ranges.is_empty() {
1023            flags |= FLAG_FIELD_RANGES;
1024            reserved[16..24].copy_from_slice(&field_range_offset.to_le_bytes());
1025            reserved[24..32].copy_from_slice(&(field_range_bytes.len() as u64).to_le_bytes());
1026        }
1027        if !self.field_postings.is_empty() {
1028            flags |= FLAG_FIELD_POSTINGS;
1029            reserved[32..40].copy_from_slice(&postings_offset.to_le_bytes());
1030            reserved[40..48].copy_from_slice(&(postings_bytes.len() as u64).to_le_bytes());
1031        }
1032        if !manifest_bytes.is_empty() {
1033            flags |= FLAG_VOLUME_ROOT;
1034            reserved[0..8].copy_from_slice(&manifest_offset.to_le_bytes());
1035            reserved[8..16].copy_from_slice(&(manifest_bytes.len() as u64).to_le_bytes());
1036        }
1037        let pcat_bytes = self.payload_catalogue_bytes.as_deref().unwrap_or(&[]);
1038        let mut header = Q42VolumeHeader {
1039            magic: Q42_MAGIC,
1040            version: Q42_VERSION_V4,
1041            flags,
1042            lex_offset,
1043            lex_length: self.lex_bytes.len() as u64,
1044            bidx_offset,
1045            bidx_length: bidx_bytes.len() as u64,
1046            block_dir_offset,
1047            block_dir_length,
1048            data_offset,
1049            data_length: self.data_blob.len() as u64,
1050            block_count,
1051            block_size: SUPERBLOCK_SIZE as u32,
1052            quins_per_block: QUINS_PER_BLOCK as u32,
1053            temporal_index_offset: 0,
1054            temporal_index_length: 0,
1055            merkle_root,
1056            assertion_timestamp,
1057            dag_root_offset,
1058            dag_root_length,
1059            natural_person_did_offset: 0,
1060            software_agent_did_offset: 0,
1061            _reserved: reserved,
1062        };
1063        if !pcat_bytes.is_empty() {
1064            let pcat_offset =
1065                data_offset + self.data_blob.len() as u64 + dag_bytes.len() as u64;
1066            set_header_payload_catalogue_range(&mut header, pcat_offset, pcat_bytes.len() as u64);
1067        }
1068
1069        let mut out = Vec::with_capacity(
1070            HEADER_SIZE
1071                + self.lex_bytes.len()
1072                + manifest_bytes.len()
1073                + bidx_bytes.len()
1074                + field_range_bytes.len()
1075                + postings_bytes.len()
1076                + block_dir_length as usize
1077                + self.data_blob.len()
1078                + dag_bytes.len()
1079                + pcat_bytes.len(),
1080        );
1081        out.extend_from_slice(&header_to_bytes(&header));
1082        out.extend_from_slice(&self.lex_bytes);
1083        out.extend_from_slice(manifest_bytes);
1084        out.extend_from_slice(&bidx_bytes);
1085        out.extend_from_slice(&field_range_bytes);
1086        out.extend_from_slice(&postings_bytes);
1087        for entry in &self.dir_entries {
1088            // Writing to a `Vec<u8>` is infallible.
1089            entry.write_to(&mut out).expect("Vec<u8> write cannot fail");
1090        }
1091        out.extend_from_slice(&self.data_blob);
1092        if !dag_bytes.is_empty() {
1093            out.extend_from_slice(&dag_bytes);
1094        }
1095        if !pcat_bytes.is_empty() {
1096            out.extend_from_slice(pcat_bytes);
1097        }
1098        out
1099    }
1100}
1101
1102/// Memory-mapped unified v2 volume reader.
1103#[cfg(not(target_arch = "wasm32"))]
1104pub struct Q42Volume {
1105    mmap: Mmap,
1106    header: Q42VolumeHeader,
1107}
1108
1109#[cfg(not(target_arch = "wasm32"))]
1110impl Q42Volume {
1111    /// Memory-map/open a `.q42` volume, validating header and section
1112    /// bounds.
1113    pub fn open(path: &Path) -> io::Result<Self> {
1114        let file = File::open(path)?;
1115        let mmap = unsafe { MmapOptions::new().map(&file)? };
1116        if mmap.len() < HEADER_SIZE {
1117            return Err(io::Error::new(
1118                io::ErrorKind::UnexpectedEof,
1119                "file too small for Q42 header",
1120            ));
1121        }
1122        let mut hdr_buf = [0u8; HEADER_SIZE];
1123        hdr_buf.copy_from_slice(&mmap[0..HEADER_SIZE]);
1124        let header = header_from_bytes(&hdr_buf)?;
1125        volume::validate_volume_structure(&header, &mmap)?;
1126        Ok(Self { mmap, header })
1127    }
1128
1129    /// The parsed volume header.
1130    pub fn header(&self) -> &Q42VolumeHeader {
1131        &self.header
1132    }
1133
1134    /// The whole volume as a byte slice.
1135    pub fn as_bytes(&self) -> &[u8] {
1136        &self.mmap
1137    }
1138
1139    /// Raw Q42LEX section bytes.
1140    pub fn lex_bytes(&self) -> &[u8] {
1141        let start = self.header.lex_offset as usize;
1142        let end = start + self.header.lex_length as usize;
1143        &self.mmap[start..end]
1144    }
1145
1146    /// Zero-copy `Q42LexMmap` view over the lexicon section.
1147    pub fn lex_view(&self) -> Result<Q42LexMmap<'_>, LexError> {
1148        Q42LexMmap::from_bytes(self.lex_bytes())
1149    }
1150
1151    /// Raw BIDX section bytes.
1152    pub fn bidx_bytes(&self) -> &[u8] {
1153        let start = self.header.bidx_offset as usize;
1154        let end = start + self.header.bidx_length as usize;
1155        &self.mmap[start..end]
1156    }
1157
1158    /// Return the root descriptor bytes embedded between the lexicon and BIDX.
1159    pub fn volume_manifest_bytes(&self) -> io::Result<Option<&[u8]>> {
1160        let Some((offset, length)) = self.header.volume_manifest_range() else {
1161            return Ok(None);
1162        };
1163        let length = usize::try_from(length).map_err(|_| {
1164            io::Error::new(
1165                io::ErrorKind::InvalidData,
1166                "Q42 manifest length exceeds platform",
1167            )
1168        })?;
1169        if length == 0 || length > MAX_VOLUME_MANIFEST_BYTES {
1170            return Err(io::Error::new(
1171                io::ErrorKind::InvalidData,
1172                "Q42 root has an invalid embedded volume manifest length",
1173            ));
1174        }
1175        let start = usize::try_from(offset).map_err(|_| {
1176            io::Error::new(
1177                io::ErrorKind::InvalidData,
1178                "Q42 manifest offset exceeds platform",
1179            )
1180        })?;
1181        let end = start.checked_add(length).ok_or_else(|| {
1182            io::Error::new(io::ErrorKind::InvalidData, "Q42 manifest range overflows")
1183        })?;
1184        self.mmap.get(start..end).map(Some).ok_or_else(|| {
1185            io::Error::new(
1186                io::ErrorKind::InvalidData,
1187                "Q42 manifest lies outside the file",
1188            )
1189        })
1190    }
1191
1192    /// Decode the front-embedded descriptor for a logical multi-segment Q42
1193    /// snapshot. A regular v3 Q42 returns `Ok(None)`.
1194    pub fn volume_manifest(&self) -> io::Result<Option<Q42VolumeManifest>> {
1195        self.volume_manifest_bytes()?
1196            .map(Q42VolumeManifest::decode)
1197            .transpose()
1198    }
1199
1200    /// Raw `PCAT` (`FLAG_PAYLOAD_CATALOGUE = 0x0080`) section bytes when present.
1201    pub fn payload_catalogue_bytes(&self) -> io::Result<Option<&[u8]>> {
1202        let Some((offset, length)) = self.header.payload_catalogue_range() else {
1203            return Ok(None);
1204        };
1205        let start = usize::try_from(offset).map_err(|_| {
1206            io::Error::new(
1207                io::ErrorKind::InvalidData,
1208                "Q42 payload catalogue offset exceeds platform",
1209            )
1210        })?;
1211        let length = usize::try_from(length).map_err(|_| {
1212            io::Error::new(
1213                io::ErrorKind::InvalidData,
1214                "Q42 payload catalogue length exceeds platform",
1215            )
1216        })?;
1217        let end = start.checked_add(length).ok_or_else(|| {
1218            io::Error::new(
1219                io::ErrorKind::InvalidData,
1220                "Q42 payload catalogue range overflows",
1221            )
1222        })?;
1223        self.mmap.get(start..end).map(Some).ok_or_else(|| {
1224            io::Error::new(
1225                io::ErrorKind::InvalidData,
1226                "Q42 payload catalogue lies outside the file",
1227            )
1228        })
1229    }
1230
1231    /// Read and CRC-32C / SHA-256 verify the `PCAT` v1 payload catalogue (`ADR 0014`),
1232    /// returning each [`PayloadCatalogueBlob`] (descriptor + UTF-8 body).
1233    /// Returns an empty `Vec` if `FLAG_PAYLOAD_CATALOGUE` is not set.
1234    pub fn read_payload_catalogue(&self) -> io::Result<Vec<PayloadCatalogueBlob>> {
1235        let Some(bytes) = self.payload_catalogue_bytes()? else {
1236            return Ok(Vec::new());
1237        };
1238        decode_payload_catalogue_with_bodies(bytes)
1239            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
1240    }
1241
1242    /// Read and CRC-32C verify the `PCAT` v1 payload catalogue metadata entries (`ADR 0014`).
1243    pub fn read_payload_catalogue_entries(&self) -> io::Result<Vec<PayloadCatalogueEntry>> {
1244        let Some(bytes) = self.payload_catalogue_bytes()? else {
1245            return Ok(Vec::new());
1246        };
1247        decode_payload_catalogue_section(bytes)
1248            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
1249    }
1250
1251    /// SuperBlock count from the header.
1252    pub fn block_count(&self) -> u64 {
1253        self.header.block_count
1254    }
1255
1256    /// Decode and verify every block without materialising the graph.  This
1257    /// checks SuperBlock counts, parity, per-block/global object ordering, and
1258    /// the BIDX range advertised for each block. Logical roots have no local
1259    /// records and must be verified through their manifest children.
1260    pub fn verify_all_blocks(&self) -> io::Result<Q42VerificationReceipt> {
1261        if self.volume_manifest()?.is_some() {
1262            return Err(io::Error::new(
1263                io::ErrorKind::InvalidInput,
1264                "verify a Q42 logical root through its manifest child segments",
1265            ));
1266        }
1267        let mut decoded = [0u8; SUPERBLOCK_SIZE];
1268        let mut quins_verified = 0u64;
1269        let mut previous_object = None;
1270        for block_index in 0..self.block_count() as usize {
1271            self.read_superblock_into(block_index, &mut decoded)?;
1272            let live = u64::from_le_bytes(decoded[16..24].try_into().unwrap()) as usize;
1273            if live == 0 || live > QUINS_PER_BLOCK {
1274                return Err(io::Error::new(
1275                    io::ErrorKind::InvalidData,
1276                    "Q42 SuperBlock has invalid live Quin count",
1277                ));
1278            }
1279            let bidx_offset = 16 + block_index * 16;
1280            let bidx = self.bidx_bytes();
1281            let advertised_min =
1282                u64::from_le_bytes(bidx[bidx_offset..bidx_offset + 8].try_into().unwrap());
1283            let advertised_max =
1284                u64::from_le_bytes(bidx[bidx_offset + 8..bidx_offset + 16].try_into().unwrap());
1285            let mut first = None;
1286            let mut last = 0u64;
1287            for quin_index in 0..live {
1288                let offset = SUPERBLOCK_HEADER + quin_index * QUIN_SIZE;
1289                let quin: crate::NQuin =
1290                    bytemuck::pod_read_unaligned(&decoded[offset..offset + QUIN_SIZE]);
1291                if !quin.verify_ecc_parity() {
1292                    return Err(io::Error::new(
1293                        io::ErrorKind::InvalidData,
1294                        format!("Q42 parity mismatch in block {block_index}, Quin {quin_index}"),
1295                    ));
1296                }
1297                if previous_object.is_some_and(|previous| quin.object < previous) {
1298                    return Err(io::Error::new(
1299                        io::ErrorKind::InvalidData,
1300                        format!("Q42 object order regresses in block {block_index}"),
1301                    ));
1302                }
1303                first.get_or_insert(quin.object);
1304                last = quin.object;
1305                previous_object = Some(quin.object);
1306                quins_verified += 1;
1307            }
1308            if first != Some(advertised_min) || last != advertised_max {
1309                return Err(io::Error::new(
1310                    io::ErrorKind::InvalidData,
1311                    format!("Q42 BIDX range disagrees with decoded block {block_index}"),
1312                ));
1313            }
1314        }
1315        Ok(Q42VerificationReceipt {
1316            blocks_verified: self.block_count(),
1317            quins_verified,
1318        })
1319    }
1320
1321    /// Binary-search BIDX for `object_hash`; returns block indices that may contain it.
1322    ///
1323    /// This allocating compatibility helper now returns the complete matching
1324    /// interval. New query paths should use [`Self::bidx_blocks_for_hash_into`]
1325    /// to enumerate a high-frequency object through a caller buffer.
1326    pub fn bidx_blocks_for_hash(&self, object_hash: u64) -> Vec<usize> {
1327        bidx_blocks_for_hash(self.bidx_bytes(), object_hash)
1328    }
1329
1330    /// Return the full contiguous BIDX interval that may contain `object_hash`.
1331    pub fn bidx_block_range_for_hash(
1332        &self,
1333        object_hash: u64,
1334    ) -> io::Result<Option<BidxBlockRange>> {
1335        volume::bidx_block_range_for_hash(self.bidx_bytes(), object_hash)
1336    }
1337
1338    /// Fill `out` with one page of matching BIDX block indices.
1339    ///
1340    /// `cursor` is an offset relative to the returned interval's start. Resume
1341    /// with `next_cursor` until it is `None`; an empty output buffer never
1342    /// causes a silent partial success.
1343    pub fn bidx_blocks_for_hash_into(
1344        &self,
1345        object_hash: u64,
1346        cursor: usize,
1347        out: &mut [usize],
1348    ) -> io::Result<Option<BidxMatchPage>> {
1349        volume::bidx_blocks_for_hash_into(self.bidx_bytes(), object_hash, cursor, out)
1350    }
1351
1352    /// Minimum and maximum object hash advertised by this segment's validated
1353    /// BIDX. Empty root/catalog segments return `None`.
1354    pub fn object_hash_bounds(&self) -> Option<(u64, u64)> {
1355        let count = self.block_count() as usize;
1356        if count == 0 {
1357            return None;
1358        }
1359        let bidx = self.bidx_bytes();
1360        let first = u64::from_le_bytes(bidx[16..24].try_into().ok()?);
1361        let last_offset = 16 + (count - 1) * 16;
1362        let last = u64::from_le_bytes(bidx[last_offset + 8..last_offset + 16].try_into().ok()?);
1363        Some((first, last))
1364    }
1365
1366    /// Read directory entry `index` from the block directory;
1367    /// out-of-range indices error.
1368    pub fn block_directory_entry(&self, index: usize) -> io::Result<BlockDirectoryEntry> {
1369        if index >= self.header.block_count as usize {
1370            return Err(io::Error::new(
1371                io::ErrorKind::InvalidInput,
1372                "block index out of range",
1373            ));
1374        }
1375        let start = self.header.block_dir_offset as usize + index * BlockDirectoryEntry::SIZE;
1376        let end = start + BlockDirectoryEntry::SIZE;
1377        let mut buf = [0u8; 16];
1378        buf.copy_from_slice(&self.mmap[start..end]);
1379        Ok(BlockDirectoryEntry::from_bytes(&buf))
1380    }
1381
1382    /// Decompress SuperBlock `index` into `out` (must be >= 40960 bytes).
1383    pub fn read_superblock_into(&self, index: usize, out: &mut [u8]) -> io::Result<usize> {
1384        if out.len() < SUPERBLOCK_SIZE {
1385            return Err(io::Error::new(
1386                io::ErrorKind::InvalidInput,
1387                "output buffer too small",
1388            ));
1389        }
1390        let entry = self.block_directory_entry(index)?;
1391        let data_offset = usize::try_from(self.header.data_offset).map_err(|_| {
1392            io::Error::new(
1393                io::ErrorKind::InvalidData,
1394                "Q42 data offset does not fit this platform",
1395            )
1396        })?;
1397        let rel_offset = usize::try_from(entry.rel_offset).map_err(|_| {
1398            io::Error::new(
1399                io::ErrorKind::InvalidData,
1400                "Q42 block offset does not fit this platform",
1401            )
1402        })?;
1403        let start = data_offset.checked_add(rel_offset).ok_or_else(|| {
1404            io::Error::new(
1405                io::ErrorKind::InvalidData,
1406                "Q42 block offset overflows usize",
1407            )
1408        })?;
1409        let end = start.checked_add(entry.comp_len as usize).ok_or_else(|| {
1410            io::Error::new(
1411                io::ErrorKind::InvalidData,
1412                "Q42 block length overflows usize",
1413            )
1414        })?;
1415        let compressed = &self.mmap[start..end];
1416        if compressed.len() < 4 {
1417            return Err(io::Error::new(
1418                io::ErrorKind::InvalidData,
1419                "Q42 LZ4 block is shorter than its size prefix",
1420            ));
1421        }
1422        let declared = u32::from_le_bytes(compressed[0..4].try_into().unwrap()) as usize;
1423        if declared != entry.uncomp_len as usize || declared != SUPERBLOCK_SIZE {
1424            return Err(io::Error::new(
1425                io::ErrorKind::InvalidData,
1426                "Q42 LZ4 size prefix disagrees with the block directory",
1427            ));
1428        }
1429        let decoded =
1430            lz4_flex::decompress_into(&compressed[4..], &mut out[..declared]).map_err(|e| {
1431                io::Error::new(
1432                    io::ErrorKind::InvalidData,
1433                    format!("LZ4 decompress block {index}: {e}"),
1434                )
1435            })?;
1436        if decoded != declared {
1437            return Err(io::Error::new(
1438                io::ErrorKind::InvalidData,
1439                "Q42 LZ4 block decoded to an unexpected length",
1440            ));
1441        }
1442        Ok(decoded)
1443    }
1444
1445    /// Read every live Quin from this volume's SuperBlocks into a heap `Vec`.
1446    ///
1447    /// Cold path (CLI / daemon vault load). A `.q42` is **not** a flat array of
1448    /// 48-byte records — each SuperBlock is LZ4-compressed and carries a 160-byte
1449    /// header whose bytes `[16..24]` hold the block's live quin count. Callers that
1450    /// `bytemuck::cast_slice` the raw file will panic (`OutputSliceWouldHaveSlop`)
1451    /// because the file length is not a multiple of `QUIN_SIZE`. Use this instead.
1452    pub fn read_all_quins(&self) -> io::Result<Vec<crate::NQuin>> {
1453        if self.volume_manifest()?.is_some() {
1454            return Err(io::Error::new(
1455                io::ErrorKind::InvalidInput,
1456                "Q42 logical root has no local graph blocks; open Q42VolumeSet or use read_q42_quins",
1457            ));
1458        }
1459        let rd = |b: &[u8], o: usize| u64::from_le_bytes(b[o..o + 8].try_into().unwrap());
1460        let mut out = Vec::new();
1461        let mut sb = vec![0u8; SUPERBLOCK_SIZE];
1462        for i in 0..self.block_count() as usize {
1463            self.read_superblock_into(i, &mut sb)?;
1464            let quin_count = rd(&sb, 16) as usize;
1465            for k in 0..quin_count {
1466                let o = SUPERBLOCK_HEADER + k * QUIN_SIZE;
1467                if o + QUIN_SIZE > sb.len() {
1468                    break;
1469                }
1470                out.push(crate::NQuin {
1471                    subject: rd(&sb, o),
1472                    predicate: rd(&sb, o + 8),
1473                    object: rd(&sb, o + 16),
1474                    context: rd(&sb, o + 24),
1475                    metadata: rd(&sb, o + 32),
1476                    parity: rd(&sb, o + 40),
1477                });
1478            }
1479        }
1480        Ok(out)
1481    }
1482}
1483
1484/// BIDX binary search (shared with sidecar format).
1485pub fn bidx_blocks_for_hash(bidx: &[u8], object_hash: u64) -> Vec<usize> {
1486    match volume::bidx_block_range_for_hash(bidx, object_hash) {
1487        Ok(Some(range)) => (range.start..range.end).collect(),
1488        Ok(None) | Err(_) => Vec::new(),
1489    }
1490}
1491
1492#[cfg(all(test, not(target_arch = "wasm32")))]
1493mod tests {
1494    use super::*;
1495    use crate::mini_parser::hash_token;
1496    use tempfile::{NamedTempFile, TempDir};
1497
1498    fn sample_quin(subj: &str, pred: &str, obj: &str) -> (NQuin, HashMap<u64, String>) {
1499        let mut lex = HashMap::new();
1500        let sh = hash_token(subj);
1501        let ph = hash_token(pred);
1502        let oh = hash_token(obj);
1503        lex.insert(sh, subj.to_string());
1504        lex.insert(ph, pred.to_string());
1505        lex.insert(oh, obj.to_string());
1506        let q = NQuin {
1507            subject: sh,
1508            predicate: ph,
1509            object: oh,
1510            context: 0,
1511            metadata: 0,
1512            parity: 0,
1513        };
1514        (q, lex)
1515    }
1516
1517    #[test]
1518    fn unified_volume_roundtrip() {
1519        let (q1, mut lex) = sample_quin("Patient", "fever", "True");
1520        let (q2, lex2) = sample_quin("Patient", "has", "pain");
1521        lex.extend(lex2);
1522
1523        let mut blocks = vec![vec![q1], vec![q2]];
1524        blocks.sort_by_key(|chunk| chunk[0].object);
1525
1526        let tmp = NamedTempFile::new().unwrap();
1527        let ranges: Vec<_> = blocks
1528            .iter()
1529            .map(|chunk| {
1530                let h = chunk[0].object;
1531                (h, h)
1532            })
1533            .collect();
1534        write_unified_volume(tmp.path(), &lex, &ranges, &blocks).unwrap();
1535
1536        let vol = Q42Volume::open(tmp.path()).unwrap();
1537        assert_eq!(vol.block_count(), 2);
1538        assert!(vol.lex_view().unwrap().contains(q1.object));
1539
1540        let hits = vol.bidx_blocks_for_hash(q1.object);
1541        assert!(!hits.is_empty(), "bidx miss for object hash {}", q1.object);
1542
1543        let mut block = [0u8; SUPERBLOCK_SIZE];
1544        vol.read_superblock_into(hits[0], &mut block).unwrap();
1545        let active = u64::from_le_bytes(block[16..24].try_into().unwrap());
1546        assert_eq!(active, 1);
1547    }
1548
1549    #[test]
1550    fn exhaustive_verifier_checks_decoded_quins_against_bidx() {
1551        let tmp = NamedTempFile::new().unwrap();
1552        let mut quin = NQuin {
1553            subject: 11,
1554            predicate: 22,
1555            object: 33,
1556            context: 44,
1557            metadata: 55,
1558            parity: 0,
1559        };
1560        quin.parity = NQuin::calculate_parity(
1561            quin.subject,
1562            quin.predicate,
1563            quin.object,
1564            quin.context,
1565            quin.metadata,
1566        );
1567        write_unified_volume(
1568            tmp.path(),
1569            &HashMap::new(),
1570            &[(quin.object, quin.object)],
1571            &[vec![quin]],
1572        )
1573        .unwrap();
1574        let volume = Q42Volume::open(tmp.path()).unwrap();
1575        assert_eq!(
1576            volume.verify_all_blocks().unwrap(),
1577            Q42VerificationReceipt {
1578                blocks_verified: 1,
1579                quins_verified: 1,
1580            }
1581        );
1582    }
1583
1584    #[test]
1585    fn logical_root_verifier_checks_all_manifest_children() {
1586        let dir = tempfile::TempDir::new().unwrap();
1587        let child = dir.path().join("child.q42");
1588        let root = dir.path().join("root.q42");
1589        let mut quin = NQuin {
1590            subject: 101,
1591            predicate: 202,
1592            object: 303,
1593            context: 404,
1594            metadata: 0,
1595            parity: 0,
1596        };
1597        quin.recalculate_parity();
1598        write_unified_volume(
1599            &child,
1600            &HashMap::new(),
1601            &[(quin.object, quin.object)],
1602            &[vec![quin]],
1603        )
1604        .unwrap();
1605        let manifest = Q42VolumeManifest {
1606            generation: 1,
1607            segments: vec![
1608                Q42VolumeManifest::segment_from_file(&child, "child.q42".to_string()).unwrap(),
1609            ],
1610            lexicon_segments: Vec::new(),
1611        };
1612        write_volume_root(&root, &manifest).unwrap();
1613        let set = Q42VolumeSet::open_root(&root).unwrap();
1614        assert_eq!(
1615            set.verify_all(&root).unwrap(),
1616            Q42VerificationReceipt {
1617                blocks_verified: 1,
1618                quins_verified: 1,
1619            }
1620        );
1621    }
1622
1623    #[test]
1624    fn bidx_heavy_hitter_is_complete_and_caller_buffered() {
1625        let (q, lex) = sample_quin("shared", "has", "high-frequency-object");
1626        let blocks = vec![vec![q]; 5];
1627        let ranges = vec![(q.object, q.object); 5];
1628        let tmp = NamedTempFile::new().unwrap();
1629        write_unified_volume(tmp.path(), &lex, &ranges, &blocks).unwrap();
1630
1631        let vol = Q42Volume::open(tmp.path()).unwrap();
1632        assert_eq!(vol.bidx_blocks_for_hash(q.object), vec![0, 1, 2, 3, 4]);
1633
1634        let mut page = [usize::MAX; 2];
1635        let first = vol
1636            .bidx_blocks_for_hash_into(q.object, 0, &mut page)
1637            .unwrap()
1638            .unwrap();
1639        assert_eq!(first.range, BidxBlockRange { start: 0, end: 5 });
1640        assert_eq!(first.returned, 2);
1641        assert_eq!(&page, &[0, 1]);
1642        assert_eq!(first.next_cursor, Some(2));
1643
1644        let second = vol
1645            .bidx_blocks_for_hash_into(q.object, first.next_cursor.unwrap(), &mut page)
1646            .unwrap()
1647            .unwrap();
1648        assert_eq!(&page, &[2, 3]);
1649        assert_eq!(second.next_cursor, Some(4));
1650
1651        let third = vol
1652            .bidx_blocks_for_hash_into(q.object, second.next_cursor.unwrap(), &mut page)
1653            .unwrap()
1654            .unwrap();
1655        assert_eq!(third.returned, 1);
1656        assert_eq!(page[0], 4);
1657        assert_eq!(third.next_cursor, None);
1658        assert!(vol.bidx_blocks_for_hash_into(q.object, 0, &mut []).is_err());
1659    }
1660
1661    #[test]
1662    fn embedded_root_manifest_opens_immutable_child_segments() {
1663        let dir = TempDir::new().unwrap();
1664        let root = dir.path().join("root.q42");
1665        let first_path = dir.path().join("segment-000.q42");
1666        let second_path = dir.path().join("segment-001.q42");
1667        let (first_q, first_lex) = sample_quin("s-1", "p", "o-1");
1668        let (second_q, second_lex) = sample_quin("s-2", "p", "o-2");
1669        let mut inputs = vec![(first_q, first_lex), (second_q, second_lex)];
1670        inputs.sort_unstable_by_key(|(quin, _)| quin.object);
1671        write_unified_volume(
1672            &first_path,
1673            &inputs[0].1,
1674            &[(inputs[0].0.object, inputs[0].0.object)],
1675            &[vec![inputs[0].0]],
1676        )
1677        .unwrap();
1678        write_unified_volume(
1679            &second_path,
1680            &inputs[1].1,
1681            &[(inputs[1].0.object, inputs[1].0.object)],
1682            &[vec![inputs[1].0]],
1683        )
1684        .unwrap();
1685        let manifest = Q42VolumeManifest {
1686            generation: 7,
1687            segments: vec![
1688                Q42VolumeManifest::segment_from_file(&first_path, "segment-000.q42".into())
1689                    .unwrap(),
1690                Q42VolumeManifest::segment_from_file(&second_path, "segment-001.q42".into())
1691                    .unwrap(),
1692            ],
1693            lexicon_segments: vec![],
1694        };
1695        write_volume_root(&root, &manifest).unwrap();
1696
1697        let root_volume = Q42Volume::open(&root).unwrap();
1698        assert_eq!(root_volume.block_count(), 0);
1699        assert_eq!(
1700            root_volume.volume_manifest().unwrap(),
1701            Some(manifest.clone())
1702        );
1703        let set = Q42VolumeSet::open_root(&root).unwrap();
1704        assert_eq!(set.manifest().generation, 7);
1705        assert_eq!(set.segments().len(), 2);
1706        set.verify_segment_hashes(&root).unwrap();
1707    }
1708
1709    #[test]
1710    fn open_rejects_mismatched_directory_length() {
1711        let (q, lex) = sample_quin("s", "p", "o");
1712        let tmp = NamedTempFile::new().unwrap();
1713        write_unified_volume(tmp.path(), &lex, &[(q.object, q.object)], &[vec![q]]).unwrap();
1714        let mut bytes = std::fs::read(tmp.path()).unwrap();
1715        bytes[48..56].copy_from_slice(&0u64.to_le_bytes());
1716        std::fs::write(tmp.path(), bytes).unwrap();
1717        assert!(Q42Volume::open(tmp.path()).is_err());
1718    }
1719
1720    #[test]
1721    fn decompression_rejects_directory_size_disagreement() {
1722        let (q, lex) = sample_quin("s", "p", "o");
1723        let tmp = NamedTempFile::new().unwrap();
1724        write_unified_volume(tmp.path(), &lex, &[(q.object, q.object)], &[vec![q]]).unwrap();
1725        let mut bytes = std::fs::read(tmp.path()).unwrap();
1726        let data_offset = u64::from_le_bytes(bytes[56..64].try_into().unwrap()) as usize;
1727        bytes[data_offset..data_offset + 4].copy_from_slice(&1u32.to_le_bytes());
1728        std::fs::write(tmp.path(), bytes).unwrap();
1729        let vol = Q42Volume::open(tmp.path()).unwrap();
1730        let mut out = [0u8; SUPERBLOCK_SIZE];
1731        assert!(vol.read_superblock_into(0, &mut out).is_err());
1732    }
1733
1734    /// Regression: `read_all_quins` must reconstruct every stored Quin across
1735    /// SuperBlocks. A `.q42` is not a flat 48-byte array — the CLI query path used
1736    /// to `bytemuck::cast_slice` the raw file and panic (`OutputSliceWouldHaveSlop`)
1737    /// because the file length is not a multiple of `QUIN_SIZE`.
1738    #[test]
1739    fn read_all_quins_roundtrip() {
1740        let (q1, mut lex) = sample_quin("Alice", "rdf:type", "values:NaturalPerson");
1741        let (q2, lex2) = sample_quin("Bob", "values:claims", "values:Right");
1742        let (q3, lex3) = sample_quin("AcmeCorp", "rdf:type", "values:CorporatePerson");
1743        lex.extend(lex2);
1744        lex.extend(lex3);
1745
1746        let mut blocks = vec![vec![q1], vec![q2], vec![q3]];
1747        blocks.sort_by_key(|chunk| chunk[0].object);
1748        let ranges: Vec<_> = blocks.iter().map(|c| (c[0].object, c[0].object)).collect();
1749
1750        let tmp = NamedTempFile::new().unwrap();
1751        write_unified_volume(tmp.path(), &lex, &ranges, &blocks).unwrap();
1752
1753        let vol = Q42Volume::open(tmp.path()).unwrap();
1754        let quins = vol.read_all_quins().unwrap();
1755        assert_eq!(
1756            quins.len(),
1757            3,
1758            "read_all_quins must return every stored quin"
1759        );
1760        for q in [q1, q2, q3] {
1761            assert!(
1762                quins.iter().any(|r| r.subject == q.subject
1763                    && r.predicate == q.predicate
1764                    && r.object == q.object),
1765                "read_all_quins dropped a quin: {q:?}"
1766            );
1767        }
1768    }
1769
1770    /// The streaming ingest path (`turtle_doc` → `ExternalSorter` → `merge`) must embed a
1771    /// populated Q42LEX section at the FRONT of the `.q42` — no separate `.lex` sidecar —
1772    /// so literal text and IRIs are recoverable from the volume alone.
1773    #[test]
1774    fn streaming_ingest_embeds_recoverable_lex() {
1775        use crate::external_sort::ExternalSorter;
1776        use crate::lexicon::generate_60bit_token;
1777        use crate::sparql_library::parsers::turtle_doc::parse_turtle_doc_into;
1778
1779        let doc = r#"
1780@prefix dc:     <http://purl.org/dc/terms/> .
1781@prefix values: <https://ns.webcivics.net/values/> .
1782@prefix doc:    <https://ns.webcivics.net/values/inst#> .
1783doc:article-1 a values:Undertaking ;
1784    dc:title "Article 1" ;
1785    values:originalText "Each Member undertakes to suppress forced labour." .
1786"#;
1787        let tmp_dir = std::env::temp_dir().join(format!("q42_lex_rt_{}", std::process::id()));
1788        let mut sorter = ExternalSorter::new(tmp_dir);
1789        parse_turtle_doc_into(doc.as_bytes(), 0, &mut sorter).unwrap();
1790        let out = NamedTempFile::new().unwrap();
1791        sorter.merge(out.path()).unwrap();
1792
1793        let vol = Q42Volume::open(out.path()).unwrap();
1794        let lex = vol
1795            .lex_view()
1796            .expect("volume must carry an embedded front-of-file Q42LEX section");
1797
1798        // The verbatim literal is recoverable from the .q42 alone — the CML prerequisite.
1799        let lit = "Each Member undertakes to suppress forced labour.";
1800        assert_eq!(
1801            lex.lookup_owned(generate_60bit_token(lit.as_bytes())),
1802            Some(lit.to_string())
1803        );
1804        // Expanded IRIs are recoverable too (queries become human-readable).
1805        // The undertaking IRI is stored namespaced (v4 lexicon): prove both the
1806        // full reassembly and the (namespace, local) split.
1807        let undertaking = "https://ns.webcivics.net/values/Undertaking";
1808        let token = generate_60bit_token(undertaking.as_bytes());
1809        assert_eq!(lex.lookup_owned(token), Some(undertaking.to_string()));
1810        let (ns, local) = lex.lookup_parts(token).expect("namespaced IRI parts");
1811        assert_eq!(ns, "https://ns.webcivics.net/values/");
1812        assert_eq!(local, "Undertaking");
1813    }
1814
1815    /// `finish_to_bytes` must yield the same recoverable volume as `finish` writes to disk —
1816    /// the in-memory path used to embed a `.q42` inside a `.hmc` bundle.
1817    #[test]
1818    fn finish_to_bytes_matches_a_readable_on_disk_volume() {
1819        let (q1, mut lex) = sample_quin("Heart", "geo:bodySystem", "circulatory");
1820        let (q2, lex2) = sample_quin("Lung", "geo:bodySystem", "respiratory");
1821        lex.extend(lex2);
1822        let mut blocks = vec![vec![q1], vec![q2]];
1823        blocks.sort_by_key(|c| c[0].object);
1824
1825        let mut builder = UnifiedVolumeBuilder::with_lex_map(&lex).unwrap();
1826        for (seq, chunk) in blocks.iter().enumerate() {
1827            builder.push_block(seq as u64, chunk).unwrap();
1828        }
1829        let bytes = builder.finish_to_bytes();
1830
1831        assert!(bytes.starts_with(&Q42_MAGIC), "bytes are a Q42 volume");
1832        // The bytes open as a valid unified volume with every fact recoverable.
1833        let tmp = NamedTempFile::new().unwrap();
1834        std::fs::write(tmp.path(), &bytes).unwrap();
1835        let vol = Q42Volume::open(tmp.path()).unwrap();
1836        assert_eq!(vol.block_count(), 2);
1837        let quins = vol.read_all_quins().unwrap();
1838        assert_eq!(quins.len(), 2);
1839        let lexv = vol.lex_view().unwrap();
1840        let vals: Vec<String> = quins
1841            .iter()
1842            .filter_map(|q| lexv.lookup_owned(q.object))
1843            .collect();
1844        assert!(vals.iter().any(|v| v == "circulatory") && vals.iter().any(|v| v == "respiratory"));
1845    }
1846
1847    #[test]
1848    fn v2_magic_detected() {
1849        let (q, lex) = sample_quin("a", "b", "c");
1850        let tmp = NamedTempFile::new().unwrap();
1851        write_unified_volume(tmp.path(), &lex, &[(q.object, q.object)], &[vec![q]]).unwrap();
1852        assert!(is_unified_volume(tmp.path()).unwrap());
1853    }
1854}
1855
1856/// Streaming append-only interface for Q42 Unified Volumes.
1857/// Allows continuous block accumulation without loading the entire volume in memory.
1858#[cfg(not(target_arch = "wasm32"))]
1859pub struct StreamingVolumeAppender {
1860    file: std::fs::File,
1861    header: Q42VolumeHeader,
1862    block_ranges: Vec<(u64, u64)>,
1863    dir_entries: Vec<BlockDirectoryEntry>,
1864    dag_store: crate::git_bridge::DagStore,
1865    author_did: u64,
1866    last_dag_hash: [u8; 32],
1867    last_object_hash: Option<u64>,
1868}
1869
1870#[cfg(not(target_arch = "wasm32"))]
1871impl StreamingVolumeAppender {
1872    /// Create a new streaming volume at `path` — writes the v4 header
1873    /// eagerly; rejects non-empty files (immutable segments only).
1874    pub fn new(path: &std::path::Path) -> std::io::Result<Self> {
1875        let mut file = std::fs::OpenOptions::new()
1876            .create(true)
1877            .read(true)
1878            .write(true)
1879            .open(path)?;
1880
1881        let header = Q42VolumeHeader {
1882            magic: Q42_MAGIC,
1883            version: Q42_VERSION_V4,
1884            flags: FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED,
1885            lex_offset: HEADER_SIZE as u64,
1886            lex_length: 0,
1887            bidx_offset: HEADER_SIZE as u64,
1888            bidx_length: 0,
1889            block_dir_offset: HEADER_SIZE as u64,
1890            block_dir_length: 0,
1891            data_offset: HEADER_SIZE as u64,
1892            data_length: 0,
1893            block_count: 0,
1894            block_size: SUPERBLOCK_SIZE as u32,
1895            quins_per_block: QUINS_PER_BLOCK as u32,
1896            temporal_index_offset: 0,
1897            temporal_index_length: 0,
1898            merkle_root: [0; 32],
1899            assertion_timestamp: 0,
1900            dag_root_offset: 0,
1901            dag_root_length: 0,
1902            natural_person_did_offset: 0,
1903            software_agent_did_offset: 0,
1904            _reserved: [0; 80],
1905        };
1906
1907        if file.metadata()?.len() != 0 {
1908            return Err(io::Error::new(
1909                io::ErrorKind::Unsupported,
1910                "StreamingVolumeAppender cannot reopen a Q42 volume; publish a new immutable segment",
1911            ));
1912        }
1913        {
1914            use std::io::{Seek, SeekFrom, Write};
1915            file.seek(SeekFrom::Start(0))?;
1916            file.write_all(&header_to_bytes(&header))?;
1917        }
1918
1919        Ok(Self {
1920            file,
1921            header,
1922            block_ranges: Vec::new(),
1923            dir_entries: Vec::new(),
1924            dag_store: crate::git_bridge::DagStore::new(),
1925            author_did: 0,
1926            last_dag_hash: [0u8; 32],
1927            last_object_hash: None,
1928        })
1929    }
1930
1931    /// Set the author DID recorded on DAG commit nodes.
1932    pub fn with_author_did(mut self, did: u64) -> Self {
1933        self.author_did = did;
1934        self
1935    }
1936
1937    /// Append one SuperBlock incrementally — same global object-sort
1938    /// enforcement, LZ4 compression, and DAG commit as the batch
1939    /// builder, but streamed straight to the file.
1940    pub fn append_block(&mut self, seq_id: u64, quins: &[NQuin]) -> std::io::Result<()> {
1941        let (min_hash, max_hash) = object_range(quins)?;
1942        if let Some(previous) = self.last_object_hash {
1943            if min_hash < previous {
1944                return Err(io::Error::new(
1945                    io::ErrorKind::InvalidInput,
1946                    "Q42 streaming blocks must be globally sorted by object hash",
1947                ));
1948            }
1949        }
1950        self.block_ranges.push((min_hash, max_hash));
1951
1952        let raw = encode_superblock(seq_id, quins);
1953        let compressed = lz4_flex::compress_prepend_size(&raw);
1954
1955        use std::io::{Seek, SeekFrom, Write};
1956        let append_offset = self.header.data_offset + self.header.data_length;
1957        self.file.seek(SeekFrom::Start(append_offset))?;
1958
1959        self.dir_entries.push(BlockDirectoryEntry {
1960            rel_offset: self.header.data_length,
1961            comp_len: compressed.len() as u32,
1962            uncomp_len: SUPERBLOCK_SIZE as u32,
1963        });
1964
1965        self.file.write_all(&compressed)?;
1966        self.header.data_length += compressed.len() as u64;
1967        self.header.block_count += 1;
1968
1969        let ts = std::time::SystemTime::now()
1970            .duration_since(std::time::UNIX_EPOCH)
1971            .map(|d| d.as_millis() as u64)
1972            .unwrap_or(0);
1973        let msg = format!("runtime block {}", seq_id);
1974
1975        self.last_dag_hash = if self.last_dag_hash == [0u8; 32] {
1976            self.dag_store
1977                .genesis_node(quins, self.author_did, ts, &msg)
1978        } else {
1979            self.dag_store
1980                .commit_node(self.last_dag_hash, quins, self.author_did, ts, &msg)
1981        };
1982        self.last_object_hash = Some(max_hash);
1983
1984        // Write BIDX and Directory at the end
1985        let bidx_bytes = encode_bidx(&self.block_ranges);
1986        self.header.bidx_offset = self.header.data_offset + self.header.data_length;
1987        self.header.bidx_length = bidx_bytes.len() as u64;
1988        self.file.write_all(&bidx_bytes)?;
1989
1990        self.header.block_dir_offset = self.header.bidx_offset + self.header.bidx_length;
1991        self.header.block_dir_length = (self.dir_entries.len() * BlockDirectoryEntry::SIZE) as u64;
1992        let mut dir_bytes = Vec::with_capacity(self.header.block_dir_length as usize);
1993        for entry in &self.dir_entries {
1994            dir_bytes.extend_from_slice(&entry.rel_offset.to_le_bytes());
1995            dir_bytes.extend_from_slice(&entry.comp_len.to_le_bytes());
1996            dir_bytes.extend_from_slice(&entry.uncomp_len.to_le_bytes());
1997        }
1998        self.file.write_all(&dir_bytes)?;
1999
2000        // Write DAG
2001        let dag_bytes = self.dag_store.serialize();
2002        self.header.dag_root_offset = self.header.block_dir_offset + self.header.block_dir_length;
2003        self.header.dag_root_length = dag_bytes.len() as u64;
2004        self.file.write_all(&dag_bytes)?;
2005
2006        use sha2::{Digest, Sha256};
2007        let mut h = Sha256::new();
2008        h.update(self.last_dag_hash);
2009        self.header.merkle_root = h.finalize().into();
2010        self.header.assertion_timestamp = ts;
2011
2012        // Update header
2013        self.file.seek(SeekFrom::Start(0))?;
2014        self.file.write_all(&header_to_bytes(&self.header))?;
2015        self.file.sync_all()?;
2016
2017        Ok(())
2018    }
2019}