1use std::collections::HashMap;
16#[cfg(not(target_arch = "wasm32"))]
17use std::fs::{File, OpenOptions};
18use std::io::{self, Write};
19#[cfg(not(target_arch = "wasm32"))]
20use std::io::{BufWriter, Read};
21#[cfg(not(target_arch = "wasm32"))]
22use std::path::Path;
23
24#[cfg(not(target_arch = "wasm32"))]
25use memmap2::{Mmap, MmapOptions};
26
27#[cfg(not(target_arch = "wasm32"))]
28use crate::q42_lex::Q42LexMmap;
29use crate::q42_lex::{LexError, LexiconEntry, LEX_MAGIC};
30use crate::{NQuin, QUINS_PER_BLOCK};
31
32#[path = "volume/mod.rs"]
33mod volume;
34
35pub use volume::{
39 decode_and_verify_car, deny_public_publication, encode_block_postings, encode_postings_section,
40 encode_raw_car, extract_entity_bytes, inclusive_entity_bytes, measure_bloom_false_positives,
41 provenance_path_for, quin_requires_sanctuary, validate_exact_range_response,
42 verify_car_bytes_as_q42_source, verify_source_sha256, BidxBlockRange, BidxMatchPage,
43 BlockFieldPostings, CidSha256, ClassificationCounts, IngestAccessPolicy,
44 IngestProvenanceRecord, OpfsCallbackRangeSource, OpfsSliceRangeSource, PublicationIntent,
45 Q42ByteRange, Q42LexiconRangeFactory, Q42LexiconSegment, Q42ObjectMatchPage,
46 Q42ObjectSearchCursor, Q42PublicationClass, Q42PublicationVerdict, Q42QueryMode,
47 Q42RangeQueryCursor, Q42RangeQueryPage, Q42RangeQueryPattern, Q42RangeQueryPlan,
48 Q42RangeQueryStrategy, Q42RangeSource, Q42RangeVolume, Q42RangeVolumeSet, Q42SegmentMatchPage,
49 Q42SegmentMatchRange, Q42SegmentRangeFactory, Q42Transport, Q42VolumeManifest,
50 Q42VolumeSegment, Q42VolumeSetQueryCursor, Q42VolumeSetQueryPage, VerifiedCarBlock,
51 VerifiedCarRangeSource, FIELD_POSTINGS_MAGIC, MAX_VOLUME_MANIFEST_BYTES,
52 RESIDENT_QUERY_MAX_BYTES, WasmSliceQ42Reader,
53};
54#[cfg(not(target_arch = "wasm32"))]
56pub use volume::{
57 admit_logical_size, append_segment_to_root, classify_q42_path, classify_q42_volume,
58 classify_q42_volume_set, compact_volume_set, compose_magnet, root_relative_path, sha1_hex_file,
59 verify_local_car_as_q42_source, verify_volume_set_from_root, CheckStatus, Completeness,
60 LocalFileRangeSource, NetworkCursor, Q42BlockCursor, Q42BlockMeta, Q42InspectReport, Q42Magnet,
61 Q42RolloverPublisher, Q42SectionReport, Q42VerifyReceipt, Q42VerifySetReport, Q42VolumeSet,
62 Q42VolumeSetMagnets, ScanBudget, StreamingQ42VolumeWriter, VerifyCheck, VerifyLevel,
63 DEFAULT_SEGMENT_MAX_BYTES,
64};
65#[cfg(not(target_arch = "wasm32"))]
66pub use volume::{
67 ipfs_gateway_range_source, ipns_gateway_range_source, HttpRangeSource,
68 IpfsGatewaySegmentFactory,
69};
70#[cfg(not(target_arch = "wasm32"))]
71pub use volume::{write_sorted_quins_volume, write_sorted_quins_volume_with_author};
72pub use crate::bundle::cbor_ld_manifest::PayloadCatalogueEntry;
73#[path = "volume/payload_catalogue.rs"]
74pub mod payload_catalogue;
75pub use payload_catalogue::{
76 decode_payload_catalogue_section, decode_payload_catalogue_with_bodies,
77 encode_payload_catalogue_section, encode_payload_catalogue_with_bodies,
78 header_payload_catalogue_range, set_header_payload_catalogue_range,
79 validate_payload_catalogue_entry, PayloadCatalogueBlob,
80};
81
82pub mod publication {
84 #[cfg(not(target_arch = "wasm32"))]
85 pub use super::volume::{classify_q42_path, classify_q42_volume, classify_q42_volume_set};
86 pub use super::volume::{
87 check_public_distribution_allowed, deny_public_publication, provenance_path_for,
88 quin_requires_sanctuary, CheckPublicDistribution, ClassificationCounts,
89 IngestAccessPolicy, IngestProvenanceRecord, PublicationIntent, Q42PublicationClass,
90 Q42PublicationVerdict, Q42Transport,
91 };
92}
93pub use publication::check_public_distribution_allowed;
94
95pub const Q42_MAGIC: [u8; 4] = [0x51, 0x34, 0x32, 0x00]; pub const Q42_VERSION_V3: u16 = 3;
97pub const Q42_VERSION_V4: u16 = 4;
102pub const HEADER_SIZE: usize = 256;
103pub const SUPERBLOCK_SIZE: usize = 40_960;
104pub const MAX_COMPRESSED_SUPERBLOCK_SIZE: usize = SUPERBLOCK_SIZE + (SUPERBLOCK_SIZE / 255) + 36;
107pub const SUPERBLOCK_HEADER: usize = 160;
108pub const QUIN_SIZE: usize = 48;
109pub const BIDX_MAGIC: [u8; 4] = *b"BIDX";
110pub const FLAG_BLOCKS_LZ4: u16 = 0x0001;
111pub const FLAG_OBJECT_SORTED: u16 = 0x0002;
112pub const FLAG_VOLUME_ROOT: u16 = 0x0004;
113pub const FLAG_FIELD_RANGES: u16 = 0x0008;
116pub const FLAG_FIELD_POSTINGS: u16 = 0x0010;
118pub const FLAG_PERMISSIVE_COMMONS: u16 = 0x0020;
121pub const FLAG_SANCTUARY: u16 = 0x0040;
125pub const FLAG_PAYLOAD_CATALOGUE: u16 = 0x0080;
128pub const FIELD_RANGE_INDEX_MAGIC: [u8; 4] = *b"FIDX";
129pub const FIELD_RANGE_INDEX_HEADER_BYTES: usize = 16;
130pub const FIELD_RANGE_INDEX_ENTRY_BYTES: usize = 48;
131
132#[derive(Clone, Copy, Debug, Eq, PartialEq)]
134pub struct Q42VerificationReceipt {
135 pub blocks_verified: u64,
136 pub quins_verified: u64,
137}
138
139#[repr(C, packed)]
142#[derive(Clone, Copy, Debug)]
143pub struct Q42VolumeHeader {
144 pub magic: [u8; 4],
145 pub version: u16,
146 pub flags: u16,
147 pub lex_offset: u64,
148 pub lex_length: u64,
149 pub bidx_offset: u64,
150 pub bidx_length: u64,
151 pub block_dir_offset: u64,
152 pub block_dir_length: u64,
153 pub data_offset: u64,
154 pub data_length: u64,
155 pub block_count: u64,
156 pub block_size: u32,
157 pub quins_per_block: u32,
158 pub temporal_index_offset: u64,
160 pub temporal_index_length: u64,
161 pub merkle_root: [u8; 32], pub assertion_timestamp: u64, pub dag_root_offset: u64, pub dag_root_length: u64, pub natural_person_did_offset: u64, pub software_agent_did_offset: u64, pub _reserved: [u8; 80], }
172
173const _: () = assert!(
174 std::mem::size_of::<Q42VolumeHeader>() == 256,
175 "Q42VolumeHeader must be exactly 256 bytes — matches HEADER_SIZE constant"
176);
177
178impl Q42VolumeHeader {
179 pub fn verify_version(&self) -> Result<(), String> {
181 let magic = self.magic;
183 let version = { self.version };
184 if magic != Q42_MAGIC {
185 return Err(format!("bad magic {magic:?}"));
186 }
187 if version != Q42_VERSION_V3 && version != Q42_VERSION_V4 {
188 return Err(format!(
189 "Q42 file is version {version}; v3/v4 required (run q42 migrate meta for v2)"
190 ));
191 }
192 Ok(())
193 }
194
195 fn volume_manifest_range(&self) -> Option<(u64, u64)> {
198 if self.flags & FLAG_VOLUME_ROOT == 0 {
199 return None;
200 }
201 let reserved = self._reserved;
202 Some((
203 u64::from_le_bytes(reserved[0..8].try_into().unwrap()),
204 u64::from_le_bytes(reserved[8..16].try_into().unwrap()),
205 ))
206 }
207
208 fn field_range_index_range(&self) -> Option<(u64, u64)> {
211 if self.flags & FLAG_FIELD_RANGES == 0 {
212 return None;
213 }
214 let reserved = self._reserved;
215 Some((
216 u64::from_le_bytes(reserved[16..24].try_into().unwrap()),
217 u64::from_le_bytes(reserved[24..32].try_into().unwrap()),
218 ))
219 }
220
221 fn field_postings_range(&self) -> Option<(u64, u64)> {
224 if self.flags & FLAG_FIELD_POSTINGS == 0 {
225 return None;
226 }
227 let reserved = self._reserved;
228 Some((
229 u64::from_le_bytes(reserved[32..40].try_into().unwrap()),
230 u64::from_le_bytes(reserved[40..48].try_into().unwrap()),
231 ))
232 }
233
234 pub fn payload_catalogue_range(&self) -> Option<(u64, u64)> {
237 header_payload_catalogue_range(self)
238 }
239
240 pub fn new_v4(
242 lex_offset: u64,
243 lex_length: u64,
244 bidx_offset: u64,
245 bidx_length: u64,
246 block_dir_offset: u64,
247 block_dir_length: u64,
248 data_offset: u64,
249 data_length: u64,
250 block_count: u64,
251 block_size: u32,
252 quins_per_block: u32,
253 ) -> Self {
254 let assertion_timestamp = std::time::SystemTime::now()
255 .duration_since(std::time::UNIX_EPOCH)
256 .map(|d| d.as_millis() as u64)
257 .unwrap_or(0);
258 Self {
259 magic: Q42_MAGIC,
260 version: Q42_VERSION_V4,
261 flags: FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED,
262 lex_offset,
263 lex_length,
264 bidx_offset,
265 bidx_length,
266 block_dir_offset,
267 block_dir_length,
268 data_offset,
269 data_length,
270 block_count,
271 block_size,
272 quins_per_block,
273 temporal_index_offset: 0,
274 temporal_index_length: 0,
275 merkle_root: [0u8; 32],
276 assertion_timestamp,
277 dag_root_offset: 0,
278 dag_root_length: 0,
279 natural_person_did_offset: 0,
280 software_agent_did_offset: 0,
281 _reserved: [0u8; 80],
282 }
283 }
284}
285
286#[repr(C, packed)]
287#[derive(Clone, Copy, Debug)]
288pub struct BlockDirectoryEntry {
289 pub rel_offset: u64,
290 pub comp_len: u32,
291 pub uncomp_len: u32,
292}
293
294impl BlockDirectoryEntry {
295 pub const SIZE: usize = 16;
296
297 pub fn write_to(&self, out: &mut impl Write) -> io::Result<()> {
300 out.write_all(&self.rel_offset.to_le_bytes())?;
301 out.write_all(&self.comp_len.to_le_bytes())?;
302 out.write_all(&self.uncomp_len.to_le_bytes())?;
303 Ok(())
304 }
305
306 fn from_bytes(buf: &[u8; 16]) -> Self {
308 Self {
309 rel_offset: u64::from_le_bytes(buf[0..8].try_into().unwrap()),
310 comp_len: u32::from_le_bytes(buf[8..12].try_into().unwrap()),
311 uncomp_len: u32::from_le_bytes(buf[12..16].try_into().unwrap()),
312 }
313 }
314}
315
316#[cfg(not(target_arch = "wasm32"))]
319pub fn migrate_v2_to_v3(path: &Path) -> io::Result<()> {
320 use std::io::{Seek, SeekFrom};
321 let mut f = OpenOptions::new().read(true).write(true).open(path)?;
322 let mut header = [0u8; HEADER_SIZE];
323 f.read_exact(&mut header)?;
324 if &header[0..4] != &Q42_MAGIC {
325 return Err(io::Error::new(
326 io::ErrorKind::InvalidData,
327 "not a Q42 volume",
328 ));
329 }
330 let version = u16::from_le_bytes([header[4], header[5]]);
331 if version >= Q42_VERSION_V3 as u16 {
332 return Ok(()); }
334 header[4..6].copy_from_slice(&(Q42_VERSION_V4 as u16).to_le_bytes());
336 let flags = u16::from_le_bytes([header[6], header[7]]) & (FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED);
339 header[6..8].copy_from_slice(&flags.to_le_bytes());
340 header[88..256].fill(0);
342 f.seek(SeekFrom::Start(0))?;
343 f.write_all(&header)?;
344 f.flush()
345}
346
347#[cfg(not(target_arch = "wasm32"))]
349pub fn is_unified_volume(path: &Path) -> io::Result<bool> {
350 let mut f = File::open(path)?;
351 let mut magic = [0u8; 4];
352 f.read_exact(&mut magic)?;
353 Ok(magic == Q42_MAGIC)
354}
355
356pub fn encode_lex(lex: &HashMap<u64, String>) -> Result<Vec<u8>, LexError> {
360 crate::q42_lex_ns::serialize_namespaced_paged_lexicon(
361 lex,
362 crate::q42_lex::DEFAULT_LEX_PAGE_ENTRIES,
363 )
364}
365
366pub fn encode_lex_with_entries(lex: &HashMap<u64, LexiconEntry>) -> Result<Vec<u8>, LexError> {
368 let mut entries: Vec<(u64, &LexiconEntry)> = lex.iter().map(|(&h, e)| (h, e)).collect();
369 entries.sort_unstable_by_key(|&(h, _)| h);
370
371 let entry_count = entries.len() as u64;
372 let strings_offset = 32 + entry_count * 16;
373
374 let mut string_blob: Vec<u8> = Vec::new();
375 let mut index = Vec::with_capacity(entries.len() * 16);
376 for (hash, entry) in &entries {
377 let str_off = string_blob.len() as u64;
378 match entry {
379 LexiconEntry::String(text) => {
380 string_blob.push(0x01);
382 let b = text.as_bytes();
383 let len = u16::try_from(b.len()).map_err(|_| LexError::TermTooLong)?;
384 string_blob.extend_from_slice(&len.to_le_bytes());
385 string_blob.extend_from_slice(&b[..len as usize]);
386 }
387 LexiconEntry::EmbeddedTriple(triple) => {
388 string_blob.push(0x02);
390 for &id in triple {
391 string_blob.extend_from_slice(&id.to_le_bytes());
392 }
393 }
394 LexiconEntry::Webizen(webid) => {
395 string_blob.push(0x03);
397 let b = webid.as_bytes();
398 let len = u16::try_from(b.len()).map_err(|_| LexError::TermTooLong)?;
399 string_blob.extend_from_slice(&len.to_le_bytes());
400 string_blob.extend_from_slice(&b[..len as usize]);
401 }
402 }
403 index.extend_from_slice(&hash.to_le_bytes());
404 index.extend_from_slice(&str_off.to_le_bytes());
405 }
406
407 let mut out = Vec::with_capacity(strings_offset as usize + string_blob.len());
408 out.extend_from_slice(&LEX_MAGIC);
409 out.extend_from_slice(&entry_count.to_le_bytes());
410 out.extend_from_slice(&strings_offset.to_le_bytes());
411 out.extend_from_slice(&1u64.to_le_bytes());
412 out.extend_from_slice(&index);
413 out.extend_from_slice(&string_blob);
414 Ok(out)
415}
416
417pub fn encode_bidx(ranges: &[(u64, u64)]) -> Vec<u8> {
419 let block_count = ranges.len() as u32;
420 let mut out = Vec::with_capacity(16 + ranges.len() * 16);
421 out.extend_from_slice(&BIDX_MAGIC);
422 out.extend_from_slice(&1u32.to_le_bytes());
423 out.extend_from_slice(&block_count.to_le_bytes());
424 out.extend_from_slice(&0u32.to_le_bytes());
425 for (min, max) in ranges {
426 out.extend_from_slice(&min.to_le_bytes());
427 out.extend_from_slice(&max.to_le_bytes());
428 }
429 out
430}
431
432pub fn encode_field_range_index(ranges: &[(u64, u64, u64, u64, u64, u64)]) -> Vec<u8> {
436 let mut out = Vec::with_capacity(
437 FIELD_RANGE_INDEX_HEADER_BYTES + ranges.len() * FIELD_RANGE_INDEX_ENTRY_BYTES,
438 );
439 out.extend_from_slice(&FIELD_RANGE_INDEX_MAGIC);
440 out.extend_from_slice(&1u32.to_le_bytes());
441 out.extend_from_slice(&(ranges.len() as u32).to_le_bytes());
442 out.extend_from_slice(&0u32.to_le_bytes());
443 for range in ranges {
444 for value in [range.0, range.1, range.2, range.3, range.4, range.5] {
445 out.extend_from_slice(&value.to_le_bytes());
446 }
447 }
448 out
449}
450
451pub fn encode_superblock(seq_id: u64, quins: &[NQuin]) -> [u8; SUPERBLOCK_SIZE] {
453 debug_assert!(quins.len() <= QUINS_PER_BLOCK);
454 let mut block = [0u8; SUPERBLOCK_SIZE];
455 block[0..8].copy_from_slice(&seq_id.to_le_bytes());
456 block[16..24].copy_from_slice(&(quins.len() as u64).to_le_bytes());
457 let zero = [0u8; QUIN_SIZE];
458 let mut off = SUPERBLOCK_HEADER;
459 for q in quins {
460 block[off..off + QUIN_SIZE].copy_from_slice(bytemuck::bytes_of(q));
461 off += QUIN_SIZE;
462 }
463 for _ in quins.len()..QUINS_PER_BLOCK {
464 block[off..off + QUIN_SIZE].copy_from_slice(&zero);
465 off += QUIN_SIZE;
466 }
467 block
468}
469
470pub fn decode_superblock_quins(block: &[u8]) -> io::Result<Vec<NQuin>> {
472 if block.len() < SUPERBLOCK_HEADER {
473 return Err(io::Error::new(
474 io::ErrorKind::InvalidData,
475 "SuperBlock shorter than its 160-byte header",
476 ));
477 }
478 let live = u64::from_le_bytes(block[16..24].try_into().unwrap()) as usize;
479 if live > QUINS_PER_BLOCK {
480 return Err(io::Error::new(
481 io::ErrorKind::InvalidData,
482 "SuperBlock live Quin count exceeds capacity",
483 ));
484 }
485 let mut out = Vec::with_capacity(live);
486 for index in 0..live {
487 let offset = SUPERBLOCK_HEADER + index * QUIN_SIZE;
488 let end = offset + QUIN_SIZE;
489 if end > block.len() {
490 return Err(io::Error::new(
491 io::ErrorKind::InvalidData,
492 "SuperBlock truncated inside a Quin slot",
493 ));
494 }
495 out.push(bytemuck::pod_read_unaligned(&block[offset..end]));
496 }
497 Ok(out)
498}
499
500pub fn header_to_bytes(h: &Q42VolumeHeader) -> [u8; HEADER_SIZE] {
503 let mut buf = [0u8; HEADER_SIZE];
504 buf[0..4].copy_from_slice(&h.magic);
506 buf[4..6].copy_from_slice(&h.version.to_le_bytes());
507 buf[6..8].copy_from_slice(&h.flags.to_le_bytes());
508 buf[8..16].copy_from_slice(&h.lex_offset.to_le_bytes());
509 buf[16..24].copy_from_slice(&h.lex_length.to_le_bytes());
510 buf[24..32].copy_from_slice(&h.bidx_offset.to_le_bytes());
511 buf[32..40].copy_from_slice(&h.bidx_length.to_le_bytes());
512 buf[40..48].copy_from_slice(&h.block_dir_offset.to_le_bytes());
513 buf[48..56].copy_from_slice(&h.block_dir_length.to_le_bytes());
514 buf[56..64].copy_from_slice(&h.data_offset.to_le_bytes());
515 buf[64..72].copy_from_slice(&h.data_length.to_le_bytes());
516 buf[72..80].copy_from_slice(&h.block_count.to_le_bytes());
517 buf[80..84].copy_from_slice(&h.block_size.to_le_bytes());
518 buf[84..88].copy_from_slice(&h.quins_per_block.to_le_bytes());
519 buf[88..96].copy_from_slice(&h.temporal_index_offset.to_le_bytes());
521 buf[96..104].copy_from_slice(&h.temporal_index_length.to_le_bytes());
522 buf[104..136].copy_from_slice(&h.merkle_root);
523 buf[136..144].copy_from_slice(&h.assertion_timestamp.to_le_bytes());
524 buf[144..152].copy_from_slice(&h.dag_root_offset.to_le_bytes());
525 buf[152..160].copy_from_slice(&h.dag_root_length.to_le_bytes());
526 buf[160..168].copy_from_slice(&h.natural_person_did_offset.to_le_bytes());
527 buf[168..176].copy_from_slice(&h.software_agent_did_offset.to_le_bytes());
528 buf[176..256].copy_from_slice(&h._reserved);
529 buf
530}
531
532fn header_from_bytes(buf: &[u8; HEADER_SIZE]) -> io::Result<Q42VolumeHeader> {
535 if buf[0..4] != Q42_MAGIC {
536 return Err(io::Error::new(
537 io::ErrorKind::InvalidData,
538 "invalid Q42 magic",
539 ));
540 }
541 let version = u16::from_le_bytes(buf[4..6].try_into().unwrap());
542 if version != Q42_VERSION_V3 && version != Q42_VERSION_V4 {
543 return Err(io::Error::new(
544 io::ErrorKind::InvalidData,
545 format!("Q42 file is version {version}; v3/v4 required"),
546 ));
547 }
548 Ok(Q42VolumeHeader {
549 magic: Q42_MAGIC,
550 version,
551 flags: u16::from_le_bytes(buf[6..8].try_into().unwrap()),
552 lex_offset: u64::from_le_bytes(buf[8..16].try_into().unwrap()),
553 lex_length: u64::from_le_bytes(buf[16..24].try_into().unwrap()),
554 bidx_offset: u64::from_le_bytes(buf[24..32].try_into().unwrap()),
555 bidx_length: u64::from_le_bytes(buf[32..40].try_into().unwrap()),
556 block_dir_offset: u64::from_le_bytes(buf[40..48].try_into().unwrap()),
557 block_dir_length: u64::from_le_bytes(buf[48..56].try_into().unwrap()),
558 data_offset: u64::from_le_bytes(buf[56..64].try_into().unwrap()),
559 data_length: u64::from_le_bytes(buf[64..72].try_into().unwrap()),
560 block_count: u64::from_le_bytes(buf[72..80].try_into().unwrap()),
561 block_size: u32::from_le_bytes(buf[80..84].try_into().unwrap()),
562 quins_per_block: u32::from_le_bytes(buf[84..88].try_into().unwrap()),
563 temporal_index_offset: u64::from_le_bytes(buf[88..96].try_into().unwrap()),
564 temporal_index_length: u64::from_le_bytes(buf[96..104].try_into().unwrap()),
565 merkle_root: buf[104..136].try_into().unwrap(),
566 assertion_timestamp: u64::from_le_bytes(buf[136..144].try_into().unwrap()),
567 dag_root_offset: u64::from_le_bytes(buf[144..152].try_into().unwrap()),
568 dag_root_length: u64::from_le_bytes(buf[152..160].try_into().unwrap()),
569 natural_person_did_offset: u64::from_le_bytes(buf[160..168].try_into().unwrap()),
570 software_agent_did_offset: u64::from_le_bytes(buf[168..176].try_into().unwrap()),
571 _reserved: buf[176..256].try_into().unwrap(),
572 })
573}
574
575#[cfg(not(target_arch = "wasm32"))]
577pub fn write_unified_volume(
578 path: &Path,
579 lex: &HashMap<u64, String>,
580 block_ranges: &[(u64, u64)],
581 blocks: &[Vec<NQuin>],
582) -> io::Result<()> {
583 if blocks.len() != block_ranges.len() {
584 return Err(io::Error::new(
585 io::ErrorKind::InvalidInput,
586 "block count mismatch",
587 ));
588 }
589 let mut writer = StreamingQ42VolumeWriter::new(lex)?;
590 for ((seq, quins), declared_range) in blocks.iter().enumerate().zip(block_ranges) {
591 let actual_range = object_range(quins)?;
592 if actual_range != *declared_range {
593 return Err(io::Error::new(
594 io::ErrorKind::InvalidInput,
595 format!(
596 "declared BIDX range {:?} does not match block {seq} object range {:?}",
597 declared_range, actual_range
598 ),
599 ));
600 }
601 writer.push_block(seq as u64, quins)?;
602 }
603 writer.finish(path)
604}
605
606#[cfg(not(target_arch = "wasm32"))]
610pub fn write_unified_volume_with_entries(
611 path: &Path,
612 lex: &HashMap<u64, LexiconEntry>,
613 block_ranges: &[(u64, u64)],
614 blocks: &[Vec<NQuin>],
615) -> io::Result<()> {
616 if blocks.len() != block_ranges.len() {
617 return Err(io::Error::new(
618 io::ErrorKind::InvalidInput,
619 "block count mismatch",
620 ));
621 }
622 let mut builder = UnifiedVolumeBuilder::with_lex_entries(lex).map_err(lex_error_to_io)?;
623 for ((seq, quins), declared_range) in blocks.iter().enumerate().zip(block_ranges) {
624 let actual_range = object_range(quins)?;
625 if actual_range != *declared_range {
626 return Err(io::Error::new(
627 io::ErrorKind::InvalidInput,
628 format!(
629 "declared BIDX range {:?} does not match block {seq} object range {:?}",
630 declared_range, actual_range
631 ),
632 ));
633 }
634 builder.push_block(seq as u64, quins)?;
635 }
636 builder.finish(path)
637}
638
639#[cfg(not(target_arch = "wasm32"))]
643pub fn write_volume_root(path: &Path, manifest: &Q42VolumeManifest) -> io::Result<()> {
644 UnifiedVolumeBuilder::with_empty_lex()
645 .with_volume_manifest(manifest)?
646 .finish(path)
647}
648
649#[cfg(not(target_arch = "wasm32"))]
652pub fn write_volume_root_for_commons(path: &Path, manifest: &Q42VolumeManifest) -> io::Result<()> {
653 UnifiedVolumeBuilder::with_empty_lex()
654 .with_permissive_commons()
655 .with_volume_manifest(manifest)?
656 .finish(path)
657}
658
659#[cfg(not(target_arch = "wasm32"))]
663pub fn write_volume_root_with_lex(
664 path: &Path,
665 lex: &HashMap<u64, String>,
666 manifest: &Q42VolumeManifest,
667) -> io::Result<()> {
668 UnifiedVolumeBuilder::with_lex_map(lex)
669 .map_err(lex_error_to_io)?
670 .with_volume_manifest(manifest)?
671 .finish(path)
672}
673
674#[cfg(not(target_arch = "wasm32"))]
675fn lex_error_to_io(error: LexError) -> io::Error {
678 io::Error::new(
679 io::ErrorKind::InvalidInput,
680 format!("invalid Q42LEX: {error:?}"),
681 )
682}
683
684fn object_range(quins: &[NQuin]) -> io::Result<(u64, u64)> {
690 let Some(first) = quins.first() else {
691 return Err(io::Error::new(
692 io::ErrorKind::InvalidInput,
693 "a Q42 SuperBlock must contain at least one Quin",
694 ));
695 };
696 if quins.len() > QUINS_PER_BLOCK {
697 return Err(io::Error::new(
698 io::ErrorKind::InvalidInput,
699 format!(
700 "Q42 SuperBlock contains {} Quins, exceeding capacity {QUINS_PER_BLOCK}",
701 quins.len()
702 ),
703 ));
704 }
705
706 let mut previous = first.object;
707 for quin in &quins[1..] {
708 if quin.object < previous {
709 return Err(io::Error::new(
710 io::ErrorKind::InvalidInput,
711 "Q42 SuperBlock Quins must be sorted by object hash",
712 ));
713 }
714 previous = quin.object;
715 }
716 Ok((first.object, previous))
717}
718
719pub struct UnifiedVolumeBuilder {
721 lex_bytes: Vec<u8>,
722 volume_manifest: Option<Vec<u8>>,
723 block_ranges: Vec<(u64, u64)>,
724 field_ranges: Vec<(u64, u64, u64, u64, u64, u64)>,
725 dir_entries: Vec<BlockDirectoryEntry>,
726 field_postings: Vec<BlockFieldPostings>,
727 data_blob: Vec<u8>,
728 dag_store: crate::git_bridge::DagStore,
730 author_did: u64,
732 last_dag_hash: [u8; 32],
734 last_object_hash: Option<u64>,
736 publication_commons: bool,
737 publication_sanctuary: bool,
738 sanctuary_quin_count: u64,
739 payload_catalogue_blobs: Vec<PayloadCatalogueBlob>,
740 payload_catalogue_bytes: Option<Vec<u8>>,
741}
742
743impl UnifiedVolumeBuilder {
744 pub fn with_lex_map(lex: &HashMap<u64, String>) -> Result<Self, LexError> {
747 Ok(Self {
748 lex_bytes: encode_lex(lex)?,
749 volume_manifest: None,
750 block_ranges: Vec::new(),
751 field_ranges: Vec::new(),
752 dir_entries: Vec::new(),
753 field_postings: Vec::new(),
754 data_blob: Vec::new(),
755 dag_store: crate::git_bridge::DagStore::new(),
756 author_did: 0,
757 last_dag_hash: [0u8; 32],
758 last_object_hash: None,
759 publication_commons: false,
760 publication_sanctuary: false,
761 sanctuary_quin_count: 0,
762 payload_catalogue_blobs: Vec::new(),
763 payload_catalogue_bytes: None,
764 })
765 }
766
767 pub fn with_lex_entries(lex: &HashMap<u64, LexiconEntry>) -> Result<Self, LexError> {
769 Ok(Self {
770 lex_bytes: encode_lex_with_entries(lex)?,
771 volume_manifest: None,
772 block_ranges: Vec::new(),
773 field_ranges: Vec::new(),
774 dir_entries: Vec::new(),
775 field_postings: Vec::new(),
776 data_blob: Vec::new(),
777 dag_store: crate::git_bridge::DagStore::new(),
778 author_did: 0,
779 last_dag_hash: [0u8; 32],
780 last_object_hash: None,
781 publication_commons: false,
782 publication_sanctuary: false,
783 sanctuary_quin_count: 0,
784 payload_catalogue_blobs: Vec::new(),
785 payload_catalogue_bytes: None,
786 })
787 }
788
789 pub fn add_payload_catalogue_utf8(
792 &mut self,
793 payload_key: &str,
794 mime_type: &str,
795 language: Option<&str>,
796 role: &str,
797 utf8_body: &str,
798 ) -> io::Result<PayloadCatalogueEntry> {
799 let blob = PayloadCatalogueBlob::new_utf8(
800 payload_key,
801 mime_type,
802 language.map(str::to_string),
803 role,
804 utf8_body,
805 )
806 .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
807 let entry = blob.entry.clone();
808 self.payload_catalogue_blobs.push(blob);
809 let encoded = encode_payload_catalogue_with_bodies(&self.payload_catalogue_blobs)
810 .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
811 self.payload_catalogue_bytes = Some(encoded);
812 Ok(entry)
813 }
814
815 pub fn with_payload_catalogue_blobs(mut self, blobs: &[PayloadCatalogueBlob]) -> io::Result<Self> {
817 if blobs.is_empty() {
818 self.payload_catalogue_blobs.clear();
819 self.payload_catalogue_bytes = None;
820 return Ok(self);
821 }
822 let encoded = encode_payload_catalogue_with_bodies(blobs)
823 .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
824 self.payload_catalogue_blobs = blobs.to_vec();
825 self.payload_catalogue_bytes = Some(encoded);
826 Ok(self)
827 }
828
829 pub fn with_payload_catalogue_entries(
831 mut self,
832 entries: &[PayloadCatalogueEntry],
833 ) -> io::Result<Self> {
834 if entries.is_empty() {
835 self.payload_catalogue_blobs.clear();
836 self.payload_catalogue_bytes = None;
837 return Ok(self);
838 }
839 let encoded = encode_payload_catalogue_section(entries)
840 .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
841 self.payload_catalogue_blobs.clear();
842 self.payload_catalogue_bytes = Some(encoded);
843 Ok(self)
844 }
845
846 pub fn with_empty_lex() -> Self {
848 Self::with_lex_map(&HashMap::new()).expect("an empty Q42LEX is valid")
849 }
850
851 pub fn with_volume_manifest(mut self, manifest: &Q42VolumeManifest) -> io::Result<Self> {
854 self.volume_manifest = Some(manifest.encode()?);
855 Ok(self)
856 }
857
858 pub fn with_author_did(mut self, did: u64) -> Self {
860 self.author_did = did;
861 self
862 }
863
864 pub fn with_permissive_commons(mut self) -> Self {
866 self.publication_commons = true;
867 self
868 }
869
870 pub fn with_sanctuary(mut self) -> Self {
872 self.publication_sanctuary = true;
873 self
874 }
875
876 pub fn push_block(&mut self, seq_id: u64, quins: &[NQuin]) -> io::Result<()> {
881 let (min_hash, max_hash) = object_range(quins)?;
882 if let Some(previous) = self.last_object_hash {
883 if min_hash < previous {
884 return Err(io::Error::new(
885 io::ErrorKind::InvalidInput,
886 format!(
887 "block {seq_id} starts at object hash {min_hash:#018X}, before prior block maximum {previous:#018X}"
888 ),
889 ));
890 }
891 }
892 self.block_ranges.push((min_hash, max_hash));
893 let mut subject_min = quins[0].subject;
894 let mut subject_max = quins[0].subject;
895 let mut predicate_min = quins[0].predicate;
896 let mut predicate_max = quins[0].predicate;
897 let mut context_min = quins[0].context;
898 let mut context_max = quins[0].context;
899 for quin in &quins[1..] {
900 subject_min = subject_min.min(quin.subject);
901 subject_max = subject_max.max(quin.subject);
902 predicate_min = predicate_min.min(quin.predicate);
903 predicate_max = predicate_max.max(quin.predicate);
904 context_min = context_min.min(quin.context);
905 context_max = context_max.max(quin.context);
906 }
907 self.sanctuary_quin_count += quins
908 .iter()
909 .filter(|quin| volume::quin_requires_sanctuary(quin))
910 .count() as u64;
911 self.field_ranges.push((
912 subject_min,
913 subject_max,
914 predicate_min,
915 predicate_max,
916 context_min,
917 context_max,
918 ));
919 self.field_postings
920 .push(BlockFieldPostings::from_quins(quins));
921 let raw = encode_superblock(seq_id, quins);
922 let compressed = lz4_flex::compress_prepend_size(&raw);
923 self.dir_entries.push(BlockDirectoryEntry {
924 rel_offset: self.data_blob.len() as u64,
925 comp_len: compressed.len() as u32,
926 uncomp_len: SUPERBLOCK_SIZE as u32,
927 });
928 self.data_blob.extend_from_slice(&compressed);
929
930 let ts = std::time::SystemTime::now()
932 .duration_since(std::time::UNIX_EPOCH)
933 .map(|d| d.as_millis() as u64)
934 .unwrap_or(0);
935 let msg = format!("ingest block {seq_id}");
936 self.last_dag_hash = if self.last_dag_hash == [0u8; 32] {
937 self.dag_store
938 .genesis_node(quins, self.author_did, ts, &msg)
939 } else {
940 self.dag_store
941 .commit_node(self.last_dag_hash, quins, self.author_did, ts, &msg)
942 };
943 self.last_object_hash = Some(max_hash);
944 Ok(())
945 }
946
947 pub fn block_count(&self) -> u64 {
949 self.block_ranges.len() as u64
950 }
951
952 #[cfg(not(target_arch = "wasm32"))]
955 pub fn finish(self, path: &Path) -> io::Result<()> {
956 let bytes = self.finish_to_bytes();
957 let out = OpenOptions::new()
958 .create(true)
959 .write(true)
960 .truncate(true)
961 .open(path)?;
962 let mut w = BufWriter::new(out);
963 w.write_all(&bytes)?;
964 w.flush()?;
965 Ok(())
966 }
967
968 pub fn finish_to_bytes(self) -> Vec<u8> {
975 let bidx_bytes = encode_bidx(&self.block_ranges);
976 let field_range_bytes = encode_field_range_index(&self.field_ranges);
977 let postings_bytes = encode_postings_section(&self.field_postings);
978 let block_count = self.block_ranges.len() as u64;
979 let manifest_bytes = self.volume_manifest.as_deref().unwrap_or(&[]);
980
981 let lex_offset = HEADER_SIZE as u64;
982 let manifest_offset = lex_offset + self.lex_bytes.len() as u64;
983 let bidx_offset = manifest_offset + manifest_bytes.len() as u64;
984 let field_range_offset = bidx_offset + bidx_bytes.len() as u64;
985 let postings_offset = field_range_offset + field_range_bytes.len() as u64;
986 let block_dir_offset = postings_offset + postings_bytes.len() as u64;
987 let block_dir_length = block_count * BlockDirectoryEntry::SIZE as u64;
988 let data_offset = block_dir_offset + block_dir_length;
989
990 let assertion_timestamp = std::time::SystemTime::now()
991 .duration_since(std::time::UNIX_EPOCH)
992 .map(|d| d.as_millis() as u64)
993 .unwrap_or(0);
994
995 let dag_bytes = self.dag_store.serialize();
997 let dag_root_offset = if dag_bytes.is_empty() {
998 0
999 } else {
1000 data_offset + self.data_blob.len() as u64
1001 };
1002 let dag_root_length = dag_bytes.len() as u64;
1003
1004 let merkle_root = if self.last_dag_hash == [0u8; 32] {
1006 [0u8; 32]
1007 } else {
1008 use sha2::{Digest, Sha256};
1010 let mut h = Sha256::new();
1011 h.update(self.last_dag_hash);
1012 h.finalize().into()
1013 };
1014
1015 let mut reserved = [0u8; 80];
1016 let mut flags = FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED;
1017 if self.publication_sanctuary || self.sanctuary_quin_count > 0 {
1018 flags |= FLAG_SANCTUARY;
1019 } else if self.publication_commons {
1020 flags |= FLAG_PERMISSIVE_COMMONS;
1021 }
1022 if !self.field_ranges.is_empty() {
1023 flags |= FLAG_FIELD_RANGES;
1024 reserved[16..24].copy_from_slice(&field_range_offset.to_le_bytes());
1025 reserved[24..32].copy_from_slice(&(field_range_bytes.len() as u64).to_le_bytes());
1026 }
1027 if !self.field_postings.is_empty() {
1028 flags |= FLAG_FIELD_POSTINGS;
1029 reserved[32..40].copy_from_slice(&postings_offset.to_le_bytes());
1030 reserved[40..48].copy_from_slice(&(postings_bytes.len() as u64).to_le_bytes());
1031 }
1032 if !manifest_bytes.is_empty() {
1033 flags |= FLAG_VOLUME_ROOT;
1034 reserved[0..8].copy_from_slice(&manifest_offset.to_le_bytes());
1035 reserved[8..16].copy_from_slice(&(manifest_bytes.len() as u64).to_le_bytes());
1036 }
1037 let pcat_bytes = self.payload_catalogue_bytes.as_deref().unwrap_or(&[]);
1038 let mut header = Q42VolumeHeader {
1039 magic: Q42_MAGIC,
1040 version: Q42_VERSION_V4,
1041 flags,
1042 lex_offset,
1043 lex_length: self.lex_bytes.len() as u64,
1044 bidx_offset,
1045 bidx_length: bidx_bytes.len() as u64,
1046 block_dir_offset,
1047 block_dir_length,
1048 data_offset,
1049 data_length: self.data_blob.len() as u64,
1050 block_count,
1051 block_size: SUPERBLOCK_SIZE as u32,
1052 quins_per_block: QUINS_PER_BLOCK as u32,
1053 temporal_index_offset: 0,
1054 temporal_index_length: 0,
1055 merkle_root,
1056 assertion_timestamp,
1057 dag_root_offset,
1058 dag_root_length,
1059 natural_person_did_offset: 0,
1060 software_agent_did_offset: 0,
1061 _reserved: reserved,
1062 };
1063 if !pcat_bytes.is_empty() {
1064 let pcat_offset =
1065 data_offset + self.data_blob.len() as u64 + dag_bytes.len() as u64;
1066 set_header_payload_catalogue_range(&mut header, pcat_offset, pcat_bytes.len() as u64);
1067 }
1068
1069 let mut out = Vec::with_capacity(
1070 HEADER_SIZE
1071 + self.lex_bytes.len()
1072 + manifest_bytes.len()
1073 + bidx_bytes.len()
1074 + field_range_bytes.len()
1075 + postings_bytes.len()
1076 + block_dir_length as usize
1077 + self.data_blob.len()
1078 + dag_bytes.len()
1079 + pcat_bytes.len(),
1080 );
1081 out.extend_from_slice(&header_to_bytes(&header));
1082 out.extend_from_slice(&self.lex_bytes);
1083 out.extend_from_slice(manifest_bytes);
1084 out.extend_from_slice(&bidx_bytes);
1085 out.extend_from_slice(&field_range_bytes);
1086 out.extend_from_slice(&postings_bytes);
1087 for entry in &self.dir_entries {
1088 entry.write_to(&mut out).expect("Vec<u8> write cannot fail");
1090 }
1091 out.extend_from_slice(&self.data_blob);
1092 if !dag_bytes.is_empty() {
1093 out.extend_from_slice(&dag_bytes);
1094 }
1095 if !pcat_bytes.is_empty() {
1096 out.extend_from_slice(pcat_bytes);
1097 }
1098 out
1099 }
1100}
1101
1102#[cfg(not(target_arch = "wasm32"))]
1104pub struct Q42Volume {
1105 mmap: Mmap,
1106 header: Q42VolumeHeader,
1107}
1108
1109#[cfg(not(target_arch = "wasm32"))]
1110impl Q42Volume {
1111 pub fn open(path: &Path) -> io::Result<Self> {
1114 let file = File::open(path)?;
1115 let mmap = unsafe { MmapOptions::new().map(&file)? };
1116 if mmap.len() < HEADER_SIZE {
1117 return Err(io::Error::new(
1118 io::ErrorKind::UnexpectedEof,
1119 "file too small for Q42 header",
1120 ));
1121 }
1122 let mut hdr_buf = [0u8; HEADER_SIZE];
1123 hdr_buf.copy_from_slice(&mmap[0..HEADER_SIZE]);
1124 let header = header_from_bytes(&hdr_buf)?;
1125 volume::validate_volume_structure(&header, &mmap)?;
1126 Ok(Self { mmap, header })
1127 }
1128
1129 pub fn header(&self) -> &Q42VolumeHeader {
1131 &self.header
1132 }
1133
1134 pub fn as_bytes(&self) -> &[u8] {
1136 &self.mmap
1137 }
1138
1139 pub fn lex_bytes(&self) -> &[u8] {
1141 let start = self.header.lex_offset as usize;
1142 let end = start + self.header.lex_length as usize;
1143 &self.mmap[start..end]
1144 }
1145
1146 pub fn lex_view(&self) -> Result<Q42LexMmap<'_>, LexError> {
1148 Q42LexMmap::from_bytes(self.lex_bytes())
1149 }
1150
1151 pub fn bidx_bytes(&self) -> &[u8] {
1153 let start = self.header.bidx_offset as usize;
1154 let end = start + self.header.bidx_length as usize;
1155 &self.mmap[start..end]
1156 }
1157
1158 pub fn volume_manifest_bytes(&self) -> io::Result<Option<&[u8]>> {
1160 let Some((offset, length)) = self.header.volume_manifest_range() else {
1161 return Ok(None);
1162 };
1163 let length = usize::try_from(length).map_err(|_| {
1164 io::Error::new(
1165 io::ErrorKind::InvalidData,
1166 "Q42 manifest length exceeds platform",
1167 )
1168 })?;
1169 if length == 0 || length > MAX_VOLUME_MANIFEST_BYTES {
1170 return Err(io::Error::new(
1171 io::ErrorKind::InvalidData,
1172 "Q42 root has an invalid embedded volume manifest length",
1173 ));
1174 }
1175 let start = usize::try_from(offset).map_err(|_| {
1176 io::Error::new(
1177 io::ErrorKind::InvalidData,
1178 "Q42 manifest offset exceeds platform",
1179 )
1180 })?;
1181 let end = start.checked_add(length).ok_or_else(|| {
1182 io::Error::new(io::ErrorKind::InvalidData, "Q42 manifest range overflows")
1183 })?;
1184 self.mmap.get(start..end).map(Some).ok_or_else(|| {
1185 io::Error::new(
1186 io::ErrorKind::InvalidData,
1187 "Q42 manifest lies outside the file",
1188 )
1189 })
1190 }
1191
1192 pub fn volume_manifest(&self) -> io::Result<Option<Q42VolumeManifest>> {
1195 self.volume_manifest_bytes()?
1196 .map(Q42VolumeManifest::decode)
1197 .transpose()
1198 }
1199
1200 pub fn payload_catalogue_bytes(&self) -> io::Result<Option<&[u8]>> {
1202 let Some((offset, length)) = self.header.payload_catalogue_range() else {
1203 return Ok(None);
1204 };
1205 let start = usize::try_from(offset).map_err(|_| {
1206 io::Error::new(
1207 io::ErrorKind::InvalidData,
1208 "Q42 payload catalogue offset exceeds platform",
1209 )
1210 })?;
1211 let length = usize::try_from(length).map_err(|_| {
1212 io::Error::new(
1213 io::ErrorKind::InvalidData,
1214 "Q42 payload catalogue length exceeds platform",
1215 )
1216 })?;
1217 let end = start.checked_add(length).ok_or_else(|| {
1218 io::Error::new(
1219 io::ErrorKind::InvalidData,
1220 "Q42 payload catalogue range overflows",
1221 )
1222 })?;
1223 self.mmap.get(start..end).map(Some).ok_or_else(|| {
1224 io::Error::new(
1225 io::ErrorKind::InvalidData,
1226 "Q42 payload catalogue lies outside the file",
1227 )
1228 })
1229 }
1230
1231 pub fn read_payload_catalogue(&self) -> io::Result<Vec<PayloadCatalogueBlob>> {
1235 let Some(bytes) = self.payload_catalogue_bytes()? else {
1236 return Ok(Vec::new());
1237 };
1238 decode_payload_catalogue_with_bodies(bytes)
1239 .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
1240 }
1241
1242 pub fn read_payload_catalogue_entries(&self) -> io::Result<Vec<PayloadCatalogueEntry>> {
1244 let Some(bytes) = self.payload_catalogue_bytes()? else {
1245 return Ok(Vec::new());
1246 };
1247 decode_payload_catalogue_section(bytes)
1248 .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
1249 }
1250
1251 pub fn block_count(&self) -> u64 {
1253 self.header.block_count
1254 }
1255
1256 pub fn verify_all_blocks(&self) -> io::Result<Q42VerificationReceipt> {
1261 if self.volume_manifest()?.is_some() {
1262 return Err(io::Error::new(
1263 io::ErrorKind::InvalidInput,
1264 "verify a Q42 logical root through its manifest child segments",
1265 ));
1266 }
1267 let mut decoded = [0u8; SUPERBLOCK_SIZE];
1268 let mut quins_verified = 0u64;
1269 let mut previous_object = None;
1270 for block_index in 0..self.block_count() as usize {
1271 self.read_superblock_into(block_index, &mut decoded)?;
1272 let live = u64::from_le_bytes(decoded[16..24].try_into().unwrap()) as usize;
1273 if live == 0 || live > QUINS_PER_BLOCK {
1274 return Err(io::Error::new(
1275 io::ErrorKind::InvalidData,
1276 "Q42 SuperBlock has invalid live Quin count",
1277 ));
1278 }
1279 let bidx_offset = 16 + block_index * 16;
1280 let bidx = self.bidx_bytes();
1281 let advertised_min =
1282 u64::from_le_bytes(bidx[bidx_offset..bidx_offset + 8].try_into().unwrap());
1283 let advertised_max =
1284 u64::from_le_bytes(bidx[bidx_offset + 8..bidx_offset + 16].try_into().unwrap());
1285 let mut first = None;
1286 let mut last = 0u64;
1287 for quin_index in 0..live {
1288 let offset = SUPERBLOCK_HEADER + quin_index * QUIN_SIZE;
1289 let quin: crate::NQuin =
1290 bytemuck::pod_read_unaligned(&decoded[offset..offset + QUIN_SIZE]);
1291 if !quin.verify_ecc_parity() {
1292 return Err(io::Error::new(
1293 io::ErrorKind::InvalidData,
1294 format!("Q42 parity mismatch in block {block_index}, Quin {quin_index}"),
1295 ));
1296 }
1297 if previous_object.is_some_and(|previous| quin.object < previous) {
1298 return Err(io::Error::new(
1299 io::ErrorKind::InvalidData,
1300 format!("Q42 object order regresses in block {block_index}"),
1301 ));
1302 }
1303 first.get_or_insert(quin.object);
1304 last = quin.object;
1305 previous_object = Some(quin.object);
1306 quins_verified += 1;
1307 }
1308 if first != Some(advertised_min) || last != advertised_max {
1309 return Err(io::Error::new(
1310 io::ErrorKind::InvalidData,
1311 format!("Q42 BIDX range disagrees with decoded block {block_index}"),
1312 ));
1313 }
1314 }
1315 Ok(Q42VerificationReceipt {
1316 blocks_verified: self.block_count(),
1317 quins_verified,
1318 })
1319 }
1320
1321 pub fn bidx_blocks_for_hash(&self, object_hash: u64) -> Vec<usize> {
1327 bidx_blocks_for_hash(self.bidx_bytes(), object_hash)
1328 }
1329
1330 pub fn bidx_block_range_for_hash(
1332 &self,
1333 object_hash: u64,
1334 ) -> io::Result<Option<BidxBlockRange>> {
1335 volume::bidx_block_range_for_hash(self.bidx_bytes(), object_hash)
1336 }
1337
1338 pub fn bidx_blocks_for_hash_into(
1344 &self,
1345 object_hash: u64,
1346 cursor: usize,
1347 out: &mut [usize],
1348 ) -> io::Result<Option<BidxMatchPage>> {
1349 volume::bidx_blocks_for_hash_into(self.bidx_bytes(), object_hash, cursor, out)
1350 }
1351
1352 pub fn object_hash_bounds(&self) -> Option<(u64, u64)> {
1355 let count = self.block_count() as usize;
1356 if count == 0 {
1357 return None;
1358 }
1359 let bidx = self.bidx_bytes();
1360 let first = u64::from_le_bytes(bidx[16..24].try_into().ok()?);
1361 let last_offset = 16 + (count - 1) * 16;
1362 let last = u64::from_le_bytes(bidx[last_offset + 8..last_offset + 16].try_into().ok()?);
1363 Some((first, last))
1364 }
1365
1366 pub fn block_directory_entry(&self, index: usize) -> io::Result<BlockDirectoryEntry> {
1369 if index >= self.header.block_count as usize {
1370 return Err(io::Error::new(
1371 io::ErrorKind::InvalidInput,
1372 "block index out of range",
1373 ));
1374 }
1375 let start = self.header.block_dir_offset as usize + index * BlockDirectoryEntry::SIZE;
1376 let end = start + BlockDirectoryEntry::SIZE;
1377 let mut buf = [0u8; 16];
1378 buf.copy_from_slice(&self.mmap[start..end]);
1379 Ok(BlockDirectoryEntry::from_bytes(&buf))
1380 }
1381
1382 pub fn read_superblock_into(&self, index: usize, out: &mut [u8]) -> io::Result<usize> {
1384 if out.len() < SUPERBLOCK_SIZE {
1385 return Err(io::Error::new(
1386 io::ErrorKind::InvalidInput,
1387 "output buffer too small",
1388 ));
1389 }
1390 let entry = self.block_directory_entry(index)?;
1391 let data_offset = usize::try_from(self.header.data_offset).map_err(|_| {
1392 io::Error::new(
1393 io::ErrorKind::InvalidData,
1394 "Q42 data offset does not fit this platform",
1395 )
1396 })?;
1397 let rel_offset = usize::try_from(entry.rel_offset).map_err(|_| {
1398 io::Error::new(
1399 io::ErrorKind::InvalidData,
1400 "Q42 block offset does not fit this platform",
1401 )
1402 })?;
1403 let start = data_offset.checked_add(rel_offset).ok_or_else(|| {
1404 io::Error::new(
1405 io::ErrorKind::InvalidData,
1406 "Q42 block offset overflows usize",
1407 )
1408 })?;
1409 let end = start.checked_add(entry.comp_len as usize).ok_or_else(|| {
1410 io::Error::new(
1411 io::ErrorKind::InvalidData,
1412 "Q42 block length overflows usize",
1413 )
1414 })?;
1415 let compressed = &self.mmap[start..end];
1416 if compressed.len() < 4 {
1417 return Err(io::Error::new(
1418 io::ErrorKind::InvalidData,
1419 "Q42 LZ4 block is shorter than its size prefix",
1420 ));
1421 }
1422 let declared = u32::from_le_bytes(compressed[0..4].try_into().unwrap()) as usize;
1423 if declared != entry.uncomp_len as usize || declared != SUPERBLOCK_SIZE {
1424 return Err(io::Error::new(
1425 io::ErrorKind::InvalidData,
1426 "Q42 LZ4 size prefix disagrees with the block directory",
1427 ));
1428 }
1429 let decoded =
1430 lz4_flex::decompress_into(&compressed[4..], &mut out[..declared]).map_err(|e| {
1431 io::Error::new(
1432 io::ErrorKind::InvalidData,
1433 format!("LZ4 decompress block {index}: {e}"),
1434 )
1435 })?;
1436 if decoded != declared {
1437 return Err(io::Error::new(
1438 io::ErrorKind::InvalidData,
1439 "Q42 LZ4 block decoded to an unexpected length",
1440 ));
1441 }
1442 Ok(decoded)
1443 }
1444
1445 pub fn read_all_quins(&self) -> io::Result<Vec<crate::NQuin>> {
1453 if self.volume_manifest()?.is_some() {
1454 return Err(io::Error::new(
1455 io::ErrorKind::InvalidInput,
1456 "Q42 logical root has no local graph blocks; open Q42VolumeSet or use read_q42_quins",
1457 ));
1458 }
1459 let rd = |b: &[u8], o: usize| u64::from_le_bytes(b[o..o + 8].try_into().unwrap());
1460 let mut out = Vec::new();
1461 let mut sb = vec![0u8; SUPERBLOCK_SIZE];
1462 for i in 0..self.block_count() as usize {
1463 self.read_superblock_into(i, &mut sb)?;
1464 let quin_count = rd(&sb, 16) as usize;
1465 for k in 0..quin_count {
1466 let o = SUPERBLOCK_HEADER + k * QUIN_SIZE;
1467 if o + QUIN_SIZE > sb.len() {
1468 break;
1469 }
1470 out.push(crate::NQuin {
1471 subject: rd(&sb, o),
1472 predicate: rd(&sb, o + 8),
1473 object: rd(&sb, o + 16),
1474 context: rd(&sb, o + 24),
1475 metadata: rd(&sb, o + 32),
1476 parity: rd(&sb, o + 40),
1477 });
1478 }
1479 }
1480 Ok(out)
1481 }
1482}
1483
1484pub fn bidx_blocks_for_hash(bidx: &[u8], object_hash: u64) -> Vec<usize> {
1486 match volume::bidx_block_range_for_hash(bidx, object_hash) {
1487 Ok(Some(range)) => (range.start..range.end).collect(),
1488 Ok(None) | Err(_) => Vec::new(),
1489 }
1490}
1491
1492#[cfg(all(test, not(target_arch = "wasm32")))]
1493mod tests {
1494 use super::*;
1495 use crate::mini_parser::hash_token;
1496 use tempfile::{NamedTempFile, TempDir};
1497
1498 fn sample_quin(subj: &str, pred: &str, obj: &str) -> (NQuin, HashMap<u64, String>) {
1499 let mut lex = HashMap::new();
1500 let sh = hash_token(subj);
1501 let ph = hash_token(pred);
1502 let oh = hash_token(obj);
1503 lex.insert(sh, subj.to_string());
1504 lex.insert(ph, pred.to_string());
1505 lex.insert(oh, obj.to_string());
1506 let q = NQuin {
1507 subject: sh,
1508 predicate: ph,
1509 object: oh,
1510 context: 0,
1511 metadata: 0,
1512 parity: 0,
1513 };
1514 (q, lex)
1515 }
1516
1517 #[test]
1518 fn unified_volume_roundtrip() {
1519 let (q1, mut lex) = sample_quin("Patient", "fever", "True");
1520 let (q2, lex2) = sample_quin("Patient", "has", "pain");
1521 lex.extend(lex2);
1522
1523 let mut blocks = vec![vec![q1], vec![q2]];
1524 blocks.sort_by_key(|chunk| chunk[0].object);
1525
1526 let tmp = NamedTempFile::new().unwrap();
1527 let ranges: Vec<_> = blocks
1528 .iter()
1529 .map(|chunk| {
1530 let h = chunk[0].object;
1531 (h, h)
1532 })
1533 .collect();
1534 write_unified_volume(tmp.path(), &lex, &ranges, &blocks).unwrap();
1535
1536 let vol = Q42Volume::open(tmp.path()).unwrap();
1537 assert_eq!(vol.block_count(), 2);
1538 assert!(vol.lex_view().unwrap().contains(q1.object));
1539
1540 let hits = vol.bidx_blocks_for_hash(q1.object);
1541 assert!(!hits.is_empty(), "bidx miss for object hash {}", q1.object);
1542
1543 let mut block = [0u8; SUPERBLOCK_SIZE];
1544 vol.read_superblock_into(hits[0], &mut block).unwrap();
1545 let active = u64::from_le_bytes(block[16..24].try_into().unwrap());
1546 assert_eq!(active, 1);
1547 }
1548
1549 #[test]
1550 fn exhaustive_verifier_checks_decoded_quins_against_bidx() {
1551 let tmp = NamedTempFile::new().unwrap();
1552 let mut quin = NQuin {
1553 subject: 11,
1554 predicate: 22,
1555 object: 33,
1556 context: 44,
1557 metadata: 55,
1558 parity: 0,
1559 };
1560 quin.parity = NQuin::calculate_parity(
1561 quin.subject,
1562 quin.predicate,
1563 quin.object,
1564 quin.context,
1565 quin.metadata,
1566 );
1567 write_unified_volume(
1568 tmp.path(),
1569 &HashMap::new(),
1570 &[(quin.object, quin.object)],
1571 &[vec![quin]],
1572 )
1573 .unwrap();
1574 let volume = Q42Volume::open(tmp.path()).unwrap();
1575 assert_eq!(
1576 volume.verify_all_blocks().unwrap(),
1577 Q42VerificationReceipt {
1578 blocks_verified: 1,
1579 quins_verified: 1,
1580 }
1581 );
1582 }
1583
1584 #[test]
1585 fn logical_root_verifier_checks_all_manifest_children() {
1586 let dir = tempfile::TempDir::new().unwrap();
1587 let child = dir.path().join("child.q42");
1588 let root = dir.path().join("root.q42");
1589 let mut quin = NQuin {
1590 subject: 101,
1591 predicate: 202,
1592 object: 303,
1593 context: 404,
1594 metadata: 0,
1595 parity: 0,
1596 };
1597 quin.recalculate_parity();
1598 write_unified_volume(
1599 &child,
1600 &HashMap::new(),
1601 &[(quin.object, quin.object)],
1602 &[vec![quin]],
1603 )
1604 .unwrap();
1605 let manifest = Q42VolumeManifest {
1606 generation: 1,
1607 segments: vec![
1608 Q42VolumeManifest::segment_from_file(&child, "child.q42".to_string()).unwrap(),
1609 ],
1610 lexicon_segments: Vec::new(),
1611 };
1612 write_volume_root(&root, &manifest).unwrap();
1613 let set = Q42VolumeSet::open_root(&root).unwrap();
1614 assert_eq!(
1615 set.verify_all(&root).unwrap(),
1616 Q42VerificationReceipt {
1617 blocks_verified: 1,
1618 quins_verified: 1,
1619 }
1620 );
1621 }
1622
1623 #[test]
1624 fn bidx_heavy_hitter_is_complete_and_caller_buffered() {
1625 let (q, lex) = sample_quin("shared", "has", "high-frequency-object");
1626 let blocks = vec![vec![q]; 5];
1627 let ranges = vec![(q.object, q.object); 5];
1628 let tmp = NamedTempFile::new().unwrap();
1629 write_unified_volume(tmp.path(), &lex, &ranges, &blocks).unwrap();
1630
1631 let vol = Q42Volume::open(tmp.path()).unwrap();
1632 assert_eq!(vol.bidx_blocks_for_hash(q.object), vec![0, 1, 2, 3, 4]);
1633
1634 let mut page = [usize::MAX; 2];
1635 let first = vol
1636 .bidx_blocks_for_hash_into(q.object, 0, &mut page)
1637 .unwrap()
1638 .unwrap();
1639 assert_eq!(first.range, BidxBlockRange { start: 0, end: 5 });
1640 assert_eq!(first.returned, 2);
1641 assert_eq!(&page, &[0, 1]);
1642 assert_eq!(first.next_cursor, Some(2));
1643
1644 let second = vol
1645 .bidx_blocks_for_hash_into(q.object, first.next_cursor.unwrap(), &mut page)
1646 .unwrap()
1647 .unwrap();
1648 assert_eq!(&page, &[2, 3]);
1649 assert_eq!(second.next_cursor, Some(4));
1650
1651 let third = vol
1652 .bidx_blocks_for_hash_into(q.object, second.next_cursor.unwrap(), &mut page)
1653 .unwrap()
1654 .unwrap();
1655 assert_eq!(third.returned, 1);
1656 assert_eq!(page[0], 4);
1657 assert_eq!(third.next_cursor, None);
1658 assert!(vol.bidx_blocks_for_hash_into(q.object, 0, &mut []).is_err());
1659 }
1660
1661 #[test]
1662 fn embedded_root_manifest_opens_immutable_child_segments() {
1663 let dir = TempDir::new().unwrap();
1664 let root = dir.path().join("root.q42");
1665 let first_path = dir.path().join("segment-000.q42");
1666 let second_path = dir.path().join("segment-001.q42");
1667 let (first_q, first_lex) = sample_quin("s-1", "p", "o-1");
1668 let (second_q, second_lex) = sample_quin("s-2", "p", "o-2");
1669 let mut inputs = vec![(first_q, first_lex), (second_q, second_lex)];
1670 inputs.sort_unstable_by_key(|(quin, _)| quin.object);
1671 write_unified_volume(
1672 &first_path,
1673 &inputs[0].1,
1674 &[(inputs[0].0.object, inputs[0].0.object)],
1675 &[vec![inputs[0].0]],
1676 )
1677 .unwrap();
1678 write_unified_volume(
1679 &second_path,
1680 &inputs[1].1,
1681 &[(inputs[1].0.object, inputs[1].0.object)],
1682 &[vec![inputs[1].0]],
1683 )
1684 .unwrap();
1685 let manifest = Q42VolumeManifest {
1686 generation: 7,
1687 segments: vec![
1688 Q42VolumeManifest::segment_from_file(&first_path, "segment-000.q42".into())
1689 .unwrap(),
1690 Q42VolumeManifest::segment_from_file(&second_path, "segment-001.q42".into())
1691 .unwrap(),
1692 ],
1693 lexicon_segments: vec![],
1694 };
1695 write_volume_root(&root, &manifest).unwrap();
1696
1697 let root_volume = Q42Volume::open(&root).unwrap();
1698 assert_eq!(root_volume.block_count(), 0);
1699 assert_eq!(
1700 root_volume.volume_manifest().unwrap(),
1701 Some(manifest.clone())
1702 );
1703 let set = Q42VolumeSet::open_root(&root).unwrap();
1704 assert_eq!(set.manifest().generation, 7);
1705 assert_eq!(set.segments().len(), 2);
1706 set.verify_segment_hashes(&root).unwrap();
1707 }
1708
1709 #[test]
1710 fn open_rejects_mismatched_directory_length() {
1711 let (q, lex) = sample_quin("s", "p", "o");
1712 let tmp = NamedTempFile::new().unwrap();
1713 write_unified_volume(tmp.path(), &lex, &[(q.object, q.object)], &[vec![q]]).unwrap();
1714 let mut bytes = std::fs::read(tmp.path()).unwrap();
1715 bytes[48..56].copy_from_slice(&0u64.to_le_bytes());
1716 std::fs::write(tmp.path(), bytes).unwrap();
1717 assert!(Q42Volume::open(tmp.path()).is_err());
1718 }
1719
1720 #[test]
1721 fn decompression_rejects_directory_size_disagreement() {
1722 let (q, lex) = sample_quin("s", "p", "o");
1723 let tmp = NamedTempFile::new().unwrap();
1724 write_unified_volume(tmp.path(), &lex, &[(q.object, q.object)], &[vec![q]]).unwrap();
1725 let mut bytes = std::fs::read(tmp.path()).unwrap();
1726 let data_offset = u64::from_le_bytes(bytes[56..64].try_into().unwrap()) as usize;
1727 bytes[data_offset..data_offset + 4].copy_from_slice(&1u32.to_le_bytes());
1728 std::fs::write(tmp.path(), bytes).unwrap();
1729 let vol = Q42Volume::open(tmp.path()).unwrap();
1730 let mut out = [0u8; SUPERBLOCK_SIZE];
1731 assert!(vol.read_superblock_into(0, &mut out).is_err());
1732 }
1733
1734 #[test]
1739 fn read_all_quins_roundtrip() {
1740 let (q1, mut lex) = sample_quin("Alice", "rdf:type", "values:NaturalPerson");
1741 let (q2, lex2) = sample_quin("Bob", "values:claims", "values:Right");
1742 let (q3, lex3) = sample_quin("AcmeCorp", "rdf:type", "values:CorporatePerson");
1743 lex.extend(lex2);
1744 lex.extend(lex3);
1745
1746 let mut blocks = vec![vec![q1], vec![q2], vec![q3]];
1747 blocks.sort_by_key(|chunk| chunk[0].object);
1748 let ranges: Vec<_> = blocks.iter().map(|c| (c[0].object, c[0].object)).collect();
1749
1750 let tmp = NamedTempFile::new().unwrap();
1751 write_unified_volume(tmp.path(), &lex, &ranges, &blocks).unwrap();
1752
1753 let vol = Q42Volume::open(tmp.path()).unwrap();
1754 let quins = vol.read_all_quins().unwrap();
1755 assert_eq!(
1756 quins.len(),
1757 3,
1758 "read_all_quins must return every stored quin"
1759 );
1760 for q in [q1, q2, q3] {
1761 assert!(
1762 quins.iter().any(|r| r.subject == q.subject
1763 && r.predicate == q.predicate
1764 && r.object == q.object),
1765 "read_all_quins dropped a quin: {q:?}"
1766 );
1767 }
1768 }
1769
1770 #[test]
1774 fn streaming_ingest_embeds_recoverable_lex() {
1775 use crate::external_sort::ExternalSorter;
1776 use crate::lexicon::generate_60bit_token;
1777 use crate::sparql_library::parsers::turtle_doc::parse_turtle_doc_into;
1778
1779 let doc = r#"
1780@prefix dc: <http://purl.org/dc/terms/> .
1781@prefix values: <https://ns.webcivics.net/values/> .
1782@prefix doc: <https://ns.webcivics.net/values/inst#> .
1783doc:article-1 a values:Undertaking ;
1784 dc:title "Article 1" ;
1785 values:originalText "Each Member undertakes to suppress forced labour." .
1786"#;
1787 let tmp_dir = std::env::temp_dir().join(format!("q42_lex_rt_{}", std::process::id()));
1788 let mut sorter = ExternalSorter::new(tmp_dir);
1789 parse_turtle_doc_into(doc.as_bytes(), 0, &mut sorter).unwrap();
1790 let out = NamedTempFile::new().unwrap();
1791 sorter.merge(out.path()).unwrap();
1792
1793 let vol = Q42Volume::open(out.path()).unwrap();
1794 let lex = vol
1795 .lex_view()
1796 .expect("volume must carry an embedded front-of-file Q42LEX section");
1797
1798 let lit = "Each Member undertakes to suppress forced labour.";
1800 assert_eq!(
1801 lex.lookup_owned(generate_60bit_token(lit.as_bytes())),
1802 Some(lit.to_string())
1803 );
1804 let undertaking = "https://ns.webcivics.net/values/Undertaking";
1808 let token = generate_60bit_token(undertaking.as_bytes());
1809 assert_eq!(lex.lookup_owned(token), Some(undertaking.to_string()));
1810 let (ns, local) = lex.lookup_parts(token).expect("namespaced IRI parts");
1811 assert_eq!(ns, "https://ns.webcivics.net/values/");
1812 assert_eq!(local, "Undertaking");
1813 }
1814
1815 #[test]
1818 fn finish_to_bytes_matches_a_readable_on_disk_volume() {
1819 let (q1, mut lex) = sample_quin("Heart", "geo:bodySystem", "circulatory");
1820 let (q2, lex2) = sample_quin("Lung", "geo:bodySystem", "respiratory");
1821 lex.extend(lex2);
1822 let mut blocks = vec![vec![q1], vec![q2]];
1823 blocks.sort_by_key(|c| c[0].object);
1824
1825 let mut builder = UnifiedVolumeBuilder::with_lex_map(&lex).unwrap();
1826 for (seq, chunk) in blocks.iter().enumerate() {
1827 builder.push_block(seq as u64, chunk).unwrap();
1828 }
1829 let bytes = builder.finish_to_bytes();
1830
1831 assert!(bytes.starts_with(&Q42_MAGIC), "bytes are a Q42 volume");
1832 let tmp = NamedTempFile::new().unwrap();
1834 std::fs::write(tmp.path(), &bytes).unwrap();
1835 let vol = Q42Volume::open(tmp.path()).unwrap();
1836 assert_eq!(vol.block_count(), 2);
1837 let quins = vol.read_all_quins().unwrap();
1838 assert_eq!(quins.len(), 2);
1839 let lexv = vol.lex_view().unwrap();
1840 let vals: Vec<String> = quins
1841 .iter()
1842 .filter_map(|q| lexv.lookup_owned(q.object))
1843 .collect();
1844 assert!(vals.iter().any(|v| v == "circulatory") && vals.iter().any(|v| v == "respiratory"));
1845 }
1846
1847 #[test]
1848 fn v2_magic_detected() {
1849 let (q, lex) = sample_quin("a", "b", "c");
1850 let tmp = NamedTempFile::new().unwrap();
1851 write_unified_volume(tmp.path(), &lex, &[(q.object, q.object)], &[vec![q]]).unwrap();
1852 assert!(is_unified_volume(tmp.path()).unwrap());
1853 }
1854}
1855
1856#[cfg(not(target_arch = "wasm32"))]
1859pub struct StreamingVolumeAppender {
1860 file: std::fs::File,
1861 header: Q42VolumeHeader,
1862 block_ranges: Vec<(u64, u64)>,
1863 dir_entries: Vec<BlockDirectoryEntry>,
1864 dag_store: crate::git_bridge::DagStore,
1865 author_did: u64,
1866 last_dag_hash: [u8; 32],
1867 last_object_hash: Option<u64>,
1868}
1869
1870#[cfg(not(target_arch = "wasm32"))]
1871impl StreamingVolumeAppender {
1872 pub fn new(path: &std::path::Path) -> std::io::Result<Self> {
1875 let mut file = std::fs::OpenOptions::new()
1876 .create(true)
1877 .read(true)
1878 .write(true)
1879 .open(path)?;
1880
1881 let header = Q42VolumeHeader {
1882 magic: Q42_MAGIC,
1883 version: Q42_VERSION_V4,
1884 flags: FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED,
1885 lex_offset: HEADER_SIZE as u64,
1886 lex_length: 0,
1887 bidx_offset: HEADER_SIZE as u64,
1888 bidx_length: 0,
1889 block_dir_offset: HEADER_SIZE as u64,
1890 block_dir_length: 0,
1891 data_offset: HEADER_SIZE as u64,
1892 data_length: 0,
1893 block_count: 0,
1894 block_size: SUPERBLOCK_SIZE as u32,
1895 quins_per_block: QUINS_PER_BLOCK as u32,
1896 temporal_index_offset: 0,
1897 temporal_index_length: 0,
1898 merkle_root: [0; 32],
1899 assertion_timestamp: 0,
1900 dag_root_offset: 0,
1901 dag_root_length: 0,
1902 natural_person_did_offset: 0,
1903 software_agent_did_offset: 0,
1904 _reserved: [0; 80],
1905 };
1906
1907 if file.metadata()?.len() != 0 {
1908 return Err(io::Error::new(
1909 io::ErrorKind::Unsupported,
1910 "StreamingVolumeAppender cannot reopen a Q42 volume; publish a new immutable segment",
1911 ));
1912 }
1913 {
1914 use std::io::{Seek, SeekFrom, Write};
1915 file.seek(SeekFrom::Start(0))?;
1916 file.write_all(&header_to_bytes(&header))?;
1917 }
1918
1919 Ok(Self {
1920 file,
1921 header,
1922 block_ranges: Vec::new(),
1923 dir_entries: Vec::new(),
1924 dag_store: crate::git_bridge::DagStore::new(),
1925 author_did: 0,
1926 last_dag_hash: [0u8; 32],
1927 last_object_hash: None,
1928 })
1929 }
1930
1931 pub fn with_author_did(mut self, did: u64) -> Self {
1933 self.author_did = did;
1934 self
1935 }
1936
1937 pub fn append_block(&mut self, seq_id: u64, quins: &[NQuin]) -> std::io::Result<()> {
1941 let (min_hash, max_hash) = object_range(quins)?;
1942 if let Some(previous) = self.last_object_hash {
1943 if min_hash < previous {
1944 return Err(io::Error::new(
1945 io::ErrorKind::InvalidInput,
1946 "Q42 streaming blocks must be globally sorted by object hash",
1947 ));
1948 }
1949 }
1950 self.block_ranges.push((min_hash, max_hash));
1951
1952 let raw = encode_superblock(seq_id, quins);
1953 let compressed = lz4_flex::compress_prepend_size(&raw);
1954
1955 use std::io::{Seek, SeekFrom, Write};
1956 let append_offset = self.header.data_offset + self.header.data_length;
1957 self.file.seek(SeekFrom::Start(append_offset))?;
1958
1959 self.dir_entries.push(BlockDirectoryEntry {
1960 rel_offset: self.header.data_length,
1961 comp_len: compressed.len() as u32,
1962 uncomp_len: SUPERBLOCK_SIZE as u32,
1963 });
1964
1965 self.file.write_all(&compressed)?;
1966 self.header.data_length += compressed.len() as u64;
1967 self.header.block_count += 1;
1968
1969 let ts = std::time::SystemTime::now()
1970 .duration_since(std::time::UNIX_EPOCH)
1971 .map(|d| d.as_millis() as u64)
1972 .unwrap_or(0);
1973 let msg = format!("runtime block {}", seq_id);
1974
1975 self.last_dag_hash = if self.last_dag_hash == [0u8; 32] {
1976 self.dag_store
1977 .genesis_node(quins, self.author_did, ts, &msg)
1978 } else {
1979 self.dag_store
1980 .commit_node(self.last_dag_hash, quins, self.author_did, ts, &msg)
1981 };
1982 self.last_object_hash = Some(max_hash);
1983
1984 let bidx_bytes = encode_bidx(&self.block_ranges);
1986 self.header.bidx_offset = self.header.data_offset + self.header.data_length;
1987 self.header.bidx_length = bidx_bytes.len() as u64;
1988 self.file.write_all(&bidx_bytes)?;
1989
1990 self.header.block_dir_offset = self.header.bidx_offset + self.header.bidx_length;
1991 self.header.block_dir_length = (self.dir_entries.len() * BlockDirectoryEntry::SIZE) as u64;
1992 let mut dir_bytes = Vec::with_capacity(self.header.block_dir_length as usize);
1993 for entry in &self.dir_entries {
1994 dir_bytes.extend_from_slice(&entry.rel_offset.to_le_bytes());
1995 dir_bytes.extend_from_slice(&entry.comp_len.to_le_bytes());
1996 dir_bytes.extend_from_slice(&entry.uncomp_len.to_le_bytes());
1997 }
1998 self.file.write_all(&dir_bytes)?;
1999
2000 let dag_bytes = self.dag_store.serialize();
2002 self.header.dag_root_offset = self.header.block_dir_offset + self.header.block_dir_length;
2003 self.header.dag_root_length = dag_bytes.len() as u64;
2004 self.file.write_all(&dag_bytes)?;
2005
2006 use sha2::{Digest, Sha256};
2007 let mut h = Sha256::new();
2008 h.update(self.last_dag_hash);
2009 self.header.merkle_root = h.finalize().into();
2010 self.header.assertion_timestamp = ts;
2011
2012 self.file.seek(SeekFrom::Start(0))?;
2014 self.file.write_all(&header_to_bytes(&self.header))?;
2015 self.file.sync_all()?;
2016
2017 Ok(())
2018 }
2019}