qualia_core_db/q42/volume/
stream_writer.rs1use std::collections::HashMap;
4use std::fs::{File, OpenOptions};
5use std::io::{self, BufWriter, Read, Write};
6use std::path::{Path, PathBuf};
7
8use tempfile::TempDir;
9
10use super::super::{
11 encode_lex, encode_superblock, header_to_bytes, BlockDirectoryEntry, Q42VolumeHeader,
12 FLAG_BLOCKS_LZ4, FLAG_FIELD_POSTINGS, FLAG_FIELD_RANGES, FLAG_OBJECT_SORTED,
13 FLAG_PERMISSIVE_COMMONS, FLAG_SANCTUARY, HEADER_SIZE, Q42_VERSION_V4, QUINS_PER_BLOCK,
14 SUPERBLOCK_SIZE,
15};
16use super::payload_catalogue::{
17 encode_payload_catalogue_section, encode_payload_catalogue_with_bodies,
18 set_header_payload_catalogue_range, PayloadCatalogueBlob,
19};
20use super::postings::{encode_block_postings, BlockFieldPostings, FIELD_POSTINGS_MAGIC};
21use super::publication::quin_requires_sanctuary;
22use crate::bundle::cbor_ld_manifest::PayloadCatalogueEntry;
23use crate::NQuin;
24
25fn invalid(message: impl Into<String>) -> io::Error {
26 io::Error::new(io::ErrorKind::InvalidInput, message.into())
27}
28
29pub struct StreamingQ42VolumeWriter {
32 _temp: TempDir,
33 bidx_path: PathBuf,
34 field_ranges_path: PathBuf,
35 postings_path: PathBuf,
36 directory_path: PathBuf,
37 data_path: PathBuf,
38 bidx: BufWriter<File>,
39 field_ranges: BufWriter<File>,
40 postings: BufWriter<File>,
41 directory: BufWriter<File>,
42 data: BufWriter<File>,
43 posting_offsets: Vec<u32>,
44 lex_bytes: Vec<u8>,
45 block_count: u64,
46 data_length: u64,
47 last_object_hash: Option<u64>,
48 publication_commons: bool,
49 publication_sanctuary: bool,
50 sanctuary_quin_count: u64,
51 merkle_root: [u8; 32],
52 payload_catalogue_blobs: Vec<PayloadCatalogueBlob>,
53 payload_catalogue_bytes: Option<Vec<u8>>,
54}
55
56impl StreamingQ42VolumeWriter {
57 pub fn new(lexicon: &HashMap<u64, String>) -> io::Result<Self> {
59 let temp = TempDir::new()?;
60 let bidx_path = temp.path().join("bidx.entries");
61 let field_ranges_path = temp.path().join("field-ranges.entries");
62 let postings_path = temp.path().join("field.postings");
63 let directory_path = temp.path().join("block.directory");
64 let data_path = temp.path().join("blocks.lz4");
65 let open = |path: &Path| OpenOptions::new().create_new(true).write(true).open(path);
66 Ok(Self {
67 bidx: BufWriter::new(open(&bidx_path)?),
68 field_ranges: BufWriter::new(open(&field_ranges_path)?),
69 postings: BufWriter::new(open(&postings_path)?),
70 directory: BufWriter::new(open(&directory_path)?),
71 data: BufWriter::new(open(&data_path)?),
72 _temp: temp,
73 bidx_path,
74 field_ranges_path,
75 postings_path,
76 directory_path,
77 data_path,
78 posting_offsets: vec![0],
79 lex_bytes: encode_lex(lexicon)
80 .map_err(|error| invalid(format!("invalid Q42LEX: {error:?}")))?,
81 block_count: 0,
82 data_length: 0,
83 last_object_hash: None,
84 publication_commons: false,
85 publication_sanctuary: false,
86 sanctuary_quin_count: 0,
87 merkle_root: [0; 32],
88 payload_catalogue_blobs: Vec::new(),
89 payload_catalogue_bytes: None,
90 })
91 }
92
93 pub fn add_payload_catalogue_utf8(
96 &mut self,
97 payload_key: &str,
98 mime_type: &str,
99 language: Option<&str>,
100 role: &str,
101 utf8_body: &str,
102 ) -> io::Result<PayloadCatalogueEntry> {
103 let blob = PayloadCatalogueBlob::new_utf8(
104 payload_key,
105 mime_type,
106 language.map(str::to_string),
107 role,
108 utf8_body,
109 )
110 .map_err(invalid)?;
111 let entry = blob.entry.clone();
112 self.payload_catalogue_blobs.push(blob);
113 let encoded = encode_payload_catalogue_with_bodies(&self.payload_catalogue_blobs)
114 .map_err(invalid)?;
115 self.payload_catalogue_bytes = Some(encoded);
116 Ok(entry)
117 }
118
119 pub fn set_payload_catalogue_blobs(&mut self, blobs: &[PayloadCatalogueBlob]) -> io::Result<()> {
121 if blobs.is_empty() {
122 self.payload_catalogue_blobs.clear();
123 self.payload_catalogue_bytes = None;
124 return Ok(());
125 }
126 let encoded = encode_payload_catalogue_with_bodies(blobs).map_err(invalid)?;
127 self.payload_catalogue_blobs = blobs.to_vec();
128 self.payload_catalogue_bytes = Some(encoded);
129 Ok(())
130 }
131
132 pub fn set_payload_catalogue_entries(
134 &mut self,
135 entries: &[PayloadCatalogueEntry],
136 ) -> io::Result<()> {
137 if entries.is_empty() {
138 self.payload_catalogue_blobs.clear();
139 self.payload_catalogue_bytes = None;
140 return Ok(());
141 }
142 let encoded = encode_payload_catalogue_section(entries).map_err(invalid)?;
143 self.payload_catalogue_blobs.clear();
144 self.payload_catalogue_bytes = Some(encoded);
145 Ok(())
146 }
147
148 pub fn declare_permissive_commons(&mut self) {
151 self.publication_commons = true;
152 }
153
154 pub fn declare_sanctuary(&mut self) {
156 self.publication_sanctuary = true;
157 }
158
159 pub fn push_block(&mut self, seq_id: u64, quins: &[NQuin]) -> io::Result<()> {
161 let Some(first) = quins.first() else {
162 return Err(invalid("Q42 SuperBlock must not be empty"));
163 };
164 if quins.len() > QUINS_PER_BLOCK {
165 return Err(invalid("Q42 SuperBlock exceeds Quin capacity"));
166 }
167 let mut previous = first.object;
168 for quin in &quins[1..] {
169 if quin.object < previous {
170 return Err(invalid("Q42 SuperBlock is not object-sorted"));
171 }
172 previous = quin.object;
173 }
174 if self
175 .last_object_hash
176 .is_some_and(|last| first.object < last)
177 {
178 return Err(invalid("Q42 blocks are not globally object-sorted"));
179 }
180 let compressed = lz4_flex::compress_prepend_size(&encode_superblock(seq_id, quins));
181 let compressed_len = u32::try_from(compressed.len())
182 .map_err(|_| invalid("compressed Q42 block exceeds u32"))?;
183 self.bidx.write_all(&first.object.to_le_bytes())?;
184 self.bidx.write_all(&previous.to_le_bytes())?;
185 let mut subject_min = first.subject;
186 let mut subject_max = first.subject;
187 let mut predicate_min = first.predicate;
188 let mut predicate_max = first.predicate;
189 let mut context_min = first.context;
190 let mut context_max = first.context;
191 for quin in &quins[1..] {
192 subject_min = subject_min.min(quin.subject);
193 subject_max = subject_max.max(quin.subject);
194 predicate_min = predicate_min.min(quin.predicate);
195 predicate_max = predicate_max.max(quin.predicate);
196 context_min = context_min.min(quin.context);
197 context_max = context_max.max(quin.context);
198 }
199 for value in [
200 subject_min,
201 subject_max,
202 predicate_min,
203 predicate_max,
204 context_min,
205 context_max,
206 ] {
207 self.field_ranges.write_all(&value.to_le_bytes())?;
208 }
209 let encoded = encode_block_postings(&BlockFieldPostings::from_quins(quins));
210 self.postings.write_all(&encoded)?;
211 let next = self
212 .posting_offsets
213 .last()
214 .copied()
215 .unwrap_or(0)
216 .checked_add(encoded.len() as u32)
217 .ok_or_else(|| invalid("Q42 field postings overflow u32"))?;
218 self.posting_offsets.push(next);
219 BlockDirectoryEntry {
220 rel_offset: self.data_length,
221 comp_len: compressed_len,
222 uncomp_len: SUPERBLOCK_SIZE as u32,
223 }
224 .write_to(&mut self.directory)?;
225 self.data.write_all(&compressed)?;
226 self.data_length = self
227 .data_length
228 .checked_add(compressed.len() as u64)
229 .ok_or_else(|| invalid("Q42 data length overflow"))?;
230 self.block_count += 1;
231 self.last_object_hash = Some(previous);
232 self.sanctuary_quin_count += quins
233 .iter()
234 .filter(|quin| quin_requires_sanctuary(quin))
235 .count() as u64;
236 {
237 use sha2::{Digest, Sha256};
238 let mut hasher = Sha256::new();
239 hasher.update(self.merkle_root);
240 hasher.update(&compressed);
241 self.merkle_root = hasher.finalize().into();
242 }
243 Ok(())
244 }
245
246 pub fn block_count(&self) -> u64 {
248 self.block_count
249 }
250
251 pub fn estimated_final_length(&self) -> io::Result<u64> {
254 let bidx_length = 16u64
255 .checked_add(
256 self.block_count
257 .checked_mul(16)
258 .ok_or_else(|| invalid("Q42 BIDX length overflow"))?,
259 )
260 .ok_or_else(|| invalid("Q42 BIDX length overflow"))?;
261 let field_ranges_length = 16u64
262 .checked_add(
263 self.block_count
264 .checked_mul(48)
265 .ok_or_else(|| invalid("Q42 field-range index length overflow"))?,
266 )
267 .ok_or_else(|| invalid("Q42 field-range index length overflow"))?;
268 let postings_length = self.encoded_postings_section_len()?;
269 let directory_length = self
270 .block_count
271 .checked_mul(BlockDirectoryEntry::SIZE as u64)
272 .ok_or_else(|| invalid("Q42 directory length overflow"))?;
273 let pcat_length = self
274 .payload_catalogue_bytes
275 .as_ref()
276 .map_or(0u64, |b| b.len() as u64);
277 (HEADER_SIZE as u64)
278 .checked_add(self.lex_bytes.len() as u64)
279 .and_then(|value| value.checked_add(bidx_length))
280 .and_then(|value| value.checked_add(field_ranges_length))
281 .and_then(|value| value.checked_add(postings_length))
282 .and_then(|value| value.checked_add(directory_length))
283 .and_then(|value| value.checked_add(self.data_length))
284 .and_then(|value| value.checked_add(pcat_length))
285 .ok_or_else(|| invalid("Q42 final length overflow"))
286 }
287
288 pub fn maximum_final_length_after_next_block(&self) -> io::Result<u64> {
292 self.estimated_final_length()?
293 .checked_add(32) .and_then(|value| {
295 value.checked_add(crate::q42_volume::MAX_COMPRESSED_SUPERBLOCK_SIZE as u64)
296 })
297 .ok_or_else(|| invalid("Q42 final length overflow"))
298 }
299
300 fn encoded_postings_section_len(&self) -> io::Result<u64> {
301 let table = (self.block_count + 1)
302 .checked_mul(4)
303 .ok_or_else(|| invalid("Q42 postings table overflow"))?;
304 let payload = u64::from(*self.posting_offsets.last().unwrap_or(&0));
305 16u64
306 .checked_add(table)
307 .and_then(|value| value.checked_add(payload))
308 .ok_or_else(|| invalid("Q42 postings section overflow"))
309 }
310
311 pub fn finish(mut self, path: &Path) -> io::Result<()> {
313 self.bidx.flush()?;
314 self.field_ranges.flush()?;
315 self.postings.flush()?;
316 self.directory.flush()?;
317 self.data.flush()?;
318 let bidx_length = 16
319 + self
320 .block_count
321 .checked_mul(16)
322 .ok_or_else(|| invalid("Q42 BIDX length overflow"))?;
323 let directory_length = self
324 .block_count
325 .checked_mul(BlockDirectoryEntry::SIZE as u64)
326 .ok_or_else(|| invalid("Q42 directory length overflow"))?;
327 let field_ranges_length = 16
328 + self
329 .block_count
330 .checked_mul(48)
331 .ok_or_else(|| invalid("Q42 field-range index length overflow"))?;
332 let postings_length = self.encoded_postings_section_len()?;
333 let lex_offset = HEADER_SIZE as u64;
334 let bidx_offset = lex_offset + self.lex_bytes.len() as u64;
335 let field_ranges_offset = bidx_offset + bidx_length;
336 let postings_offset = field_ranges_offset + field_ranges_length;
337 let directory_offset = postings_offset + postings_length;
338 let data_offset = directory_offset + directory_length;
339 let mut flags =
340 FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED | FLAG_FIELD_RANGES | FLAG_FIELD_POSTINGS;
341 if self.publication_sanctuary || self.sanctuary_quin_count > 0 {
342 flags |= FLAG_SANCTUARY;
343 } else if self.publication_commons {
344 flags |= FLAG_PERMISSIVE_COMMONS;
345 }
346 let mut header = Q42VolumeHeader {
347 magic: super::super::Q42_MAGIC,
348 version: Q42_VERSION_V4,
349 flags,
350 lex_offset,
351 lex_length: self.lex_bytes.len() as u64,
352 bidx_offset,
353 bidx_length,
354 block_dir_offset: directory_offset,
355 block_dir_length: directory_length,
356 data_offset,
357 data_length: self.data_length,
358 block_count: self.block_count,
359 block_size: SUPERBLOCK_SIZE as u32,
360 quins_per_block: QUINS_PER_BLOCK as u32,
361 temporal_index_offset: 0,
362 temporal_index_length: 0,
363 merkle_root: self.merkle_root,
364 assertion_timestamp: 0,
365 dag_root_offset: 0,
366 dag_root_length: 0,
367 natural_person_did_offset: 0,
368 software_agent_did_offset: 0,
369 _reserved: {
370 let mut reserved = [0; 80];
371 reserved[16..24].copy_from_slice(&field_ranges_offset.to_le_bytes());
372 reserved[24..32].copy_from_slice(&(field_ranges_length as u64).to_le_bytes());
373 reserved[32..40].copy_from_slice(&postings_offset.to_le_bytes());
374 reserved[40..48].copy_from_slice(&postings_length.to_le_bytes());
375 reserved
376 },
377 };
378 if let Some(ref pcat_bytes) = self.payload_catalogue_bytes {
379 if !pcat_bytes.is_empty() {
380 let pcat_offset = data_offset
381 .checked_add(self.data_length)
382 .ok_or_else(|| invalid("Q42 payload catalogue offset overflow"))?;
383 set_header_payload_catalogue_range(
384 &mut header,
385 pcat_offset,
386 pcat_bytes.len() as u64,
387 );
388 }
389 }
390 let mut output = BufWriter::new(
391 OpenOptions::new()
392 .create(true)
393 .write(true)
394 .truncate(true)
395 .open(path)?,
396 );
397 output.write_all(&header_to_bytes(&header))?;
398 output.write_all(&self.lex_bytes)?;
399 output.write_all(&super::super::BIDX_MAGIC)?;
400 output.write_all(&1u32.to_le_bytes())?;
401 output.write_all(&(self.block_count as u32).to_le_bytes())?;
402 output.write_all(&0u32.to_le_bytes())?;
403 copy_file(&self.bidx_path, &mut output)?;
404 output.write_all(&super::super::FIELD_RANGE_INDEX_MAGIC)?;
405 output.write_all(&1u32.to_le_bytes())?;
406 output.write_all(&(self.block_count as u32).to_le_bytes())?;
407 output.write_all(&0u32.to_le_bytes())?;
408 copy_file(&self.field_ranges_path, &mut output)?;
409 output.write_all(&FIELD_POSTINGS_MAGIC)?;
410 output.write_all(&1u32.to_le_bytes())?;
411 output.write_all(&(self.block_count as u32).to_le_bytes())?;
412 output.write_all(&0u32.to_le_bytes())?;
413 for offset in &self.posting_offsets {
414 output.write_all(&offset.to_le_bytes())?;
415 }
416 copy_file(&self.postings_path, &mut output)?;
417 copy_file(&self.directory_path, &mut output)?;
418 copy_file(&self.data_path, &mut output)?;
419 if let Some(ref pcat_bytes) = self.payload_catalogue_bytes {
420 if !pcat_bytes.is_empty() {
421 output.write_all(pcat_bytes)?;
422 }
423 }
424 output.flush()
425 }
426}
427
428fn copy_file(path: &Path, output: &mut BufWriter<File>) -> io::Result<()> {
429 let mut input = File::open(path)?;
430 let mut buffer = [0u8; 64 * 1024];
431 loop {
432 let count = input.read(&mut buffer)?;
433 if count == 0 {
434 return Ok(());
435 }
436 output.write_all(&buffer[..count])?;
437 }
438}
439
440#[cfg(test)]
441mod tests {
442 use super::*;
443 use crate::q42_volume::Q42Volume;
444 use tempfile::NamedTempFile;
445
446 #[test]
447 fn streams_a_readable_volume_without_payload_accumulation() {
448 let mut lex = HashMap::new();
449 lex.insert(1, "s".to_string());
450 lex.insert(2, "p".to_string());
451 lex.insert(3, "o".to_string());
452 let mut writer = StreamingQ42VolumeWriter::new(&lex).unwrap();
453 writer
454 .push_block(
455 0,
456 &[NQuin {
457 subject: 1,
458 predicate: 2,
459 object: 3,
460 context: 0,
461 metadata: 0,
462 parity: 0,
463 }],
464 )
465 .unwrap();
466 let output = NamedTempFile::new().unwrap();
467 writer.finish(output.path()).unwrap();
468 let volume = Q42Volume::open(output.path()).unwrap();
469 assert_eq!(volume.block_count(), 1);
470 assert_eq!(volume.object_hash_bounds(), Some((3, 3)));
471 }
472}