Skip to main content

qualia_core_db/q42/volume/
stream_writer.rs

1//! Cold, bounded-memory unified-Q42 writer for large sorted streams.
2
3use std::collections::HashMap;
4use std::fs::{File, OpenOptions};
5use std::io::{self, BufWriter, Read, Write};
6use std::path::{Path, PathBuf};
7
8use tempfile::TempDir;
9
10use super::super::{
11    encode_lex, encode_superblock, header_to_bytes, BlockDirectoryEntry, Q42VolumeHeader,
12    FLAG_BLOCKS_LZ4, FLAG_FIELD_POSTINGS, FLAG_FIELD_RANGES, FLAG_OBJECT_SORTED,
13    FLAG_PERMISSIVE_COMMONS, FLAG_SANCTUARY, HEADER_SIZE, Q42_VERSION_V4, QUINS_PER_BLOCK,
14    SUPERBLOCK_SIZE,
15};
16use super::payload_catalogue::{
17    encode_payload_catalogue_section, encode_payload_catalogue_with_bodies,
18    set_header_payload_catalogue_range, PayloadCatalogueBlob,
19};
20use super::postings::{encode_block_postings, BlockFieldPostings, FIELD_POSTINGS_MAGIC};
21use super::publication::quin_requires_sanctuary;
22use crate::bundle::cbor_ld_manifest::PayloadCatalogueEntry;
23use crate::NQuin;
24
25fn invalid(message: impl Into<String>) -> io::Error {
26    io::Error::new(io::ErrorKind::InvalidInput, message.into())
27}
28
29/// Writes a sorted Q42 segment without retaining its compressed payload or
30/// per-block metadata in heap collections. Temporary streams are RAII-owned.
31pub struct StreamingQ42VolumeWriter {
32    _temp: TempDir,
33    bidx_path: PathBuf,
34    field_ranges_path: PathBuf,
35    postings_path: PathBuf,
36    directory_path: PathBuf,
37    data_path: PathBuf,
38    bidx: BufWriter<File>,
39    field_ranges: BufWriter<File>,
40    postings: BufWriter<File>,
41    directory: BufWriter<File>,
42    data: BufWriter<File>,
43    posting_offsets: Vec<u32>,
44    lex_bytes: Vec<u8>,
45    block_count: u64,
46    data_length: u64,
47    last_object_hash: Option<u64>,
48    publication_commons: bool,
49    publication_sanctuary: bool,
50    sanctuary_quin_count: u64,
51    merkle_root: [u8; 32],
52    payload_catalogue_blobs: Vec<PayloadCatalogueBlob>,
53    payload_catalogue_bytes: Option<Vec<u8>>,
54}
55
56impl StreamingQ42VolumeWriter {
57    /// Creates a default `StreamingQ42VolumeWriter`.
58    pub fn new(lexicon: &HashMap<u64, String>) -> io::Result<Self> {
59        let temp = TempDir::new()?;
60        let bidx_path = temp.path().join("bidx.entries");
61        let field_ranges_path = temp.path().join("field-ranges.entries");
62        let postings_path = temp.path().join("field.postings");
63        let directory_path = temp.path().join("block.directory");
64        let data_path = temp.path().join("blocks.lz4");
65        let open = |path: &Path| OpenOptions::new().create_new(true).write(true).open(path);
66        Ok(Self {
67            bidx: BufWriter::new(open(&bidx_path)?),
68            field_ranges: BufWriter::new(open(&field_ranges_path)?),
69            postings: BufWriter::new(open(&postings_path)?),
70            directory: BufWriter::new(open(&directory_path)?),
71            data: BufWriter::new(open(&data_path)?),
72            _temp: temp,
73            bidx_path,
74            field_ranges_path,
75            postings_path,
76            directory_path,
77            data_path,
78            posting_offsets: vec![0],
79            lex_bytes: encode_lex(lexicon)
80                .map_err(|error| invalid(format!("invalid Q42LEX: {error:?}")))?,
81            block_count: 0,
82            data_length: 0,
83            last_object_hash: None,
84            publication_commons: false,
85            publication_sanctuary: false,
86            sanctuary_quin_count: 0,
87            merkle_root: [0; 32],
88            payload_catalogue_blobs: Vec::new(),
89            payload_catalogue_bytes: None,
90        })
91    }
92
93    /// Append a UTF-8 prose/descriptor payload (`ADR 0014`) to the `PCAT` section of this volume.
94    /// Rejects media MIME types (`image/*`, `audio/*`, `video/*`, `.p64`, `.10d`).
95    pub fn add_payload_catalogue_utf8(
96        &mut self,
97        payload_key: &str,
98        mime_type: &str,
99        language: Option<&str>,
100        role: &str,
101        utf8_body: &str,
102    ) -> io::Result<PayloadCatalogueEntry> {
103        let blob = PayloadCatalogueBlob::new_utf8(
104            payload_key,
105            mime_type,
106            language.map(str::to_string),
107            role,
108            utf8_body,
109        )
110        .map_err(invalid)?;
111        let entry = blob.entry.clone();
112        self.payload_catalogue_blobs.push(blob);
113        let encoded = encode_payload_catalogue_with_bodies(&self.payload_catalogue_blobs)
114            .map_err(invalid)?;
115        self.payload_catalogue_bytes = Some(encoded);
116        Ok(entry)
117    }
118
119    /// Set the `PCAT` v1 payload catalogue from pre-constructed [`PayloadCatalogueBlob`] records.
120    pub fn set_payload_catalogue_blobs(&mut self, blobs: &[PayloadCatalogueBlob]) -> io::Result<()> {
121        if blobs.is_empty() {
122            self.payload_catalogue_blobs.clear();
123            self.payload_catalogue_bytes = None;
124            return Ok(());
125        }
126        let encoded = encode_payload_catalogue_with_bodies(blobs).map_err(invalid)?;
127        self.payload_catalogue_blobs = blobs.to_vec();
128        self.payload_catalogue_bytes = Some(encoded);
129        Ok(())
130    }
131
132    /// Set the `PCAT` v1 payload catalogue from metadata-only [`PayloadCatalogueEntry`] descriptors.
133    pub fn set_payload_catalogue_entries(
134        &mut self,
135        entries: &[PayloadCatalogueEntry],
136    ) -> io::Result<()> {
137        if entries.is_empty() {
138            self.payload_catalogue_blobs.clear();
139            self.payload_catalogue_bytes = None;
140            return Ok(());
141        }
142        let encoded = encode_payload_catalogue_section(entries).map_err(invalid)?;
143        self.payload_catalogue_blobs.clear();
144        self.payload_catalogue_bytes = Some(encoded);
145        Ok(())
146    }
147
148    /// Mark this segment as a Permissive Commons catalog. Ignored if any
149    /// pushed Quin requires Sanctuary.
150    pub fn declare_permissive_commons(&mut self) {
151        self.publication_commons = true;
152    }
153
154    /// Mark this segment as Sanctuary / Selfhood. Public hash addressing is denied.
155    pub fn declare_sanctuary(&mut self) {
156        self.publication_sanctuary = true;
157    }
158
159    /// `StreamingQ42VolumeWriter::push_block`.
160    pub fn push_block(&mut self, seq_id: u64, quins: &[NQuin]) -> io::Result<()> {
161        let Some(first) = quins.first() else {
162            return Err(invalid("Q42 SuperBlock must not be empty"));
163        };
164        if quins.len() > QUINS_PER_BLOCK {
165            return Err(invalid("Q42 SuperBlock exceeds Quin capacity"));
166        }
167        let mut previous = first.object;
168        for quin in &quins[1..] {
169            if quin.object < previous {
170                return Err(invalid("Q42 SuperBlock is not object-sorted"));
171            }
172            previous = quin.object;
173        }
174        if self
175            .last_object_hash
176            .is_some_and(|last| first.object < last)
177        {
178            return Err(invalid("Q42 blocks are not globally object-sorted"));
179        }
180        let compressed = lz4_flex::compress_prepend_size(&encode_superblock(seq_id, quins));
181        let compressed_len = u32::try_from(compressed.len())
182            .map_err(|_| invalid("compressed Q42 block exceeds u32"))?;
183        self.bidx.write_all(&first.object.to_le_bytes())?;
184        self.bidx.write_all(&previous.to_le_bytes())?;
185        let mut subject_min = first.subject;
186        let mut subject_max = first.subject;
187        let mut predicate_min = first.predicate;
188        let mut predicate_max = first.predicate;
189        let mut context_min = first.context;
190        let mut context_max = first.context;
191        for quin in &quins[1..] {
192            subject_min = subject_min.min(quin.subject);
193            subject_max = subject_max.max(quin.subject);
194            predicate_min = predicate_min.min(quin.predicate);
195            predicate_max = predicate_max.max(quin.predicate);
196            context_min = context_min.min(quin.context);
197            context_max = context_max.max(quin.context);
198        }
199        for value in [
200            subject_min,
201            subject_max,
202            predicate_min,
203            predicate_max,
204            context_min,
205            context_max,
206        ] {
207            self.field_ranges.write_all(&value.to_le_bytes())?;
208        }
209        let encoded = encode_block_postings(&BlockFieldPostings::from_quins(quins));
210        self.postings.write_all(&encoded)?;
211        let next = self
212            .posting_offsets
213            .last()
214            .copied()
215            .unwrap_or(0)
216            .checked_add(encoded.len() as u32)
217            .ok_or_else(|| invalid("Q42 field postings overflow u32"))?;
218        self.posting_offsets.push(next);
219        BlockDirectoryEntry {
220            rel_offset: self.data_length,
221            comp_len: compressed_len,
222            uncomp_len: SUPERBLOCK_SIZE as u32,
223        }
224        .write_to(&mut self.directory)?;
225        self.data.write_all(&compressed)?;
226        self.data_length = self
227            .data_length
228            .checked_add(compressed.len() as u64)
229            .ok_or_else(|| invalid("Q42 data length overflow"))?;
230        self.block_count += 1;
231        self.last_object_hash = Some(previous);
232        self.sanctuary_quin_count += quins
233            .iter()
234            .filter(|quin| quin_requires_sanctuary(quin))
235            .count() as u64;
236        {
237            use sha2::{Digest, Sha256};
238            let mut hasher = Sha256::new();
239            hasher.update(self.merkle_root);
240            hasher.update(&compressed);
241            self.merkle_root = hasher.finalize().into();
242        }
243        Ok(())
244    }
245
246    /// `block` count.
247    pub fn block_count(&self) -> u64 {
248        self.block_count
249    }
250
251    /// The exact final length if the writer were finished now. It includes the
252    /// front matter and fixed BIDX/directory records, not just compressed data.
253    pub fn estimated_final_length(&self) -> io::Result<u64> {
254        let bidx_length = 16u64
255            .checked_add(
256                self.block_count
257                    .checked_mul(16)
258                    .ok_or_else(|| invalid("Q42 BIDX length overflow"))?,
259            )
260            .ok_or_else(|| invalid("Q42 BIDX length overflow"))?;
261        let field_ranges_length = 16u64
262            .checked_add(
263                self.block_count
264                    .checked_mul(48)
265                    .ok_or_else(|| invalid("Q42 field-range index length overflow"))?,
266            )
267            .ok_or_else(|| invalid("Q42 field-range index length overflow"))?;
268        let postings_length = self.encoded_postings_section_len()?;
269        let directory_length = self
270            .block_count
271            .checked_mul(BlockDirectoryEntry::SIZE as u64)
272            .ok_or_else(|| invalid("Q42 directory length overflow"))?;
273        let pcat_length = self
274            .payload_catalogue_bytes
275            .as_ref()
276            .map_or(0u64, |b| b.len() as u64);
277        (HEADER_SIZE as u64)
278            .checked_add(self.lex_bytes.len() as u64)
279            .and_then(|value| value.checked_add(bidx_length))
280            .and_then(|value| value.checked_add(field_ranges_length))
281            .and_then(|value| value.checked_add(postings_length))
282            .and_then(|value| value.checked_add(directory_length))
283            .and_then(|value| value.checked_add(self.data_length))
284            .and_then(|value| value.checked_add(pcat_length))
285            .ok_or_else(|| invalid("Q42 final length overflow"))
286    }
287
288    /// A safe upper bound for the final length after one additional block.
289    /// It lets volume publishers split before a block crosses their byte cap
290    /// without retaining compressed payloads.
291    pub fn maximum_final_length_after_next_block(&self) -> io::Result<u64> {
292        self.estimated_final_length()?
293            .checked_add(32) // one BIDX interval plus one directory entry
294            .and_then(|value| {
295                value.checked_add(crate::q42_volume::MAX_COMPRESSED_SUPERBLOCK_SIZE as u64)
296            })
297            .ok_or_else(|| invalid("Q42 final length overflow"))
298    }
299
300    fn encoded_postings_section_len(&self) -> io::Result<u64> {
301        let table = (self.block_count + 1)
302            .checked_mul(4)
303            .ok_or_else(|| invalid("Q42 postings table overflow"))?;
304        let payload = u64::from(*self.posting_offsets.last().unwrap_or(&0));
305        16u64
306            .checked_add(table)
307            .and_then(|value| value.checked_add(payload))
308            .ok_or_else(|| invalid("Q42 postings section overflow"))
309    }
310
311    /// Flushes and finishes the stream.
312    pub fn finish(mut self, path: &Path) -> io::Result<()> {
313        self.bidx.flush()?;
314        self.field_ranges.flush()?;
315        self.postings.flush()?;
316        self.directory.flush()?;
317        self.data.flush()?;
318        let bidx_length = 16
319            + self
320                .block_count
321                .checked_mul(16)
322                .ok_or_else(|| invalid("Q42 BIDX length overflow"))?;
323        let directory_length = self
324            .block_count
325            .checked_mul(BlockDirectoryEntry::SIZE as u64)
326            .ok_or_else(|| invalid("Q42 directory length overflow"))?;
327        let field_ranges_length = 16
328            + self
329                .block_count
330                .checked_mul(48)
331                .ok_or_else(|| invalid("Q42 field-range index length overflow"))?;
332        let postings_length = self.encoded_postings_section_len()?;
333        let lex_offset = HEADER_SIZE as u64;
334        let bidx_offset = lex_offset + self.lex_bytes.len() as u64;
335        let field_ranges_offset = bidx_offset + bidx_length;
336        let postings_offset = field_ranges_offset + field_ranges_length;
337        let directory_offset = postings_offset + postings_length;
338        let data_offset = directory_offset + directory_length;
339        let mut flags =
340            FLAG_BLOCKS_LZ4 | FLAG_OBJECT_SORTED | FLAG_FIELD_RANGES | FLAG_FIELD_POSTINGS;
341        if self.publication_sanctuary || self.sanctuary_quin_count > 0 {
342            flags |= FLAG_SANCTUARY;
343        } else if self.publication_commons {
344            flags |= FLAG_PERMISSIVE_COMMONS;
345        }
346        let mut header = Q42VolumeHeader {
347            magic: super::super::Q42_MAGIC,
348            version: Q42_VERSION_V4,
349            flags,
350            lex_offset,
351            lex_length: self.lex_bytes.len() as u64,
352            bidx_offset,
353            bidx_length,
354            block_dir_offset: directory_offset,
355            block_dir_length: directory_length,
356            data_offset,
357            data_length: self.data_length,
358            block_count: self.block_count,
359            block_size: SUPERBLOCK_SIZE as u32,
360            quins_per_block: QUINS_PER_BLOCK as u32,
361            temporal_index_offset: 0,
362            temporal_index_length: 0,
363            merkle_root: self.merkle_root,
364            assertion_timestamp: 0,
365            dag_root_offset: 0,
366            dag_root_length: 0,
367            natural_person_did_offset: 0,
368            software_agent_did_offset: 0,
369            _reserved: {
370                let mut reserved = [0; 80];
371                reserved[16..24].copy_from_slice(&field_ranges_offset.to_le_bytes());
372                reserved[24..32].copy_from_slice(&(field_ranges_length as u64).to_le_bytes());
373                reserved[32..40].copy_from_slice(&postings_offset.to_le_bytes());
374                reserved[40..48].copy_from_slice(&postings_length.to_le_bytes());
375                reserved
376            },
377        };
378        if let Some(ref pcat_bytes) = self.payload_catalogue_bytes {
379            if !pcat_bytes.is_empty() {
380                let pcat_offset = data_offset
381                    .checked_add(self.data_length)
382                    .ok_or_else(|| invalid("Q42 payload catalogue offset overflow"))?;
383                set_header_payload_catalogue_range(
384                    &mut header,
385                    pcat_offset,
386                    pcat_bytes.len() as u64,
387                );
388            }
389        }
390        let mut output = BufWriter::new(
391            OpenOptions::new()
392                .create(true)
393                .write(true)
394                .truncate(true)
395                .open(path)?,
396        );
397        output.write_all(&header_to_bytes(&header))?;
398        output.write_all(&self.lex_bytes)?;
399        output.write_all(&super::super::BIDX_MAGIC)?;
400        output.write_all(&1u32.to_le_bytes())?;
401        output.write_all(&(self.block_count as u32).to_le_bytes())?;
402        output.write_all(&0u32.to_le_bytes())?;
403        copy_file(&self.bidx_path, &mut output)?;
404        output.write_all(&super::super::FIELD_RANGE_INDEX_MAGIC)?;
405        output.write_all(&1u32.to_le_bytes())?;
406        output.write_all(&(self.block_count as u32).to_le_bytes())?;
407        output.write_all(&0u32.to_le_bytes())?;
408        copy_file(&self.field_ranges_path, &mut output)?;
409        output.write_all(&FIELD_POSTINGS_MAGIC)?;
410        output.write_all(&1u32.to_le_bytes())?;
411        output.write_all(&(self.block_count as u32).to_le_bytes())?;
412        output.write_all(&0u32.to_le_bytes())?;
413        for offset in &self.posting_offsets {
414            output.write_all(&offset.to_le_bytes())?;
415        }
416        copy_file(&self.postings_path, &mut output)?;
417        copy_file(&self.directory_path, &mut output)?;
418        copy_file(&self.data_path, &mut output)?;
419        if let Some(ref pcat_bytes) = self.payload_catalogue_bytes {
420            if !pcat_bytes.is_empty() {
421                output.write_all(pcat_bytes)?;
422            }
423        }
424        output.flush()
425    }
426}
427
428fn copy_file(path: &Path, output: &mut BufWriter<File>) -> io::Result<()> {
429    let mut input = File::open(path)?;
430    let mut buffer = [0u8; 64 * 1024];
431    loop {
432        let count = input.read(&mut buffer)?;
433        if count == 0 {
434            return Ok(());
435        }
436        output.write_all(&buffer[..count])?;
437    }
438}
439
440#[cfg(test)]
441mod tests {
442    use super::*;
443    use crate::q42_volume::Q42Volume;
444    use tempfile::NamedTempFile;
445
446    #[test]
447    fn streams_a_readable_volume_without_payload_accumulation() {
448        let mut lex = HashMap::new();
449        lex.insert(1, "s".to_string());
450        lex.insert(2, "p".to_string());
451        lex.insert(3, "o".to_string());
452        let mut writer = StreamingQ42VolumeWriter::new(&lex).unwrap();
453        writer
454            .push_block(
455                0,
456                &[NQuin {
457                    subject: 1,
458                    predicate: 2,
459                    object: 3,
460                    context: 0,
461                    metadata: 0,
462                    parity: 0,
463                }],
464            )
465            .unwrap();
466        let output = NamedTempFile::new().unwrap();
467        writer.finish(output.path()).unwrap();
468        let volume = Q42Volume::open(output.path()).unwrap();
469        assert_eq!(volume.block_count(), 1);
470        assert_eq!(volume.object_hash_bounds(), Some((3, 3)));
471    }
472}