Skip to main content

qualia_core_db/q42/volume/
compact.rs

1//! Compact a logical Q42 volume set (or a single-file volume) into a new generation.
2//!
3//! SuperBlocks are streamed in object order through a caller-owned 40960-byte
4//! buffer. The hot per-Quin loop only copies into a stack `[NQuin; QUINS_PER_BLOCK]`.
5//! Integrator wires `mod compact;` in `volume/mod.rs`.
6
7use std::collections::HashMap;
8use std::io;
9use std::path::{Path, PathBuf};
10
11use super::super::{
12    write_volume_root_with_lex, Q42Volume, Q42VolumeManifest, StreamingQ42VolumeWriter,
13    FLAG_PERMISSIVE_COMMONS, FLAG_SANCTUARY, QUIN_SIZE, SUPERBLOCK_HEADER, SUPERBLOCK_SIZE,
14};
15use super::manifest::Q42VolumeSet;
16use super::publication::quin_requires_sanctuary;
17use super::publish::{Q42RolloverPublisher, DEFAULT_SEGMENT_MAX_BYTES};
18use crate::{NQuin, QUINS_PER_BLOCK};
19
20/// Wraps `message` as an `InvalidInput` io error (malformed/empty input).
21fn invalid(message: impl Into<String>) -> io::Error {
22    io::Error::new(io::ErrorKind::InvalidInput, message.into())
23}
24
25/// Compact `root` into `out_dir`.
26///
27/// A logical volume-set root is rewritten through [`Q42RolloverPublisher`]
28/// (or [`StreamingQ42VolumeWriter`] when every input may remain Permissive
29/// Commons). A single-file volume is rewritten through the streaming writer
30/// so the result carries PIDX + FIDX + BIDX. The new catalog is published
31/// with the append temp+rename pattern.
32pub fn compact_volume_set(root: &Path, out_dir: &Path) -> io::Result<PathBuf> {
33    std::fs::create_dir_all(out_dir)?;
34    let volume = Q42Volume::open(root)?;
35    if volume.volume_manifest()?.is_some() {
36        compact_logical_set(root, out_dir)
37    } else {
38        compact_single_volume(&volume, root, out_dir)
39    }
40}
41
42/// Rewrites a logical volume set: merges all segment lexicons, scans every
43/// segment for Sanctuary-classified quins (a set declaring Commons that
44/// contains sanctuary data is NOT written as Commons), then rewrites via the
45/// streaming writer (all-Commons) or the rollover publisher, and publishes the
46/// incremented-generation root catalog atomically.
47fn compact_logical_set(root: &Path, out_dir: &Path) -> io::Result<PathBuf> {
48    let set = Q42VolumeSet::open_root(root)?;
49    if set.segments().is_empty() {
50        return Err(invalid("Q42 volume set has no data segments"));
51    }
52
53    let mut lex = HashMap::new();
54    merge_lexicon(set.root(), &mut lex)?;
55    for segment in set.segments() {
56        merge_lexicon(segment, &mut lex)?;
57    }
58    for segment in set.lexicon_segments() {
59        merge_lexicon(segment, &mut lex)?;
60    }
61
62    let all_commons = set.segments().iter().all(volume_declares_commons);
63    let saw_sanctuary = scan_set_for_sanctuary(&set)?;
64    let declare_commons = all_commons && !saw_sanctuary;
65    let generation = set.manifest().generation.saturating_add(1);
66    let stem = output_stem(root);
67
68    let child_paths = if declare_commons {
69        rewrite_set_through_streaming_writer(&set, out_dir, &stem, &lex, true)?
70    } else {
71        rewrite_set_through_publisher(&set, out_dir, &stem, lex.clone())?
72    };
73
74    let mut segments = Vec::with_capacity(child_paths.len());
75    for path in &child_paths {
76        let name = path
77            .file_name()
78            .and_then(|n| n.to_str())
79            .ok_or_else(|| invalid("Q42 compacted child name"))?
80            .to_string();
81        segments.push(Q42VolumeManifest::segment_from_file(path, name)?);
82    }
83    let manifest = Q42VolumeManifest {
84        generation,
85        segments,
86        lexicon_segments: Vec::new(),
87    };
88    let root_path = out_dir.join(format!("{stem}-root.q42"));
89    publish_root_atomically(&root_path, &lex, &manifest)?;
90    Ok(root_path)
91}
92
93/// Rewrites a single-file volume through the streaming writer, preserving
94/// the Commons flag only when no sanctuary quins are found. Errors on an
95/// empty volume.
96fn compact_single_volume(volume: &Q42Volume, root: &Path, out_dir: &Path) -> io::Result<PathBuf> {
97    if volume.block_count() == 0 {
98        return Err(invalid("Q42 volume has no SuperBlocks to compact"));
99    }
100    let mut lex = HashMap::new();
101    merge_lexicon(volume, &mut lex)?;
102    let declare_commons = volume_declares_commons(volume) && !scan_volume_for_sanctuary(volume)?;
103
104    let mut writer = StreamingQ42VolumeWriter::new(&lex)?;
105    if declare_commons {
106        writer.declare_permissive_commons();
107    }
108    stream_volume_blocks(volume, |seq, quins| writer.push_block(seq, quins))?;
109
110    let out = out_dir.join(format!("{}.q42", output_stem(root)));
111    writer.finish(&out)?;
112    Ok(out)
113}
114
115/// Streams the set through `Q42RolloverPublisher` (which auto-splits at the
116/// segment byte cap), then returns the child-segment paths and removes the
117/// publisher's provisional root catalog (the caller rewrites it at the next
118/// generation).
119fn rewrite_set_through_publisher(
120    set: &Q42VolumeSet,
121    out_dir: &Path,
122    stem: &str,
123    lex: HashMap<u64, String>,
124) -> io::Result<Vec<PathBuf>> {
125    let mut publisher = Q42RolloverPublisher::new(out_dir, stem, lex)?;
126    stream_set_blocks(set, |seq, quins| publisher.push_block(seq, quins))?;
127    let produced = publisher.finish()?;
128    let catalog = Q42Volume::open(&produced)?;
129    let manifest = catalog.volume_manifest()?.ok_or_else(|| {
130        io::Error::new(
131            io::ErrorKind::InvalidData,
132            "Q42 rollover publisher produced no root catalog",
133        )
134    })?;
135    let parent = produced.parent().unwrap_or(out_dir);
136    let children = manifest
137        .segments
138        .iter()
139        .map(|segment| parent.join(&segment.locator))
140        .collect();
141    // The publisher already wrote a generation-1 catalog; the caller replaces
142    // it atomically with the incremented generation.
143    let _ = std::fs::remove_file(&produced);
144    Ok(children)
145}
146
147/// Streams the set through `CompactRollover` (local rollover split with a
148/// publication flag) and returns the produced child paths.
149fn rewrite_set_through_streaming_writer(
150    set: &Q42VolumeSet,
151    out_dir: &Path,
152    stem: &str,
153    lex: &HashMap<u64, String>,
154    declare_commons: bool,
155) -> io::Result<Vec<PathBuf>> {
156    let mut writer = CompactRollover::new(out_dir, stem, lex, declare_commons)?;
157    stream_set_blocks(set, |seq, quins| writer.push_block(seq, quins))?;
158    writer.finish_children()
159}
160
161/// Same rollover contract as [`Q42RolloverPublisher`], with publication flags.
162struct CompactRollover {
163    dir: PathBuf,
164    stem: String,
165    lex: HashMap<u64, String>,
166    max_bytes: u64,
167    writer: StreamingQ42VolumeWriter,
168    child_paths: Vec<PathBuf>,
169    next_index: u32,
170    declare_commons: bool,
171}
172
173impl CompactRollover {
174    /// Creates a rollover splitter writing `stem-NNNN.q42` children into `dir`,
175    /// honouring `declare_commons` on each child.
176    fn new(
177        dir: impl Into<PathBuf>,
178        stem: impl Into<String>,
179        lex: &HashMap<u64, String>,
180        declare_commons: bool,
181    ) -> io::Result<Self> {
182        let dir = dir.into();
183        std::fs::create_dir_all(&dir)?;
184        Ok(Self {
185            writer: new_streaming_writer(lex, declare_commons)?,
186            lex: lex.clone(),
187            dir,
188            stem: stem.into(),
189            max_bytes: DEFAULT_SEGMENT_MAX_BYTES,
190            child_paths: Vec::new(),
191            next_index: 0,
192            declare_commons,
193        })
194    }
195
196    /// Pushes one SuperBlock; rolls to a new child file first when the
197    /// projected final length would exceed the segment cap.
198    fn push_block(&mut self, seq_id: u64, quins: &[NQuin]) -> io::Result<()> {
199        let projected = self.writer.maximum_final_length_after_next_block()?;
200        if self.writer.block_count() > 0 && projected > self.max_bytes {
201            self.roll()?;
202        }
203        self.writer.push_block(seq_id, quins)
204    }
205
206    /// Finalizes the current child writer to disk and swaps in a fresh one.
207    fn roll(&mut self) -> io::Result<()> {
208        let path = self.child_path(self.next_index);
209        let writer = std::mem::replace(
210            &mut self.writer,
211            new_streaming_writer(&self.lex, self.declare_commons)?,
212        );
213        writer.finish(&path)?;
214        self.child_paths.push(path);
215        self.next_index += 1;
216        Ok(())
217    }
218
219    /// Path for child segment `index` (`stem-0000.q42` pattern).
220    fn child_path(&self, index: u32) -> PathBuf {
221        self.dir.join(format!("{}-{:04}.q42", self.stem, index))
222    }
223
224    /// Flushes the last child (if it has blocks, or none were produced) and
225    /// returns all child paths in write order.
226    fn finish_children(mut self) -> io::Result<Vec<PathBuf>> {
227        if self.writer.block_count() > 0 || self.child_paths.is_empty() {
228            let path = self.child_path(self.next_index);
229            self.writer.finish(&path)?;
230            self.child_paths.push(path);
231        }
232        Ok(self.child_paths)
233    }
234}
235
236/// Opens a `StreamingQ42VolumeWriter` over `lex`, setting the Permissive
237/// Commons declaration when `declare_commons`.
238fn new_streaming_writer(
239    lex: &HashMap<u64, String>,
240    declare_commons: bool,
241) -> io::Result<StreamingQ42VolumeWriter> {
242    let mut writer = StreamingQ42VolumeWriter::new(lex)?;
243    if declare_commons {
244        writer.declare_permissive_commons();
245    }
246    Ok(writer)
247}
248
249/// Writes the root catalog to `<name>.tmp` then renames it over `root_path` —
250    /// atomic publish so readers never see a partial catalog.
251fn publish_root_atomically(
252    root_path: &Path,
253    lex: &HashMap<u64, String>,
254    manifest: &Q42VolumeManifest,
255) -> io::Result<()> {
256    let parent = root_path
257        .parent()
258        .ok_or_else(|| invalid("Q42 compact root has no parent"))?;
259    let tmp = parent.join(format!(
260        ".{}.tmp",
261        root_path
262            .file_name()
263            .and_then(|n| n.to_str())
264            .unwrap_or("root.q42")
265    ));
266    write_volume_root_with_lex(&tmp, lex, manifest)?;
267    std::fs::rename(&tmp, root_path)
268}
269
270/// Streams every SuperBlock of every set segment in order, renumbering
271    /// `seq_id` densely; errors when the set has no blocks.
272fn stream_set_blocks(
273    set: &Q42VolumeSet,
274    mut push: impl FnMut(u64, &[NQuin]) -> io::Result<()>,
275) -> io::Result<()> {
276    let mut seq = 0u64;
277    for segment in set.segments() {
278        stream_volume_blocks(segment, |_, quins| {
279            let result = push(seq, quins);
280            seq += 1;
281            result
282        })?;
283    }
284    if seq == 0 {
285        return Err(invalid("Q42 volume set has no SuperBlocks to compact"));
286    }
287    Ok(())
288}
289
290/// Reads each SuperBlock into a fixed stack buffer, decodes the live quins,
291    /// and pushes them through `push` keyed by block index.
292fn stream_volume_blocks(
293    volume: &Q42Volume,
294    mut push: impl FnMut(u64, &[NQuin]) -> io::Result<()>,
295) -> io::Result<()> {
296    let mut decoded = [0u8; SUPERBLOCK_SIZE];
297    let mut block = [NQuin::default(); QUINS_PER_BLOCK];
298    for index in 0..volume.block_count() as usize {
299        volume.read_superblock_into(index, &mut decoded)?;
300        let live = decode_live_quins(&decoded, &mut block)?;
301        push(index as u64, &block[..live])?;
302    }
303    Ok(())
304}
305
306/// Decodes the live-quin prefix of a SuperBlock into `out` (zero-copy
307    /// `pod_read_unaligned`); fails closed on a zero/oversized live count.
308fn decode_live_quins(
309    decoded: &[u8; SUPERBLOCK_SIZE],
310    out: &mut [NQuin; QUINS_PER_BLOCK],
311) -> io::Result<usize> {
312    let live = u64::from_le_bytes(decoded[16..24].try_into().unwrap()) as usize;
313    if live == 0 || live > QUINS_PER_BLOCK {
314        return Err(io::Error::new(
315            io::ErrorKind::InvalidData,
316            "Q42 SuperBlock has invalid live Quin count",
317        ));
318    }
319    for i in 0..live {
320        let offset = SUPERBLOCK_HEADER + i * QUIN_SIZE;
321        out[i] = bytemuck::pod_read_unaligned(&decoded[offset..offset + QUIN_SIZE]);
322    }
323    Ok(live)
324}
325
326/// Merges a volume's Q42LEX hash→string entries into `dest` (first writer
327    /// wins on hash collisions).
328fn merge_lexicon(volume: &Q42Volume, dest: &mut HashMap<u64, String>) -> io::Result<()> {
329    let view = volume.lex_view().map_err(|error| {
330        io::Error::new(
331            io::ErrorKind::InvalidData,
332            format!("invalid Q42LEX: {error:?}"),
333        )
334    })?;
335    for i in 0..view.entry_count() {
336        let Some(hash) = view.hash_at(i) else {
337            continue;
338        };
339        if let Some((ns, local)) = view.string_parts_at(i) {
340            let mut text = String::with_capacity(ns.len() + local.len());
341            text.push_str(ns);
342            text.push_str(local);
343            dest.entry(hash).or_insert(text);
344        }
345    }
346    Ok(())
347}
348
349/// `true` when the volume header declares `FLAG_PERMISSIVE_COMMONS`.
350fn volume_declares_commons(volume: &Q42Volume) -> bool {
351    volume.header().flags & FLAG_PERMISSIVE_COMMONS != 0
352}
353
354/// `true` when any segment carries `FLAG_SANCTUARY` or contains a quin
355    /// requiring sanctuary handling — prevents wrongly declaring Commons.
356fn scan_set_for_sanctuary(set: &Q42VolumeSet) -> io::Result<bool> {
357    for segment in set.segments() {
358        if segment.header().flags & FLAG_SANCTUARY != 0 || scan_volume_for_sanctuary(segment)? {
359            return Ok(true);
360        }
361    }
362    Ok(false)
363}
364
365/// Scans every live quin in the volume for `quin_requires_sanctuary` — `true`
366    /// on first hit.
367fn scan_volume_for_sanctuary(volume: &Q42Volume) -> io::Result<bool> {
368    let mut decoded = [0u8; SUPERBLOCK_SIZE];
369    let mut block = [NQuin::default(); QUINS_PER_BLOCK];
370    for index in 0..volume.block_count() as usize {
371        volume.read_superblock_into(index, &mut decoded)?;
372        let live = decode_live_quins(&decoded, &mut block)?;
373        if block[..live].iter().any(quin_requires_sanctuary) {
374            return Ok(true);
375        }
376    }
377    Ok(false)
378}
379
380/// Output filename stem: the input's file stem minus any `-root` suffix
381    /// (`"volume"` fallback).
382fn output_stem(root: &Path) -> String {
383    let stem = root
384        .file_stem()
385        .and_then(|s| s.to_str())
386        .unwrap_or("volume");
387    stem.strip_suffix("-root").unwrap_or(stem).to_string()
388}
389
390#[cfg(test)]
391mod tests {
392    use super::super::super::{
393        write_volume_root, FLAG_FIELD_POSTINGS, FLAG_FIELD_RANGES, FLAG_OBJECT_SORTED,
394    };
395    use super::*;
396    use crate::q42_volume::write_unified_volume;
397
398    fn quin(object: u64) -> NQuin {
399        NQuin {
400            subject: object,
401            predicate: 1,
402            object,
403            context: 0,
404            metadata: 0,
405            parity: object ^ 1 ^ object,
406        }
407    }
408
409    fn medical_quin(object: u64) -> NQuin {
410        let mut q = quin(object);
411        q.set_sensitivity_byte(NQuin::SENSITIVITY_CLASSIFIED);
412        q.set_sensitivity_tier(NQuin::SENSITIVITY_TIER_MEDICAL);
413        q.parity = q.subject ^ q.predicate ^ q.object ^ q.context;
414        q
415    }
416
417    fn lex_for(entries: &[(u64, &str)]) -> HashMap<u64, String> {
418        entries
419            .iter()
420            .map(|(hash, text)| (*hash, (*text).to_string()))
421            .collect()
422    }
423
424    fn write_child(path: &Path, quins: &[NQuin], lex: &HashMap<u64, String>, commons: bool) {
425        if commons {
426            let mut writer = StreamingQ42VolumeWriter::new(lex).unwrap();
427            writer.declare_permissive_commons();
428            writer.push_block(0, quins).unwrap();
429            writer.finish(path).unwrap();
430        } else {
431            let first = quins[0].object;
432            let last = quins[quins.len() - 1].object;
433            write_unified_volume(path, lex, &[(first, last)], &[quins.to_vec()]).unwrap();
434        }
435    }
436
437    fn write_two_child_root(
438        dir: &Path,
439        first: &[NQuin],
440        first_lex: &HashMap<u64, String>,
441        first_commons: bool,
442        second: &[NQuin],
443        second_lex: &HashMap<u64, String>,
444        second_commons: bool,
445    ) -> PathBuf {
446        let a = dir.join("segment-000.q42");
447        let b = dir.join("segment-001.q42");
448        write_child(&a, first, first_lex, first_commons);
449        write_child(&b, second, second_lex, second_commons);
450        let root = dir.join("set-root.q42");
451        let manifest = Q42VolumeManifest {
452            generation: 3,
453            segments: vec![
454                Q42VolumeManifest::segment_from_file(&a, "segment-000.q42".into()).unwrap(),
455                Q42VolumeManifest::segment_from_file(&b, "segment-001.q42".into()).unwrap(),
456            ],
457            lexicon_segments: Vec::new(),
458        };
459        write_volume_root(&root, &manifest).unwrap();
460        root
461    }
462
463    fn compacted_quins(root: &Path) -> Vec<NQuin> {
464        let set = Q42VolumeSet::open_root(root).unwrap();
465        let mut out = Vec::new();
466        for segment in set.segments() {
467            out.extend(segment.read_all_quins().unwrap());
468        }
469        out
470    }
471
472    #[test]
473    fn two_children_compact_to_a_readable_root() {
474        let src = tempfile::TempDir::new().unwrap();
475        let out = tempfile::TempDir::new().unwrap();
476        let first_lex = lex_for(&[(1, "p"), (10, "o-a")]);
477        let second_lex = lex_for(&[(1, "p"), (20, "o-b")]);
478        let root = write_two_child_root(
479            src.path(),
480            &[quin(10)],
481            &first_lex,
482            false,
483            &[quin(20)],
484            &second_lex,
485            false,
486        );
487
488        let compacted = compact_volume_set(&root, out.path()).unwrap();
489        let set = Q42VolumeSet::open_root(&compacted).unwrap();
490        assert_eq!(set.manifest().generation, 4);
491        assert!(!set.segments().is_empty());
492        let quins = compacted_quins(&compacted);
493        assert_eq!(quins.len(), 2);
494        assert!(quins.iter().any(|q| q.object == 10));
495        assert!(quins.iter().any(|q| q.object == 20));
496        assert_eq!(set.lookup_hash(10), Some("o-a"));
497        assert_eq!(set.lookup_hash(20), Some("o-b"));
498    }
499
500    #[test]
501    fn object_order_preserved() {
502        let src = tempfile::TempDir::new().unwrap();
503        let out = tempfile::TempDir::new().unwrap();
504        let lex = lex_for(&[(1, "p")]);
505        let root = write_two_child_root(
506            src.path(),
507            &[quin(2), quin(4)],
508            &lex,
509            false,
510            &[quin(6), quin(8)],
511            &lex,
512            false,
513        );
514
515        let compacted = compact_volume_set(&root, out.path()).unwrap();
516        let objects: Vec<u64> = compacted_quins(&compacted)
517            .iter()
518            .map(|q| q.object)
519            .collect();
520        assert_eq!(objects, vec![2, 4, 6, 8]);
521        assert!(objects.windows(2).all(|pair| pair[0] <= pair[1]));
522    }
523
524    #[test]
525    fn sanctuary_child_cannot_become_commons() {
526        let src = tempfile::TempDir::new().unwrap();
527        let out = tempfile::TempDir::new().unwrap();
528        let lex = lex_for(&[(1, "p")]);
529        let root = write_two_child_root(
530            src.path(),
531            &[quin(3)],
532            &lex,
533            true,
534            &[medical_quin(9)],
535            &lex,
536            false,
537        );
538
539        let compacted = compact_volume_set(&root, out.path()).unwrap();
540        let set = Q42VolumeSet::open_root(&compacted).unwrap();
541        for segment in set.segments() {
542            let flags = segment.header().flags;
543            assert_eq!(
544                flags & FLAG_PERMISSIVE_COMMONS,
545                0,
546                "Sanctuary input must not mint FLAG_PERMISSIVE_COMMONS"
547            );
548            assert_ne!(flags & FLAG_SANCTUARY, 0);
549        }
550        assert!(compacted_quins(&compacted)
551            .iter()
552            .any(quin_requires_sanctuary));
553    }
554
555    #[test]
556    fn single_file_rewrites_with_field_indexes() {
557        let src = tempfile::TempDir::new().unwrap();
558        let out = tempfile::TempDir::new().unwrap();
559        let path = src.path().join("plain.q42");
560        let lex = lex_for(&[(1, "s"), (2, "p"), (3, "o")]);
561        write_unified_volume(&path, &lex, &[(3, 3)], &[vec![quin(3)]]).unwrap();
562
563        let compacted = compact_volume_set(&path, out.path()).unwrap();
564        let volume = Q42Volume::open(&compacted).unwrap();
565        let flags = volume.header().flags;
566        assert_ne!(flags & FLAG_OBJECT_SORTED, 0);
567        assert_ne!(flags & FLAG_FIELD_RANGES, 0);
568        assert_ne!(flags & FLAG_FIELD_POSTINGS, 0);
569        assert_eq!(volume.read_all_quins().unwrap()[0].object, 3);
570        assert_eq!(volume.lex_view().unwrap().lookup_hash(3), Some("o"));
571    }
572}