Skip to main content

qualia_core_db/q42/volume/
publish.rs

1//! Segment rollover and append-as-new-segment for logical Q42 volumes.
2
3use std::collections::HashMap;
4use std::io;
5use std::path::{Path, PathBuf};
6
7use super::super::{write_volume_root, Q42Volume, Q42VolumeManifest, StreamingQ42VolumeWriter};
8use crate::NQuin;
9
10/// Default child-segment cap (512 MiB). Includes header, lexicon, indexes, data.
11pub const DEFAULT_SEGMENT_MAX_BYTES: u64 = 512 * 1024 * 1024;
12
13/// Writes object-sorted SuperBlocks into successive `.q42` children and
14/// publishes an atomic root catalog when finished.
15pub struct Q42RolloverPublisher {
16    dir: PathBuf,
17    stem: String,
18    lex: HashMap<u64, String>,
19    max_bytes: u64,
20    writer: StreamingQ42VolumeWriter,
21    child_paths: Vec<PathBuf>,
22    next_index: u32,
23}
24
25impl Q42RolloverPublisher {
26    /// Creates a default `Q42RolloverPublisher`.
27    pub fn new(
28        dir: impl Into<PathBuf>,
29        stem: impl Into<String>,
30        lex: HashMap<u64, String>,
31    ) -> io::Result<Self> {
32        Self::with_limit(dir, stem, lex, DEFAULT_SEGMENT_MAX_BYTES)
33    }
34
35    /// Builder: sets `limit`.
36    pub fn with_limit(
37        dir: impl Into<PathBuf>,
38        stem: impl Into<String>,
39        lex: HashMap<u64, String>,
40        max_bytes: u64,
41    ) -> io::Result<Self> {
42        if max_bytes < 8 * 1024 {
43            return Err(io::Error::new(
44                io::ErrorKind::InvalidInput,
45                "Q42 segment cap is too small",
46            ));
47        }
48        let dir = dir.into();
49        std::fs::create_dir_all(&dir)?;
50        Ok(Self {
51            writer: StreamingQ42VolumeWriter::new(&lex)?,
52            lex,
53            dir,
54            stem: stem.into(),
55            max_bytes,
56            child_paths: Vec::new(),
57            next_index: 0,
58        })
59    }
60
61    /// Pushes `block`; error at capacity.
62    pub fn push_block(&mut self, seq_id: u64, quins: &[NQuin]) -> io::Result<()> {
63        let projected = self.writer.maximum_final_length_after_next_block()?;
64        if self.writer.block_count() > 0 && projected > self.max_bytes {
65            self.roll()?;
66        }
67        self.writer.push_block(seq_id, quins)
68    }
69
70    fn roll(&mut self) -> io::Result<()> {
71        let path = self.child_path(self.next_index);
72        let writer = std::mem::replace(&mut self.writer, StreamingQ42VolumeWriter::new(&self.lex)?);
73        writer.finish(&path)?;
74        self.child_paths.push(path);
75        self.next_index += 1;
76        Ok(())
77    }
78
79    fn child_path(&self, index: u32) -> PathBuf {
80        self.dir.join(format!("{}-{:04}.q42", self.stem, index))
81    }
82
83    /// Finish the open child and write `stem-root.q42` naming every child.
84    pub fn finish(mut self) -> io::Result<PathBuf> {
85        if self.writer.block_count() > 0 || self.child_paths.is_empty() {
86            let path = self.child_path(self.next_index);
87            self.writer.finish(&path)?;
88            self.child_paths.push(path);
89        }
90        let mut segments = Vec::new();
91        for path in &self.child_paths {
92            let name = path
93                .file_name()
94                .and_then(|n| n.to_str())
95                .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Q42 child name"))?
96                .to_string();
97            segments.push(Q42VolumeManifest::segment_from_file(path, name)?);
98        }
99        let root = self.dir.join(format!("{}-root.q42", self.stem));
100        write_volume_root(
101            &root,
102            &Q42VolumeManifest {
103                generation: 1,
104                segments,
105                lexicon_segments: Vec::new(),
106            },
107        )?;
108        Ok(root)
109    }
110}
111
112/// Append a finished child segment to an existing root by publishing a new
113/// generation (`stem-root.q42` overwritten atomically via temp + rename).
114pub fn append_segment_to_root(root_path: &Path, new_child: &Path) -> io::Result<()> {
115    let parent = root_path
116        .parent()
117        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Q42 root has no parent"))?;
118    let volume = Q42Volume::open(root_path)?;
119    let mut manifest = volume.volume_manifest()?.ok_or_else(|| {
120        io::Error::new(
121            io::ErrorKind::InvalidInput,
122            "append requires a volume-root catalog",
123        )
124    })?;
125    let name = new_child
126        .file_name()
127        .and_then(|n| n.to_str())
128        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "child name"))?
129        .to_string();
130    manifest
131        .segments
132        .push(Q42VolumeManifest::segment_from_file(new_child, name)?);
133    manifest.generation = manifest.generation.saturating_add(1);
134    let tmp = parent.join(format!(
135        ".{}.tmp",
136        root_path
137            .file_name()
138            .and_then(|n| n.to_str())
139            .unwrap_or("root.q42")
140    ));
141    write_volume_root(&tmp, &manifest)?;
142    std::fs::rename(&tmp, root_path)?;
143    Ok(())
144}
145
146#[cfg(test)]
147mod tests {
148    use super::*;
149    use crate::NQuin;
150
151    fn quin(object: u64) -> NQuin {
152        NQuin {
153            subject: object,
154            predicate: 1,
155            object,
156            context: 0,
157            metadata: 0,
158            parity: object ^ 1 ^ object,
159        }
160    }
161
162    #[test]
163    fn rolls_when_the_next_block_would_exceed_the_cap() {
164        let dir = tempfile::TempDir::new().unwrap();
165        let mut lex = HashMap::new();
166        lex.insert(1, "p".into());
167        let mut pubr = Q42RolloverPublisher::with_limit(dir.path(), "set", lex, 16 * 1024).unwrap();
168        for i in 0..8u64 {
169            pubr.push_block(i, &[quin(i + 1)]).unwrap();
170        }
171        let root = pubr.finish().unwrap();
172        let volume = Q42Volume::open(&root).unwrap();
173        let manifest = volume.volume_manifest().unwrap().unwrap();
174        assert!(
175            manifest.segments.len() >= 2,
176            "expected rollover, got {} children",
177            manifest.segments.len()
178        );
179    }
180
181    #[test]
182    fn append_publishes_a_new_generation() {
183        let dir = tempfile::TempDir::new().unwrap();
184        let mut lex = HashMap::new();
185        lex.insert(1, "p".into());
186        let mut pubr =
187            Q42RolloverPublisher::with_limit(dir.path(), "gen", lex.clone(), 1024 * 1024).unwrap();
188        pubr.push_block(0, &[quin(1)]).unwrap();
189        let root = pubr.finish().unwrap();
190        let extra = dir.path().join("extra.q42");
191        let mut w = StreamingQ42VolumeWriter::new(&lex).unwrap();
192        w.push_block(1, &[quin(2)]).unwrap();
193        w.finish(&extra).unwrap();
194        append_segment_to_root(&root, &extra).unwrap();
195        let volume = Q42Volume::open(&root).unwrap();
196        let manifest = volume.volume_manifest().unwrap().unwrap();
197        assert_eq!(manifest.generation, 2);
198        assert_eq!(manifest.segments.len(), 2);
199    }
200}