qualia_core_db/q42/volume/
publish.rs1use std::collections::HashMap;
4use std::io;
5use std::path::{Path, PathBuf};
6
7use super::super::{write_volume_root, Q42Volume, Q42VolumeManifest, StreamingQ42VolumeWriter};
8use crate::NQuin;
9
10pub const DEFAULT_SEGMENT_MAX_BYTES: u64 = 512 * 1024 * 1024;
12
13pub struct Q42RolloverPublisher {
16 dir: PathBuf,
17 stem: String,
18 lex: HashMap<u64, String>,
19 max_bytes: u64,
20 writer: StreamingQ42VolumeWriter,
21 child_paths: Vec<PathBuf>,
22 next_index: u32,
23}
24
25impl Q42RolloverPublisher {
26 pub fn new(
28 dir: impl Into<PathBuf>,
29 stem: impl Into<String>,
30 lex: HashMap<u64, String>,
31 ) -> io::Result<Self> {
32 Self::with_limit(dir, stem, lex, DEFAULT_SEGMENT_MAX_BYTES)
33 }
34
35 pub fn with_limit(
37 dir: impl Into<PathBuf>,
38 stem: impl Into<String>,
39 lex: HashMap<u64, String>,
40 max_bytes: u64,
41 ) -> io::Result<Self> {
42 if max_bytes < 8 * 1024 {
43 return Err(io::Error::new(
44 io::ErrorKind::InvalidInput,
45 "Q42 segment cap is too small",
46 ));
47 }
48 let dir = dir.into();
49 std::fs::create_dir_all(&dir)?;
50 Ok(Self {
51 writer: StreamingQ42VolumeWriter::new(&lex)?,
52 lex,
53 dir,
54 stem: stem.into(),
55 max_bytes,
56 child_paths: Vec::new(),
57 next_index: 0,
58 })
59 }
60
61 pub fn push_block(&mut self, seq_id: u64, quins: &[NQuin]) -> io::Result<()> {
63 let projected = self.writer.maximum_final_length_after_next_block()?;
64 if self.writer.block_count() > 0 && projected > self.max_bytes {
65 self.roll()?;
66 }
67 self.writer.push_block(seq_id, quins)
68 }
69
70 fn roll(&mut self) -> io::Result<()> {
71 let path = self.child_path(self.next_index);
72 let writer = std::mem::replace(&mut self.writer, StreamingQ42VolumeWriter::new(&self.lex)?);
73 writer.finish(&path)?;
74 self.child_paths.push(path);
75 self.next_index += 1;
76 Ok(())
77 }
78
79 fn child_path(&self, index: u32) -> PathBuf {
80 self.dir.join(format!("{}-{:04}.q42", self.stem, index))
81 }
82
83 pub fn finish(mut self) -> io::Result<PathBuf> {
85 if self.writer.block_count() > 0 || self.child_paths.is_empty() {
86 let path = self.child_path(self.next_index);
87 self.writer.finish(&path)?;
88 self.child_paths.push(path);
89 }
90 let mut segments = Vec::new();
91 for path in &self.child_paths {
92 let name = path
93 .file_name()
94 .and_then(|n| n.to_str())
95 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Q42 child name"))?
96 .to_string();
97 segments.push(Q42VolumeManifest::segment_from_file(path, name)?);
98 }
99 let root = self.dir.join(format!("{}-root.q42", self.stem));
100 write_volume_root(
101 &root,
102 &Q42VolumeManifest {
103 generation: 1,
104 segments,
105 lexicon_segments: Vec::new(),
106 },
107 )?;
108 Ok(root)
109 }
110}
111
112pub fn append_segment_to_root(root_path: &Path, new_child: &Path) -> io::Result<()> {
115 let parent = root_path
116 .parent()
117 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Q42 root has no parent"))?;
118 let volume = Q42Volume::open(root_path)?;
119 let mut manifest = volume.volume_manifest()?.ok_or_else(|| {
120 io::Error::new(
121 io::ErrorKind::InvalidInput,
122 "append requires a volume-root catalog",
123 )
124 })?;
125 let name = new_child
126 .file_name()
127 .and_then(|n| n.to_str())
128 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "child name"))?
129 .to_string();
130 manifest
131 .segments
132 .push(Q42VolumeManifest::segment_from_file(new_child, name)?);
133 manifest.generation = manifest.generation.saturating_add(1);
134 let tmp = parent.join(format!(
135 ".{}.tmp",
136 root_path
137 .file_name()
138 .and_then(|n| n.to_str())
139 .unwrap_or("root.q42")
140 ));
141 write_volume_root(&tmp, &manifest)?;
142 std::fs::rename(&tmp, root_path)?;
143 Ok(())
144}
145
146#[cfg(test)]
147mod tests {
148 use super::*;
149 use crate::NQuin;
150
151 fn quin(object: u64) -> NQuin {
152 NQuin {
153 subject: object,
154 predicate: 1,
155 object,
156 context: 0,
157 metadata: 0,
158 parity: object ^ 1 ^ object,
159 }
160 }
161
162 #[test]
163 fn rolls_when_the_next_block_would_exceed_the_cap() {
164 let dir = tempfile::TempDir::new().unwrap();
165 let mut lex = HashMap::new();
166 lex.insert(1, "p".into());
167 let mut pubr = Q42RolloverPublisher::with_limit(dir.path(), "set", lex, 16 * 1024).unwrap();
168 for i in 0..8u64 {
169 pubr.push_block(i, &[quin(i + 1)]).unwrap();
170 }
171 let root = pubr.finish().unwrap();
172 let volume = Q42Volume::open(&root).unwrap();
173 let manifest = volume.volume_manifest().unwrap().unwrap();
174 assert!(
175 manifest.segments.len() >= 2,
176 "expected rollover, got {} children",
177 manifest.segments.len()
178 );
179 }
180
181 #[test]
182 fn append_publishes_a_new_generation() {
183 let dir = tempfile::TempDir::new().unwrap();
184 let mut lex = HashMap::new();
185 lex.insert(1, "p".into());
186 let mut pubr =
187 Q42RolloverPublisher::with_limit(dir.path(), "gen", lex.clone(), 1024 * 1024).unwrap();
188 pubr.push_block(0, &[quin(1)]).unwrap();
189 let root = pubr.finish().unwrap();
190 let extra = dir.path().join("extra.q42");
191 let mut w = StreamingQ42VolumeWriter::new(&lex).unwrap();
192 w.push_block(1, &[quin(2)]).unwrap();
193 w.finish(&extra).unwrap();
194 append_segment_to_root(&root, &extra).unwrap();
195 let volume = Q42Volume::open(&root).unwrap();
196 let manifest = volume.volume_manifest().unwrap().unwrap();
197 assert_eq!(manifest.generation, 2);
198 assert_eq!(manifest.segments.len(), 2);
199 }
200}