1use std::collections::HashMap;
8use std::io;
9use std::path::{Path, PathBuf};
10
11use super::super::{
12 write_volume_root_with_lex, Q42Volume, Q42VolumeManifest, StreamingQ42VolumeWriter,
13 FLAG_PERMISSIVE_COMMONS, FLAG_SANCTUARY, QUIN_SIZE, SUPERBLOCK_HEADER, SUPERBLOCK_SIZE,
14};
15use super::manifest::Q42VolumeSet;
16use super::publication::quin_requires_sanctuary;
17use super::publish::{Q42RolloverPublisher, DEFAULT_SEGMENT_MAX_BYTES};
18use crate::{NQuin, QUINS_PER_BLOCK};
19
20fn invalid(message: impl Into<String>) -> io::Error {
22 io::Error::new(io::ErrorKind::InvalidInput, message.into())
23}
24
25pub fn compact_volume_set(root: &Path, out_dir: &Path) -> io::Result<PathBuf> {
33 std::fs::create_dir_all(out_dir)?;
34 let volume = Q42Volume::open(root)?;
35 if volume.volume_manifest()?.is_some() {
36 compact_logical_set(root, out_dir)
37 } else {
38 compact_single_volume(&volume, root, out_dir)
39 }
40}
41
42fn compact_logical_set(root: &Path, out_dir: &Path) -> io::Result<PathBuf> {
48 let set = Q42VolumeSet::open_root(root)?;
49 if set.segments().is_empty() {
50 return Err(invalid("Q42 volume set has no data segments"));
51 }
52
53 let mut lex = HashMap::new();
54 merge_lexicon(set.root(), &mut lex)?;
55 for segment in set.segments() {
56 merge_lexicon(segment, &mut lex)?;
57 }
58 for segment in set.lexicon_segments() {
59 merge_lexicon(segment, &mut lex)?;
60 }
61
62 let all_commons = set.segments().iter().all(volume_declares_commons);
63 let saw_sanctuary = scan_set_for_sanctuary(&set)?;
64 let declare_commons = all_commons && !saw_sanctuary;
65 let generation = set.manifest().generation.saturating_add(1);
66 let stem = output_stem(root);
67
68 let child_paths = if declare_commons {
69 rewrite_set_through_streaming_writer(&set, out_dir, &stem, &lex, true)?
70 } else {
71 rewrite_set_through_publisher(&set, out_dir, &stem, lex.clone())?
72 };
73
74 let mut segments = Vec::with_capacity(child_paths.len());
75 for path in &child_paths {
76 let name = path
77 .file_name()
78 .and_then(|n| n.to_str())
79 .ok_or_else(|| invalid("Q42 compacted child name"))?
80 .to_string();
81 segments.push(Q42VolumeManifest::segment_from_file(path, name)?);
82 }
83 let manifest = Q42VolumeManifest {
84 generation,
85 segments,
86 lexicon_segments: Vec::new(),
87 };
88 let root_path = out_dir.join(format!("{stem}-root.q42"));
89 publish_root_atomically(&root_path, &lex, &manifest)?;
90 Ok(root_path)
91}
92
93fn compact_single_volume(volume: &Q42Volume, root: &Path, out_dir: &Path) -> io::Result<PathBuf> {
97 if volume.block_count() == 0 {
98 return Err(invalid("Q42 volume has no SuperBlocks to compact"));
99 }
100 let mut lex = HashMap::new();
101 merge_lexicon(volume, &mut lex)?;
102 let declare_commons = volume_declares_commons(volume) && !scan_volume_for_sanctuary(volume)?;
103
104 let mut writer = StreamingQ42VolumeWriter::new(&lex)?;
105 if declare_commons {
106 writer.declare_permissive_commons();
107 }
108 stream_volume_blocks(volume, |seq, quins| writer.push_block(seq, quins))?;
109
110 let out = out_dir.join(format!("{}.q42", output_stem(root)));
111 writer.finish(&out)?;
112 Ok(out)
113}
114
115fn rewrite_set_through_publisher(
120 set: &Q42VolumeSet,
121 out_dir: &Path,
122 stem: &str,
123 lex: HashMap<u64, String>,
124) -> io::Result<Vec<PathBuf>> {
125 let mut publisher = Q42RolloverPublisher::new(out_dir, stem, lex)?;
126 stream_set_blocks(set, |seq, quins| publisher.push_block(seq, quins))?;
127 let produced = publisher.finish()?;
128 let catalog = Q42Volume::open(&produced)?;
129 let manifest = catalog.volume_manifest()?.ok_or_else(|| {
130 io::Error::new(
131 io::ErrorKind::InvalidData,
132 "Q42 rollover publisher produced no root catalog",
133 )
134 })?;
135 let parent = produced.parent().unwrap_or(out_dir);
136 let children = manifest
137 .segments
138 .iter()
139 .map(|segment| parent.join(&segment.locator))
140 .collect();
141 let _ = std::fs::remove_file(&produced);
144 Ok(children)
145}
146
147fn rewrite_set_through_streaming_writer(
150 set: &Q42VolumeSet,
151 out_dir: &Path,
152 stem: &str,
153 lex: &HashMap<u64, String>,
154 declare_commons: bool,
155) -> io::Result<Vec<PathBuf>> {
156 let mut writer = CompactRollover::new(out_dir, stem, lex, declare_commons)?;
157 stream_set_blocks(set, |seq, quins| writer.push_block(seq, quins))?;
158 writer.finish_children()
159}
160
161struct CompactRollover {
163 dir: PathBuf,
164 stem: String,
165 lex: HashMap<u64, String>,
166 max_bytes: u64,
167 writer: StreamingQ42VolumeWriter,
168 child_paths: Vec<PathBuf>,
169 next_index: u32,
170 declare_commons: bool,
171}
172
173impl CompactRollover {
174 fn new(
177 dir: impl Into<PathBuf>,
178 stem: impl Into<String>,
179 lex: &HashMap<u64, String>,
180 declare_commons: bool,
181 ) -> io::Result<Self> {
182 let dir = dir.into();
183 std::fs::create_dir_all(&dir)?;
184 Ok(Self {
185 writer: new_streaming_writer(lex, declare_commons)?,
186 lex: lex.clone(),
187 dir,
188 stem: stem.into(),
189 max_bytes: DEFAULT_SEGMENT_MAX_BYTES,
190 child_paths: Vec::new(),
191 next_index: 0,
192 declare_commons,
193 })
194 }
195
196 fn push_block(&mut self, seq_id: u64, quins: &[NQuin]) -> io::Result<()> {
199 let projected = self.writer.maximum_final_length_after_next_block()?;
200 if self.writer.block_count() > 0 && projected > self.max_bytes {
201 self.roll()?;
202 }
203 self.writer.push_block(seq_id, quins)
204 }
205
206 fn roll(&mut self) -> io::Result<()> {
208 let path = self.child_path(self.next_index);
209 let writer = std::mem::replace(
210 &mut self.writer,
211 new_streaming_writer(&self.lex, self.declare_commons)?,
212 );
213 writer.finish(&path)?;
214 self.child_paths.push(path);
215 self.next_index += 1;
216 Ok(())
217 }
218
219 fn child_path(&self, index: u32) -> PathBuf {
221 self.dir.join(format!("{}-{:04}.q42", self.stem, index))
222 }
223
224 fn finish_children(mut self) -> io::Result<Vec<PathBuf>> {
227 if self.writer.block_count() > 0 || self.child_paths.is_empty() {
228 let path = self.child_path(self.next_index);
229 self.writer.finish(&path)?;
230 self.child_paths.push(path);
231 }
232 Ok(self.child_paths)
233 }
234}
235
236fn new_streaming_writer(
239 lex: &HashMap<u64, String>,
240 declare_commons: bool,
241) -> io::Result<StreamingQ42VolumeWriter> {
242 let mut writer = StreamingQ42VolumeWriter::new(lex)?;
243 if declare_commons {
244 writer.declare_permissive_commons();
245 }
246 Ok(writer)
247}
248
249fn publish_root_atomically(
252 root_path: &Path,
253 lex: &HashMap<u64, String>,
254 manifest: &Q42VolumeManifest,
255) -> io::Result<()> {
256 let parent = root_path
257 .parent()
258 .ok_or_else(|| invalid("Q42 compact root has no parent"))?;
259 let tmp = parent.join(format!(
260 ".{}.tmp",
261 root_path
262 .file_name()
263 .and_then(|n| n.to_str())
264 .unwrap_or("root.q42")
265 ));
266 write_volume_root_with_lex(&tmp, lex, manifest)?;
267 std::fs::rename(&tmp, root_path)
268}
269
270fn stream_set_blocks(
273 set: &Q42VolumeSet,
274 mut push: impl FnMut(u64, &[NQuin]) -> io::Result<()>,
275) -> io::Result<()> {
276 let mut seq = 0u64;
277 for segment in set.segments() {
278 stream_volume_blocks(segment, |_, quins| {
279 let result = push(seq, quins);
280 seq += 1;
281 result
282 })?;
283 }
284 if seq == 0 {
285 return Err(invalid("Q42 volume set has no SuperBlocks to compact"));
286 }
287 Ok(())
288}
289
290fn stream_volume_blocks(
293 volume: &Q42Volume,
294 mut push: impl FnMut(u64, &[NQuin]) -> io::Result<()>,
295) -> io::Result<()> {
296 let mut decoded = [0u8; SUPERBLOCK_SIZE];
297 let mut block = [NQuin::default(); QUINS_PER_BLOCK];
298 for index in 0..volume.block_count() as usize {
299 volume.read_superblock_into(index, &mut decoded)?;
300 let live = decode_live_quins(&decoded, &mut block)?;
301 push(index as u64, &block[..live])?;
302 }
303 Ok(())
304}
305
306fn decode_live_quins(
309 decoded: &[u8; SUPERBLOCK_SIZE],
310 out: &mut [NQuin; QUINS_PER_BLOCK],
311) -> io::Result<usize> {
312 let live = u64::from_le_bytes(decoded[16..24].try_into().unwrap()) as usize;
313 if live == 0 || live > QUINS_PER_BLOCK {
314 return Err(io::Error::new(
315 io::ErrorKind::InvalidData,
316 "Q42 SuperBlock has invalid live Quin count",
317 ));
318 }
319 for i in 0..live {
320 let offset = SUPERBLOCK_HEADER + i * QUIN_SIZE;
321 out[i] = bytemuck::pod_read_unaligned(&decoded[offset..offset + QUIN_SIZE]);
322 }
323 Ok(live)
324}
325
326fn merge_lexicon(volume: &Q42Volume, dest: &mut HashMap<u64, String>) -> io::Result<()> {
329 let view = volume.lex_view().map_err(|error| {
330 io::Error::new(
331 io::ErrorKind::InvalidData,
332 format!("invalid Q42LEX: {error:?}"),
333 )
334 })?;
335 for i in 0..view.entry_count() {
336 let Some(hash) = view.hash_at(i) else {
337 continue;
338 };
339 if let Some((ns, local)) = view.string_parts_at(i) {
340 let mut text = String::with_capacity(ns.len() + local.len());
341 text.push_str(ns);
342 text.push_str(local);
343 dest.entry(hash).or_insert(text);
344 }
345 }
346 Ok(())
347}
348
349fn volume_declares_commons(volume: &Q42Volume) -> bool {
351 volume.header().flags & FLAG_PERMISSIVE_COMMONS != 0
352}
353
354fn scan_set_for_sanctuary(set: &Q42VolumeSet) -> io::Result<bool> {
357 for segment in set.segments() {
358 if segment.header().flags & FLAG_SANCTUARY != 0 || scan_volume_for_sanctuary(segment)? {
359 return Ok(true);
360 }
361 }
362 Ok(false)
363}
364
365fn scan_volume_for_sanctuary(volume: &Q42Volume) -> io::Result<bool> {
368 let mut decoded = [0u8; SUPERBLOCK_SIZE];
369 let mut block = [NQuin::default(); QUINS_PER_BLOCK];
370 for index in 0..volume.block_count() as usize {
371 volume.read_superblock_into(index, &mut decoded)?;
372 let live = decode_live_quins(&decoded, &mut block)?;
373 if block[..live].iter().any(quin_requires_sanctuary) {
374 return Ok(true);
375 }
376 }
377 Ok(false)
378}
379
380fn output_stem(root: &Path) -> String {
383 let stem = root
384 .file_stem()
385 .and_then(|s| s.to_str())
386 .unwrap_or("volume");
387 stem.strip_suffix("-root").unwrap_or(stem).to_string()
388}
389
390#[cfg(test)]
391mod tests {
392 use super::super::super::{
393 write_volume_root, FLAG_FIELD_POSTINGS, FLAG_FIELD_RANGES, FLAG_OBJECT_SORTED,
394 };
395 use super::*;
396 use crate::q42_volume::write_unified_volume;
397
398 fn quin(object: u64) -> NQuin {
399 NQuin {
400 subject: object,
401 predicate: 1,
402 object,
403 context: 0,
404 metadata: 0,
405 parity: object ^ 1 ^ object,
406 }
407 }
408
409 fn medical_quin(object: u64) -> NQuin {
410 let mut q = quin(object);
411 q.set_sensitivity_byte(NQuin::SENSITIVITY_CLASSIFIED);
412 q.set_sensitivity_tier(NQuin::SENSITIVITY_TIER_MEDICAL);
413 q.parity = q.subject ^ q.predicate ^ q.object ^ q.context;
414 q
415 }
416
417 fn lex_for(entries: &[(u64, &str)]) -> HashMap<u64, String> {
418 entries
419 .iter()
420 .map(|(hash, text)| (*hash, (*text).to_string()))
421 .collect()
422 }
423
424 fn write_child(path: &Path, quins: &[NQuin], lex: &HashMap<u64, String>, commons: bool) {
425 if commons {
426 let mut writer = StreamingQ42VolumeWriter::new(lex).unwrap();
427 writer.declare_permissive_commons();
428 writer.push_block(0, quins).unwrap();
429 writer.finish(path).unwrap();
430 } else {
431 let first = quins[0].object;
432 let last = quins[quins.len() - 1].object;
433 write_unified_volume(path, lex, &[(first, last)], &[quins.to_vec()]).unwrap();
434 }
435 }
436
437 fn write_two_child_root(
438 dir: &Path,
439 first: &[NQuin],
440 first_lex: &HashMap<u64, String>,
441 first_commons: bool,
442 second: &[NQuin],
443 second_lex: &HashMap<u64, String>,
444 second_commons: bool,
445 ) -> PathBuf {
446 let a = dir.join("segment-000.q42");
447 let b = dir.join("segment-001.q42");
448 write_child(&a, first, first_lex, first_commons);
449 write_child(&b, second, second_lex, second_commons);
450 let root = dir.join("set-root.q42");
451 let manifest = Q42VolumeManifest {
452 generation: 3,
453 segments: vec![
454 Q42VolumeManifest::segment_from_file(&a, "segment-000.q42".into()).unwrap(),
455 Q42VolumeManifest::segment_from_file(&b, "segment-001.q42".into()).unwrap(),
456 ],
457 lexicon_segments: Vec::new(),
458 };
459 write_volume_root(&root, &manifest).unwrap();
460 root
461 }
462
463 fn compacted_quins(root: &Path) -> Vec<NQuin> {
464 let set = Q42VolumeSet::open_root(root).unwrap();
465 let mut out = Vec::new();
466 for segment in set.segments() {
467 out.extend(segment.read_all_quins().unwrap());
468 }
469 out
470 }
471
472 #[test]
473 fn two_children_compact_to_a_readable_root() {
474 let src = tempfile::TempDir::new().unwrap();
475 let out = tempfile::TempDir::new().unwrap();
476 let first_lex = lex_for(&[(1, "p"), (10, "o-a")]);
477 let second_lex = lex_for(&[(1, "p"), (20, "o-b")]);
478 let root = write_two_child_root(
479 src.path(),
480 &[quin(10)],
481 &first_lex,
482 false,
483 &[quin(20)],
484 &second_lex,
485 false,
486 );
487
488 let compacted = compact_volume_set(&root, out.path()).unwrap();
489 let set = Q42VolumeSet::open_root(&compacted).unwrap();
490 assert_eq!(set.manifest().generation, 4);
491 assert!(!set.segments().is_empty());
492 let quins = compacted_quins(&compacted);
493 assert_eq!(quins.len(), 2);
494 assert!(quins.iter().any(|q| q.object == 10));
495 assert!(quins.iter().any(|q| q.object == 20));
496 assert_eq!(set.lookup_hash(10), Some("o-a"));
497 assert_eq!(set.lookup_hash(20), Some("o-b"));
498 }
499
500 #[test]
501 fn object_order_preserved() {
502 let src = tempfile::TempDir::new().unwrap();
503 let out = tempfile::TempDir::new().unwrap();
504 let lex = lex_for(&[(1, "p")]);
505 let root = write_two_child_root(
506 src.path(),
507 &[quin(2), quin(4)],
508 &lex,
509 false,
510 &[quin(6), quin(8)],
511 &lex,
512 false,
513 );
514
515 let compacted = compact_volume_set(&root, out.path()).unwrap();
516 let objects: Vec<u64> = compacted_quins(&compacted)
517 .iter()
518 .map(|q| q.object)
519 .collect();
520 assert_eq!(objects, vec![2, 4, 6, 8]);
521 assert!(objects.windows(2).all(|pair| pair[0] <= pair[1]));
522 }
523
524 #[test]
525 fn sanctuary_child_cannot_become_commons() {
526 let src = tempfile::TempDir::new().unwrap();
527 let out = tempfile::TempDir::new().unwrap();
528 let lex = lex_for(&[(1, "p")]);
529 let root = write_two_child_root(
530 src.path(),
531 &[quin(3)],
532 &lex,
533 true,
534 &[medical_quin(9)],
535 &lex,
536 false,
537 );
538
539 let compacted = compact_volume_set(&root, out.path()).unwrap();
540 let set = Q42VolumeSet::open_root(&compacted).unwrap();
541 for segment in set.segments() {
542 let flags = segment.header().flags;
543 assert_eq!(
544 flags & FLAG_PERMISSIVE_COMMONS,
545 0,
546 "Sanctuary input must not mint FLAG_PERMISSIVE_COMMONS"
547 );
548 assert_ne!(flags & FLAG_SANCTUARY, 0);
549 }
550 assert!(compacted_quins(&compacted)
551 .iter()
552 .any(quin_requires_sanctuary));
553 }
554
555 #[test]
556 fn single_file_rewrites_with_field_indexes() {
557 let src = tempfile::TempDir::new().unwrap();
558 let out = tempfile::TempDir::new().unwrap();
559 let path = src.path().join("plain.q42");
560 let lex = lex_for(&[(1, "s"), (2, "p"), (3, "o")]);
561 write_unified_volume(&path, &lex, &[(3, 3)], &[vec![quin(3)]]).unwrap();
562
563 let compacted = compact_volume_set(&path, out.path()).unwrap();
564 let volume = Q42Volume::open(&compacted).unwrap();
565 let flags = volume.header().flags;
566 assert_ne!(flags & FLAG_OBJECT_SORTED, 0);
567 assert_ne!(flags & FLAG_FIELD_RANGES, 0);
568 assert_ne!(flags & FLAG_FIELD_POSTINGS, 0);
569 assert_eq!(volume.read_all_quins().unwrap()[0].object, 3);
570 assert_eq!(volume.lex_view().unwrap().lookup_hash(3), Some("o"));
571 }
572}