1use std::io;
8use std::path::Path;
9
10use serde::Serialize;
11
12use super::super::{Q42Volume, FLAG_FIELD_POSTINGS, FLAG_FIELD_RANGES, FLAG_OBJECT_SORTED};
13use super::manifest::Q42VolumeSet;
14use super::postings::validate_postings_section;
15
16#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
17#[serde(rename_all = "snake_case")]
18pub enum VerifyLevel {
19 Structure,
20 Blocks,
21 Lexicon,
22 Indexes,
23 Full,
24}
25
26impl VerifyLevel {
27 pub fn parse(raw: &str) -> Result<Self, String> {
29 match raw.trim().to_ascii_lowercase().as_str() {
30 "structure" => Ok(Self::Structure),
31 "blocks" => Ok(Self::Blocks),
32 "lexicon" => Ok(Self::Lexicon),
33 "indexes" => Ok(Self::Indexes),
34 "full" => Ok(Self::Full),
35 other => Err(format!("unknown verify level '{other}'")),
36 }
37 }
38}
39
40#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
41#[serde(rename_all = "snake_case")]
42pub enum CheckStatus {
43 Pass,
44 Fail,
45 NotApplicable,
46 NotChecked,
47 Incomplete,
48}
49
50#[derive(Clone, Debug, Serialize)]
51pub struct VerifyCheck {
52 pub name: String,
53 pub status: CheckStatus,
54 pub detail: String,
55}
56
57#[derive(Clone, Debug, Serialize)]
58pub struct Q42VerifyReceipt {
59 pub path: String,
60 pub level: VerifyLevel,
61 pub overall: CheckStatus,
62 pub checks: Vec<VerifyCheck>,
63}
64
65#[derive(Clone, Copy, Debug, Eq, PartialEq)]
66enum FileRole {
67 Standalone,
68 VolumeRoot,
69 DataChild { shared_lexicon: bool },
70 LexiconShard,
71}
72
73impl Q42VerifyReceipt {
74 pub fn from_path(path: &Path, level: VerifyLevel) -> io::Result<Self> {
76 let volume = Q42Volume::open(path)?;
77 Ok(Self::from_volume(path, &volume, level))
78 }
79
80 pub fn from_volume(path: &Path, volume: &Q42Volume, level: VerifyLevel) -> Self {
82 let role = if volume.volume_manifest().ok().flatten().is_some() {
83 FileRole::VolumeRoot
84 } else {
85 FileRole::Standalone
86 };
87 Self::from_volume_role(path, volume, level, role)
88 }
89
90 fn from_volume_role(
91 path: &Path,
92 volume: &Q42Volume,
93 level: VerifyLevel,
94 role: FileRole,
95 ) -> Self {
96 let mut checks = Vec::new();
97 let run_structure = matches!(
98 level,
99 VerifyLevel::Structure
100 | VerifyLevel::Full
101 | VerifyLevel::Blocks
102 | VerifyLevel::Indexes
103 | VerifyLevel::Lexicon
104 );
105 let run_blocks = matches!(level, VerifyLevel::Blocks | VerifyLevel::Full);
106 let run_lex = matches!(level, VerifyLevel::Lexicon | VerifyLevel::Full);
107 let run_idx = matches!(level, VerifyLevel::Indexes | VerifyLevel::Full);
108
109 if run_structure {
110 checks.push(check(
111 "structure.open",
112 CheckStatus::Pass,
113 "header and section bounds accepted on open",
114 ));
115 let flags = volume.header().flags;
116 if volume.block_count() > 0 && flags & FLAG_OBJECT_SORTED == 0 {
117 checks.push(check(
118 "structure.object_sorted",
119 CheckStatus::Fail,
120 "blocks present but object-sorted flag is clear",
121 ));
122 } else {
123 checks.push(check(
124 "structure.object_sorted",
125 CheckStatus::Pass,
126 "object sort flag matches block presence",
127 ));
128 }
129 }
130
131 if run_blocks {
132 if volume.volume_manifest().ok().flatten().is_some() {
133 checks.push(check(
134 "blocks.decode",
135 CheckStatus::NotApplicable,
136 "volume root has no local SuperBlocks; verify children",
137 ));
138 } else {
139 match volume.verify_all_blocks() {
140 Ok(receipt) => checks.push(check(
141 "blocks.decode",
142 CheckStatus::Pass,
143 format!(
144 "{} blocks, {} Quins, parity and object order ok",
145 receipt.blocks_verified, receipt.quins_verified
146 ),
147 )),
148 Err(error) => {
149 checks.push(check("blocks.decode", CheckStatus::Fail, error.to_string()))
150 }
151 }
152 }
153 } else if level == VerifyLevel::Full {
154 checks.push(check(
155 "blocks.decode",
156 CheckStatus::NotChecked,
157 "block decode was not requested",
158 ));
159 }
160
161 if run_lex {
162 match volume.lex_view() {
163 Ok(lex) => {
164 let entries = lex.entry_count();
165 let (status, detail) = match role {
166 FileRole::DataChild {
167 shared_lexicon: true,
168 } if entries == 0 => (
169 CheckStatus::Pass,
170 "0 local terms; this data child uses the volume-set lexicon shards"
171 .into(),
172 ),
173 FileRole::VolumeRoot if entries == 0 => (
174 CheckStatus::Pass,
175 "root catalog has no local terms; lexicon shards are verified separately"
176 .into(),
177 ),
178 FileRole::LexiconShard if entries == 0 => (
179 CheckStatus::Fail,
180 "lexicon shard contains 0 terms".into(),
181 ),
182 FileRole::Standalone | FileRole::DataChild { shared_lexicon: false }
183 if entries == 0 && volume.block_count() > 0 =>
184 {
185 (
186 CheckStatus::Incomplete,
187 "data file has 0 embedded terms and no shared lexicon shards"
188 .into(),
189 )
190 }
191 _ => (
192 CheckStatus::Pass,
193 format!("{entries} recoverable terms in embedded Q42LEX"),
194 ),
195 };
196 checks.push(check("lexicon.entries", status, detail));
197 }
198 Err(error) => checks.push(check(
199 "lexicon.entries",
200 CheckStatus::Fail,
201 format!("{error:?}"),
202 )),
203 }
204 }
205
206 if run_structure {
207 let merkle = volume.header().merkle_root;
208 let dag_len = volume.header().dag_root_length;
209 let merkle_empty = merkle.iter().all(|b| *b == 0) && dag_len == 0;
210 let (status, detail) = match (role, merkle_empty) {
211 (FileRole::Standalone, true) if volume.block_count() > 0 => (
212 CheckStatus::Incomplete,
213 "merkle_root and DAG are empty on a standalone data file",
214 ),
215 (FileRole::DataChild { .. } | FileRole::VolumeRoot, true) => (
216 CheckStatus::NotApplicable,
217 "per-file DAG empty; volume-set identity is the root manifest SHA-256",
218 ),
219 (FileRole::LexiconShard, true) => (
220 CheckStatus::NotApplicable,
221 "lexicon shard has no SuperBlock DAG",
222 ),
223 (_, false) => (
224 CheckStatus::Pass,
225 "header merkle_root / DAG section present",
226 ),
227 _ => (CheckStatus::NotApplicable, "no data blocks"),
228 };
229 checks.push(check("structure.merkle", status, detail));
230 }
231
232 if run_idx {
233 let bidx = volume.bidx_bytes();
234 if bidx.is_empty() && volume.block_count() > 0 {
235 checks.push(check(
236 "indexes.bidx",
237 CheckStatus::Fail,
238 "blocks present but BIDX is empty",
239 ));
240 } else if bidx.is_empty() {
241 checks.push(check(
242 "indexes.bidx",
243 CheckStatus::NotApplicable,
244 "no blocks",
245 ));
246 } else {
247 match super::index::validate_bidx(bidx, volume.block_count() as usize) {
248 Ok(()) => checks.push(check(
249 "indexes.bidx",
250 CheckStatus::Pass,
251 "BIDX layout and monotonicity ok",
252 )),
253 Err(error) => {
254 checks.push(check("indexes.bidx", CheckStatus::Fail, error.to_string()))
255 }
256 }
257 }
258 let flags = volume.header().flags;
259 if flags & FLAG_FIELD_RANGES != 0 {
260 checks.push(check(
261 "indexes.field_ranges",
262 CheckStatus::Pass,
263 "FIDX present and accepted on open",
264 ));
265 } else if volume.block_count() > 0 && !matches!(role, FileRole::LexiconShard) {
266 checks.push(check(
267 "indexes.field_ranges",
268 CheckStatus::Incomplete,
269 "no FIDX on a data segment",
270 ));
271 }
272 if flags & FLAG_FIELD_POSTINGS != 0 {
273 if let Some((offset, length)) = volume.header().field_postings_range() {
274 let start = offset as usize;
275 let end = start + length as usize;
276 match volume.as_bytes().get(start..end) {
277 Some(bytes) => {
278 match validate_postings_section(bytes, volume.block_count() as usize) {
279 Ok(()) => checks.push(check(
280 "indexes.postings",
281 CheckStatus::Pass,
282 "PIDX layout ok",
283 )),
284 Err(error) => checks.push(check(
285 "indexes.postings",
286 CheckStatus::Fail,
287 error.to_string(),
288 )),
289 }
290 }
291 None => checks.push(check(
292 "indexes.postings",
293 CheckStatus::Fail,
294 "PIDX range outside file",
295 )),
296 }
297 }
298 } else if volume.block_count() > 0 && !matches!(role, FileRole::LexiconShard) {
299 checks.push(check(
300 "indexes.postings",
301 CheckStatus::Incomplete,
302 "no PIDX on a data segment",
303 ));
304 }
305 }
306
307 let overall = overall_status(level, &checks);
308 Self {
309 path: path.display().to_string(),
310 level,
311 overall,
312 checks,
313 }
314 }
315
316 pub fn to_text(&self) -> String {
318 let mut out = format!(
319 "Q42 verify {} level={:?} overall={:?}\n",
320 self.path, self.level, self.overall
321 );
322 for check in &self.checks {
323 out.push_str(&format!(
324 " {:<22} {:<14} {}\n",
325 check.name,
326 format!("{:?}", check.status),
327 check.detail
328 ));
329 }
330 out
331 }
332}
333
334fn check(name: &str, status: CheckStatus, detail: impl Into<String>) -> VerifyCheck {
335 VerifyCheck {
336 name: name.into(),
337 status,
338 detail: detail.into(),
339 }
340}
341
342fn overall_status(level: VerifyLevel, checks: &[VerifyCheck]) -> CheckStatus {
343 if checks.iter().any(|c| c.status == CheckStatus::Fail) {
344 return CheckStatus::Fail;
345 }
346 if level == VerifyLevel::Full {
347 if checks
348 .iter()
349 .any(|c| matches!(c.status, CheckStatus::NotChecked | CheckStatus::Incomplete))
350 {
351 return CheckStatus::Incomplete;
352 }
353 }
354 if checks.iter().any(|c| c.status == CheckStatus::Incomplete) {
355 return CheckStatus::Incomplete;
356 }
357 CheckStatus::Pass
358}
359
360#[derive(Clone, Debug, Serialize)]
362pub struct Q42VerifySetReport {
363 pub root: String,
364 pub overall: CheckStatus,
365 pub members: Vec<Q42VerifyReceipt>,
366}
367
368impl Q42VerifySetReport {
369 pub fn to_text(&self) -> String {
371 let mut out = format!(
372 "Q42 verify-set {} members={} overall={:?}\n",
373 self.root,
374 self.members.len(),
375 self.overall
376 );
377 for receipt in &self.members {
378 out.push_str(&receipt.to_text());
379 }
380 out
381 }
382}
383
384pub fn verify_volume_set_from_root(
386 path: &Path,
387 level: VerifyLevel,
388) -> io::Result<Q42VerifySetReport> {
389 let root = Q42Volume::open(path)?;
390 let Some(manifest) = root.volume_manifest()? else {
391 let receipt = Q42VerifyReceipt::from_volume(path, &root, level);
392 return Ok(Q42VerifySetReport {
393 overall: receipt.overall,
394 root: path.display().to_string(),
395 members: vec![receipt],
396 });
397 };
398
399 let parent = path.parent().unwrap_or(Path::new("."));
400 let shared_lexicon = !manifest.lexicon_segments.is_empty();
401 let mut members = vec![Q42VerifyReceipt::from_volume_role(
402 path,
403 &root,
404 level,
405 FileRole::VolumeRoot,
406 )];
407
408 let mut digest_ok = true;
409 let digest_detail;
410 match Q42VolumeSet::open_root(path) {
411 Ok(set) => match set.verify_segment_hashes(path) {
412 Ok(()) => {
413 digest_detail = format!(
414 "{} data + {} lexicon shard digest(s) match the root manifest",
415 manifest.segments.len(),
416 manifest.lexicon_segments.len()
417 )
418 }
419 Err(error) => {
420 digest_ok = false;
421 digest_detail = error.to_string();
422 }
423 },
424 Err(error) => {
425 digest_ok = false;
426 digest_detail = error.to_string();
427 }
428 }
429 members[0].checks.push(check(
430 "set.digests",
431 if digest_ok {
432 CheckStatus::Pass
433 } else {
434 CheckStatus::Fail
435 },
436 digest_detail,
437 ));
438 members[0].overall = overall_status(level, &members[0].checks);
439
440 for entry in &manifest.segments {
441 let child = parent.join(&entry.locator);
442 if !child.is_file() {
443 members.push(Q42VerifyReceipt {
444 path: child.display().to_string(),
445 level,
446 overall: CheckStatus::Fail,
447 checks: vec![check(
448 "set.member",
449 CheckStatus::Fail,
450 "data child is missing",
451 )],
452 });
453 continue;
454 }
455 let volume = Q42Volume::open(&child)?;
456 members.push(Q42VerifyReceipt::from_volume_role(
457 &child,
458 &volume,
459 level,
460 FileRole::DataChild { shared_lexicon },
461 ));
462 }
463 for entry in &manifest.lexicon_segments {
464 let child = parent.join(&entry.locator);
465 if !child.is_file() {
466 members.push(Q42VerifyReceipt {
467 path: child.display().to_string(),
468 level,
469 overall: CheckStatus::Fail,
470 checks: vec![check(
471 "set.member",
472 CheckStatus::Fail,
473 "lexicon shard is missing",
474 )],
475 });
476 continue;
477 }
478 let volume = Q42Volume::open(&child)?;
479 members.push(Q42VerifyReceipt::from_volume_role(
480 &child,
481 &volume,
482 level,
483 FileRole::LexiconShard,
484 ));
485 }
486
487 let overall = fold_set_overall(level, &members);
488 Ok(Q42VerifySetReport {
489 root: path.display().to_string(),
490 overall,
491 members,
492 })
493}
494
495fn fold_set_overall(level: VerifyLevel, members: &[Q42VerifyReceipt]) -> CheckStatus {
496 if members.iter().any(|m| m.overall == CheckStatus::Fail) {
497 return CheckStatus::Fail;
498 }
499 if members.iter().any(|m| m.overall == CheckStatus::Incomplete)
500 || (level == VerifyLevel::Full
501 && members
502 .iter()
503 .any(|m| m.checks.iter().any(|c| c.status == CheckStatus::NotChecked)))
504 {
505 return CheckStatus::Incomplete;
506 }
507 CheckStatus::Pass
508}
509
510#[cfg(test)]
511mod tests {
512 use super::*;
513 use crate::q42_volume::{write_unified_volume, StreamingQ42VolumeWriter};
514 use crate::NQuin;
515 use std::collections::HashMap;
516
517 #[test]
518 fn full_on_hashed_only_file_is_incomplete_not_pass() {
519 let file = tempfile::NamedTempFile::new().unwrap();
520 write_unified_volume(
521 file.path(),
522 &HashMap::new(),
523 &[(3, 3)],
524 &[vec![NQuin {
525 subject: 1,
526 predicate: 2,
527 object: 3,
528 context: 0,
529 metadata: 0,
530 parity: 1 ^ 2 ^ 3,
531 }]],
532 )
533 .unwrap();
534 let receipt = Q42VerifyReceipt::from_path(file.path(), VerifyLevel::Full).unwrap();
535 assert_eq!(receipt.overall, CheckStatus::Incomplete);
536 assert!(receipt.checks.iter().any(|c| c.name == "lexicon.entries"));
537 }
538
539 #[test]
540 fn full_on_lex_and_postings_passes() {
541 let file = tempfile::NamedTempFile::new().unwrap();
542 let mut lex = HashMap::new();
543 lex.insert(1, "s".into());
544 lex.insert(2, "p".into());
545 lex.insert(3, "o".into());
546 let mut writer = StreamingQ42VolumeWriter::new(&lex).unwrap();
547 writer
548 .push_block(
549 0,
550 &[NQuin {
551 subject: 1,
552 predicate: 2,
553 object: 3,
554 context: 0,
555 metadata: 0,
556 parity: 1 ^ 2 ^ 3,
557 }],
558 )
559 .unwrap();
560 writer.finish(file.path()).unwrap();
561 let receipt = Q42VerifyReceipt::from_path(file.path(), VerifyLevel::Full).unwrap();
562 assert_eq!(receipt.overall, CheckStatus::Pass, "{:?}", receipt.checks);
563 }
564
565 #[test]
566 fn volume_set_full_passes_with_shared_lexicon() {
567 use crate::q42_volume::{
568 write_volume_root_for_commons, Q42VolumeManifest, StreamingQ42VolumeWriter,
569 };
570
571 let dir = tempfile::TempDir::new().unwrap();
572 let mut lex = HashMap::new();
573 lex.insert(1, "s".into());
574 lex.insert(2, "p".into());
575 lex.insert(3, "o".into());
576 let child = dir.path().join("child.q42");
577 let mut data = StreamingQ42VolumeWriter::new(&HashMap::new()).unwrap();
578 data.declare_permissive_commons();
579 data.push_block(
580 0,
581 &[NQuin {
582 subject: 1,
583 predicate: 2,
584 object: 3,
585 context: 0,
586 metadata: 0,
587 parity: 1 ^ 2 ^ 3,
588 }],
589 )
590 .unwrap();
591 data.finish(&child).unwrap();
592 let shard = dir.path().join("lex-00000.q42");
593 let mut words = StreamingQ42VolumeWriter::new(&lex).unwrap();
594 words.declare_permissive_commons();
595 words.finish(&shard).unwrap();
596 let root = dir.path().join("root.q42");
597 write_volume_root_for_commons(
598 &root,
599 &Q42VolumeManifest {
600 generation: 1,
601 segments: vec![
602 Q42VolumeManifest::segment_from_file(&child, "child.q42".into()).unwrap(),
603 ],
604 lexicon_segments: vec![Q42VolumeManifest::lexicon_segment_from_file(
605 &shard,
606 "lex-00000.q42".into(),
607 )
608 .unwrap()],
609 },
610 )
611 .unwrap();
612
613 let report = verify_volume_set_from_root(&root, VerifyLevel::Full).unwrap();
614 assert_eq!(report.members.len(), 3, "{:?}", report.members);
615 assert_eq!(report.overall, CheckStatus::Pass, "{}", report.to_text());
616 assert!(report.members.iter().any(|m| m
617 .checks
618 .iter()
619 .any(|c| c.name == "set.digests" && c.status == CheckStatus::Pass)));
620 }
621}