Skip to main content

qualia_core_db/q42/volume/
verify.rs

1//! Layered Q42 verification. `full` cannot PASS if any required check was skipped.
2//!
3//! A volume-set root is not a complete artifact by itself. `verify` walks the
4//! root, every data child, and every lexicon shard, and checks the SHA-256
5//! values stored in the root manifest.
6
7use std::io;
8use std::path::Path;
9
10use serde::Serialize;
11
12use super::super::{Q42Volume, FLAG_FIELD_POSTINGS, FLAG_FIELD_RANGES, FLAG_OBJECT_SORTED};
13use super::manifest::Q42VolumeSet;
14use super::postings::validate_postings_section;
15
16#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
17#[serde(rename_all = "snake_case")]
18pub enum VerifyLevel {
19    Structure,
20    Blocks,
21    Lexicon,
22    Indexes,
23    Full,
24}
25
26impl VerifyLevel {
27    /// Parses a verify level name; error on unknown.
28    pub fn parse(raw: &str) -> Result<Self, String> {
29        match raw.trim().to_ascii_lowercase().as_str() {
30            "structure" => Ok(Self::Structure),
31            "blocks" => Ok(Self::Blocks),
32            "lexicon" => Ok(Self::Lexicon),
33            "indexes" => Ok(Self::Indexes),
34            "full" => Ok(Self::Full),
35            other => Err(format!("unknown verify level '{other}'")),
36        }
37    }
38}
39
40#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
41#[serde(rename_all = "snake_case")]
42pub enum CheckStatus {
43    Pass,
44    Fail,
45    NotApplicable,
46    NotChecked,
47    Incomplete,
48}
49
50#[derive(Clone, Debug, Serialize)]
51pub struct VerifyCheck {
52    pub name: String,
53    pub status: CheckStatus,
54    pub detail: String,
55}
56
57#[derive(Clone, Debug, Serialize)]
58pub struct Q42VerifyReceipt {
59    pub path: String,
60    pub level: VerifyLevel,
61    pub overall: CheckStatus,
62    pub checks: Vec<VerifyCheck>,
63}
64
65#[derive(Clone, Copy, Debug, Eq, PartialEq)]
66enum FileRole {
67    Standalone,
68    VolumeRoot,
69    DataChild { shared_lexicon: bool },
70    LexiconShard,
71}
72
73impl Q42VerifyReceipt {
74    /// Verifies the volume at `path` at `level`.
75    pub fn from_path(path: &Path, level: VerifyLevel) -> io::Result<Self> {
76        let volume = Q42Volume::open(path)?;
77        Ok(Self::from_volume(path, &volume, level))
78    }
79
80    /// Builds the receipt for an already-open `volume`.
81    pub fn from_volume(path: &Path, volume: &Q42Volume, level: VerifyLevel) -> Self {
82        let role = if volume.volume_manifest().ok().flatten().is_some() {
83            FileRole::VolumeRoot
84        } else {
85            FileRole::Standalone
86        };
87        Self::from_volume_role(path, volume, level, role)
88    }
89
90    fn from_volume_role(
91        path: &Path,
92        volume: &Q42Volume,
93        level: VerifyLevel,
94        role: FileRole,
95    ) -> Self {
96        let mut checks = Vec::new();
97        let run_structure = matches!(
98            level,
99            VerifyLevel::Structure
100                | VerifyLevel::Full
101                | VerifyLevel::Blocks
102                | VerifyLevel::Indexes
103                | VerifyLevel::Lexicon
104        );
105        let run_blocks = matches!(level, VerifyLevel::Blocks | VerifyLevel::Full);
106        let run_lex = matches!(level, VerifyLevel::Lexicon | VerifyLevel::Full);
107        let run_idx = matches!(level, VerifyLevel::Indexes | VerifyLevel::Full);
108
109        if run_structure {
110            checks.push(check(
111                "structure.open",
112                CheckStatus::Pass,
113                "header and section bounds accepted on open",
114            ));
115            let flags = volume.header().flags;
116            if volume.block_count() > 0 && flags & FLAG_OBJECT_SORTED == 0 {
117                checks.push(check(
118                    "structure.object_sorted",
119                    CheckStatus::Fail,
120                    "blocks present but object-sorted flag is clear",
121                ));
122            } else {
123                checks.push(check(
124                    "structure.object_sorted",
125                    CheckStatus::Pass,
126                    "object sort flag matches block presence",
127                ));
128            }
129        }
130
131        if run_blocks {
132            if volume.volume_manifest().ok().flatten().is_some() {
133                checks.push(check(
134                    "blocks.decode",
135                    CheckStatus::NotApplicable,
136                    "volume root has no local SuperBlocks; verify children",
137                ));
138            } else {
139                match volume.verify_all_blocks() {
140                    Ok(receipt) => checks.push(check(
141                        "blocks.decode",
142                        CheckStatus::Pass,
143                        format!(
144                            "{} blocks, {} Quins, parity and object order ok",
145                            receipt.blocks_verified, receipt.quins_verified
146                        ),
147                    )),
148                    Err(error) => {
149                        checks.push(check("blocks.decode", CheckStatus::Fail, error.to_string()))
150                    }
151                }
152            }
153        } else if level == VerifyLevel::Full {
154            checks.push(check(
155                "blocks.decode",
156                CheckStatus::NotChecked,
157                "block decode was not requested",
158            ));
159        }
160
161        if run_lex {
162            match volume.lex_view() {
163                Ok(lex) => {
164                    let entries = lex.entry_count();
165                    let (status, detail) = match role {
166                        FileRole::DataChild {
167                            shared_lexicon: true,
168                        } if entries == 0 => (
169                            CheckStatus::Pass,
170                            "0 local terms; this data child uses the volume-set lexicon shards"
171                                .into(),
172                        ),
173                        FileRole::VolumeRoot if entries == 0 => (
174                            CheckStatus::Pass,
175                            "root catalog has no local terms; lexicon shards are verified separately"
176                                .into(),
177                        ),
178                        FileRole::LexiconShard if entries == 0 => (
179                            CheckStatus::Fail,
180                            "lexicon shard contains 0 terms".into(),
181                        ),
182                        FileRole::Standalone | FileRole::DataChild { shared_lexicon: false }
183                            if entries == 0 && volume.block_count() > 0 =>
184                        {
185                            (
186                                CheckStatus::Incomplete,
187                                "data file has 0 embedded terms and no shared lexicon shards"
188                                    .into(),
189                            )
190                        }
191                        _ => (
192                            CheckStatus::Pass,
193                            format!("{entries} recoverable terms in embedded Q42LEX"),
194                        ),
195                    };
196                    checks.push(check("lexicon.entries", status, detail));
197                }
198                Err(error) => checks.push(check(
199                    "lexicon.entries",
200                    CheckStatus::Fail,
201                    format!("{error:?}"),
202                )),
203            }
204        }
205
206        if run_structure {
207            let merkle = volume.header().merkle_root;
208            let dag_len = volume.header().dag_root_length;
209            let merkle_empty = merkle.iter().all(|b| *b == 0) && dag_len == 0;
210            let (status, detail) = match (role, merkle_empty) {
211                (FileRole::Standalone, true) if volume.block_count() > 0 => (
212                    CheckStatus::Incomplete,
213                    "merkle_root and DAG are empty on a standalone data file",
214                ),
215                (FileRole::DataChild { .. } | FileRole::VolumeRoot, true) => (
216                    CheckStatus::NotApplicable,
217                    "per-file DAG empty; volume-set identity is the root manifest SHA-256",
218                ),
219                (FileRole::LexiconShard, true) => (
220                    CheckStatus::NotApplicable,
221                    "lexicon shard has no SuperBlock DAG",
222                ),
223                (_, false) => (
224                    CheckStatus::Pass,
225                    "header merkle_root / DAG section present",
226                ),
227                _ => (CheckStatus::NotApplicable, "no data blocks"),
228            };
229            checks.push(check("structure.merkle", status, detail));
230        }
231
232        if run_idx {
233            let bidx = volume.bidx_bytes();
234            if bidx.is_empty() && volume.block_count() > 0 {
235                checks.push(check(
236                    "indexes.bidx",
237                    CheckStatus::Fail,
238                    "blocks present but BIDX is empty",
239                ));
240            } else if bidx.is_empty() {
241                checks.push(check(
242                    "indexes.bidx",
243                    CheckStatus::NotApplicable,
244                    "no blocks",
245                ));
246            } else {
247                match super::index::validate_bidx(bidx, volume.block_count() as usize) {
248                    Ok(()) => checks.push(check(
249                        "indexes.bidx",
250                        CheckStatus::Pass,
251                        "BIDX layout and monotonicity ok",
252                    )),
253                    Err(error) => {
254                        checks.push(check("indexes.bidx", CheckStatus::Fail, error.to_string()))
255                    }
256                }
257            }
258            let flags = volume.header().flags;
259            if flags & FLAG_FIELD_RANGES != 0 {
260                checks.push(check(
261                    "indexes.field_ranges",
262                    CheckStatus::Pass,
263                    "FIDX present and accepted on open",
264                ));
265            } else if volume.block_count() > 0 && !matches!(role, FileRole::LexiconShard) {
266                checks.push(check(
267                    "indexes.field_ranges",
268                    CheckStatus::Incomplete,
269                    "no FIDX on a data segment",
270                ));
271            }
272            if flags & FLAG_FIELD_POSTINGS != 0 {
273                if let Some((offset, length)) = volume.header().field_postings_range() {
274                    let start = offset as usize;
275                    let end = start + length as usize;
276                    match volume.as_bytes().get(start..end) {
277                        Some(bytes) => {
278                            match validate_postings_section(bytes, volume.block_count() as usize) {
279                                Ok(()) => checks.push(check(
280                                    "indexes.postings",
281                                    CheckStatus::Pass,
282                                    "PIDX layout ok",
283                                )),
284                                Err(error) => checks.push(check(
285                                    "indexes.postings",
286                                    CheckStatus::Fail,
287                                    error.to_string(),
288                                )),
289                            }
290                        }
291                        None => checks.push(check(
292                            "indexes.postings",
293                            CheckStatus::Fail,
294                            "PIDX range outside file",
295                        )),
296                    }
297                }
298            } else if volume.block_count() > 0 && !matches!(role, FileRole::LexiconShard) {
299                checks.push(check(
300                    "indexes.postings",
301                    CheckStatus::Incomplete,
302                    "no PIDX on a data segment",
303                ));
304            }
305        }
306
307        let overall = overall_status(level, &checks);
308        Self {
309            path: path.display().to_string(),
310            level,
311            overall,
312            checks,
313        }
314    }
315
316    /// Human-readable verification report.
317    pub fn to_text(&self) -> String {
318        let mut out = format!(
319            "Q42 verify  {}  level={:?}  overall={:?}\n",
320            self.path, self.level, self.overall
321        );
322        for check in &self.checks {
323            out.push_str(&format!(
324                "  {:<22} {:<14} {}\n",
325                check.name,
326                format!("{:?}", check.status),
327                check.detail
328            ));
329        }
330        out
331    }
332}
333
334fn check(name: &str, status: CheckStatus, detail: impl Into<String>) -> VerifyCheck {
335    VerifyCheck {
336        name: name.into(),
337        status,
338        detail: detail.into(),
339    }
340}
341
342fn overall_status(level: VerifyLevel, checks: &[VerifyCheck]) -> CheckStatus {
343    if checks.iter().any(|c| c.status == CheckStatus::Fail) {
344        return CheckStatus::Fail;
345    }
346    if level == VerifyLevel::Full {
347        if checks
348            .iter()
349            .any(|c| matches!(c.status, CheckStatus::NotChecked | CheckStatus::Incomplete))
350        {
351            return CheckStatus::Incomplete;
352        }
353    }
354    if checks.iter().any(|c| c.status == CheckStatus::Incomplete) {
355        return CheckStatus::Incomplete;
356    }
357    CheckStatus::Pass
358}
359
360/// One root plus every physical child named by its catalog.
361#[derive(Clone, Debug, Serialize)]
362pub struct Q42VerifySetReport {
363    pub root: String,
364    pub overall: CheckStatus,
365    pub members: Vec<Q42VerifyReceipt>,
366}
367
368impl Q42VerifySetReport {
369    /// Human-readable multi-volume report.
370    pub fn to_text(&self) -> String {
371        let mut out = format!(
372            "Q42 verify-set  {}  members={}  overall={:?}\n",
373            self.root,
374            self.members.len(),
375            self.overall
376        );
377        for receipt in &self.members {
378            out.push_str(&receipt.to_text());
379        }
380        out
381    }
382}
383
384/// Verify a standalone file, or a volume-set root and every named child.
385pub fn verify_volume_set_from_root(
386    path: &Path,
387    level: VerifyLevel,
388) -> io::Result<Q42VerifySetReport> {
389    let root = Q42Volume::open(path)?;
390    let Some(manifest) = root.volume_manifest()? else {
391        let receipt = Q42VerifyReceipt::from_volume(path, &root, level);
392        return Ok(Q42VerifySetReport {
393            overall: receipt.overall,
394            root: path.display().to_string(),
395            members: vec![receipt],
396        });
397    };
398
399    let parent = path.parent().unwrap_or(Path::new("."));
400    let shared_lexicon = !manifest.lexicon_segments.is_empty();
401    let mut members = vec![Q42VerifyReceipt::from_volume_role(
402        path,
403        &root,
404        level,
405        FileRole::VolumeRoot,
406    )];
407
408    let mut digest_ok = true;
409    let digest_detail;
410    match Q42VolumeSet::open_root(path) {
411        Ok(set) => match set.verify_segment_hashes(path) {
412            Ok(()) => {
413                digest_detail = format!(
414                    "{} data + {} lexicon shard digest(s) match the root manifest",
415                    manifest.segments.len(),
416                    manifest.lexicon_segments.len()
417                )
418            }
419            Err(error) => {
420                digest_ok = false;
421                digest_detail = error.to_string();
422            }
423        },
424        Err(error) => {
425            digest_ok = false;
426            digest_detail = error.to_string();
427        }
428    }
429    members[0].checks.push(check(
430        "set.digests",
431        if digest_ok {
432            CheckStatus::Pass
433        } else {
434            CheckStatus::Fail
435        },
436        digest_detail,
437    ));
438    members[0].overall = overall_status(level, &members[0].checks);
439
440    for entry in &manifest.segments {
441        let child = parent.join(&entry.locator);
442        if !child.is_file() {
443            members.push(Q42VerifyReceipt {
444                path: child.display().to_string(),
445                level,
446                overall: CheckStatus::Fail,
447                checks: vec![check(
448                    "set.member",
449                    CheckStatus::Fail,
450                    "data child is missing",
451                )],
452            });
453            continue;
454        }
455        let volume = Q42Volume::open(&child)?;
456        members.push(Q42VerifyReceipt::from_volume_role(
457            &child,
458            &volume,
459            level,
460            FileRole::DataChild { shared_lexicon },
461        ));
462    }
463    for entry in &manifest.lexicon_segments {
464        let child = parent.join(&entry.locator);
465        if !child.is_file() {
466            members.push(Q42VerifyReceipt {
467                path: child.display().to_string(),
468                level,
469                overall: CheckStatus::Fail,
470                checks: vec![check(
471                    "set.member",
472                    CheckStatus::Fail,
473                    "lexicon shard is missing",
474                )],
475            });
476            continue;
477        }
478        let volume = Q42Volume::open(&child)?;
479        members.push(Q42VerifyReceipt::from_volume_role(
480            &child,
481            &volume,
482            level,
483            FileRole::LexiconShard,
484        ));
485    }
486
487    let overall = fold_set_overall(level, &members);
488    Ok(Q42VerifySetReport {
489        root: path.display().to_string(),
490        overall,
491        members,
492    })
493}
494
495fn fold_set_overall(level: VerifyLevel, members: &[Q42VerifyReceipt]) -> CheckStatus {
496    if members.iter().any(|m| m.overall == CheckStatus::Fail) {
497        return CheckStatus::Fail;
498    }
499    if members.iter().any(|m| m.overall == CheckStatus::Incomplete)
500        || (level == VerifyLevel::Full
501            && members
502                .iter()
503                .any(|m| m.checks.iter().any(|c| c.status == CheckStatus::NotChecked)))
504    {
505        return CheckStatus::Incomplete;
506    }
507    CheckStatus::Pass
508}
509
510#[cfg(test)]
511mod tests {
512    use super::*;
513    use crate::q42_volume::{write_unified_volume, StreamingQ42VolumeWriter};
514    use crate::NQuin;
515    use std::collections::HashMap;
516
517    #[test]
518    fn full_on_hashed_only_file_is_incomplete_not_pass() {
519        let file = tempfile::NamedTempFile::new().unwrap();
520        write_unified_volume(
521            file.path(),
522            &HashMap::new(),
523            &[(3, 3)],
524            &[vec![NQuin {
525                subject: 1,
526                predicate: 2,
527                object: 3,
528                context: 0,
529                metadata: 0,
530                parity: 1 ^ 2 ^ 3,
531            }]],
532        )
533        .unwrap();
534        let receipt = Q42VerifyReceipt::from_path(file.path(), VerifyLevel::Full).unwrap();
535        assert_eq!(receipt.overall, CheckStatus::Incomplete);
536        assert!(receipt.checks.iter().any(|c| c.name == "lexicon.entries"));
537    }
538
539    #[test]
540    fn full_on_lex_and_postings_passes() {
541        let file = tempfile::NamedTempFile::new().unwrap();
542        let mut lex = HashMap::new();
543        lex.insert(1, "s".into());
544        lex.insert(2, "p".into());
545        lex.insert(3, "o".into());
546        let mut writer = StreamingQ42VolumeWriter::new(&lex).unwrap();
547        writer
548            .push_block(
549                0,
550                &[NQuin {
551                    subject: 1,
552                    predicate: 2,
553                    object: 3,
554                    context: 0,
555                    metadata: 0,
556                    parity: 1 ^ 2 ^ 3,
557                }],
558            )
559            .unwrap();
560        writer.finish(file.path()).unwrap();
561        let receipt = Q42VerifyReceipt::from_path(file.path(), VerifyLevel::Full).unwrap();
562        assert_eq!(receipt.overall, CheckStatus::Pass, "{:?}", receipt.checks);
563    }
564
565    #[test]
566    fn volume_set_full_passes_with_shared_lexicon() {
567        use crate::q42_volume::{
568            write_volume_root_for_commons, Q42VolumeManifest, StreamingQ42VolumeWriter,
569        };
570
571        let dir = tempfile::TempDir::new().unwrap();
572        let mut lex = HashMap::new();
573        lex.insert(1, "s".into());
574        lex.insert(2, "p".into());
575        lex.insert(3, "o".into());
576        let child = dir.path().join("child.q42");
577        let mut data = StreamingQ42VolumeWriter::new(&HashMap::new()).unwrap();
578        data.declare_permissive_commons();
579        data.push_block(
580            0,
581            &[NQuin {
582                subject: 1,
583                predicate: 2,
584                object: 3,
585                context: 0,
586                metadata: 0,
587                parity: 1 ^ 2 ^ 3,
588            }],
589        )
590        .unwrap();
591        data.finish(&child).unwrap();
592        let shard = dir.path().join("lex-00000.q42");
593        let mut words = StreamingQ42VolumeWriter::new(&lex).unwrap();
594        words.declare_permissive_commons();
595        words.finish(&shard).unwrap();
596        let root = dir.path().join("root.q42");
597        write_volume_root_for_commons(
598            &root,
599            &Q42VolumeManifest {
600                generation: 1,
601                segments: vec![
602                    Q42VolumeManifest::segment_from_file(&child, "child.q42".into()).unwrap(),
603                ],
604                lexicon_segments: vec![Q42VolumeManifest::lexicon_segment_from_file(
605                    &shard,
606                    "lex-00000.q42".into(),
607                )
608                .unwrap()],
609            },
610        )
611        .unwrap();
612
613        let report = verify_volume_set_from_root(&root, VerifyLevel::Full).unwrap();
614        assert_eq!(report.members.len(), 3, "{:?}", report.members);
615        assert_eq!(report.overall, CheckStatus::Pass, "{}", report.to_text());
616        assert!(report.members.iter().any(|m| m
617            .checks
618            .iter()
619            .any(|c| c.name == "set.digests" && c.status == CheckStatus::Pass)));
620    }
621}