Skip to main content

qualia_core_db/query/
source_audit.rs

1//! Pre-ingest source audit: triple counts, oversize terms, predicate histogram,
2//! and RDF-vs-unsupported classification for ontology library roots.
3
4use std::collections::HashMap;
5use std::fs::File;
6use std::io::{self, BufReader};
7use std::path::{Path, PathBuf};
8
9use serde::Serialize;
10
11use crate::q_hash;
12use crate::query::ingest_formats::{format_from_path, parse_triples_format, RawTriple};
13use crate::query::ingest_job::IngestRdfFormat;
14use crate::query::integrity_omit::MAX_LEX_TERM_BYTES;
15
16/// How a path participates in Cross-Format Validation today.
17#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
18#[serde(rename_all = "snake_case")]
19pub enum SourceSupport {
20    RdfImportVerify,
21    ZipBag,
22    UnsupportedTabular,
23    UnsupportedOther,
24    Missing,
25}
26
27#[derive(Clone, Debug, Serialize)]
28pub struct PredicateCount {
29    pub iri: String,
30    pub count: u64,
31}
32
33#[derive(Clone, Debug, Serialize)]
34pub struct SourceAuditReport {
35    pub path: String,
36    pub format: String,
37    pub support: SourceSupport,
38    pub source_triple_count: u64,
39    pub skipped_or_filtered: u64,
40    pub oversize_literal_count: u64,
41    pub first_oversize_preview: Option<String>,
42    pub blank_node_triples: u64,
43    pub top_predicates: Vec<PredicateCount>,
44    pub comment_gloss_predicate_hits: u64,
45    pub notes: Vec<String>,
46}
47
48#[derive(Clone, Debug, Serialize)]
49pub struct LibraryEntry {
50    pub path: String,
51    pub bytes: u64,
52    pub support: SourceSupport,
53    pub format_hint: String,
54    pub note: String,
55}
56
57#[derive(Clone, Debug, Serialize)]
58pub struct LibraryAuditMatrix {
59    pub source_root: String,
60    pub entries: Vec<LibraryEntry>,
61}
62
63fn support_for_path(path: &Path) -> (SourceSupport, String) {
64    let name = path
65        .file_name()
66        .and_then(|n| n.to_str())
67        .unwrap_or("")
68        .to_ascii_lowercase();
69    if !path.exists() {
70        return (SourceSupport::Missing, "missing".into());
71    }
72    if name.ends_with(".zip") {
73        return (SourceSupport::ZipBag, "zip".into());
74    }
75    if name.ends_with(".csv") || name.ends_with(".xlsx") || name.ends_with(".xls") {
76        return (SourceSupport::UnsupportedTabular, "tabular".into());
77    }
78    let lower = path.to_string_lossy().to_ascii_lowercase();
79    match format_from_path(&lower) {
80        IngestRdfFormat::Auto
81            if !(lower.ends_with(".owl")
82                || lower.ends_with(".rdf")
83                || lower.ends_with(".ttl")
84                || lower.ends_with(".nt")
85                || lower.ends_with(".nq")
86                || lower.ends_with(".trig")
87                || lower.ends_with(".jsonld")
88                || lower.ends_with(".json-ld")) =>
89        {
90            (SourceSupport::UnsupportedOther, "unknown".into())
91        }
92        fmt => (SourceSupport::RdfImportVerify, format!("{fmt:?}")),
93    }
94}
95
96/// Scan one RDF source the same way import would (Rio dispatch), without writing a volume.
97pub fn audit_rdf_source(path: &Path, top_n: usize) -> io::Result<SourceAuditReport> {
98    let (support, format_label) = support_for_path(path);
99    let path_s = path.display().to_string();
100    if support != SourceSupport::RdfImportVerify {
101        return Ok(SourceAuditReport {
102            path: path_s,
103            format: format_label,
104            support,
105            source_triple_count: 0,
106            skipped_or_filtered: 0,
107            oversize_literal_count: 0,
108            first_oversize_preview: None,
109            blank_node_triples: 0,
110            top_predicates: Vec::new(),
111            comment_gloss_predicate_hits: 0,
112            notes: vec![
113                "Not an RDF serialisation supported by import/verify-graph for Cross-Format Validation."
114                    .into(),
115            ],
116        });
117    }
118
119    let lower = path.to_string_lossy().to_ascii_lowercase();
120    let fmt = format_from_path(&lower);
121    let file = File::open(path)?;
122    let reader = BufReader::new(file);
123    let mut predicate_counts: HashMap<String, u64> = HashMap::new();
124    let mut source_triple_count = 0u64;
125    let mut oversize_literal_count = 0u64;
126    let mut first_oversize_preview = None;
127    let mut blank_node_triples = 0u64;
128    let gloss: std::collections::HashSet<u64> =
129        crate::query::integrity_omit::COMMENT_GLOSS_PREDICATE_IRIS
130            .iter()
131            .map(|iri| q_hash(iri))
132            .collect();
133    let mut comment_gloss_predicate_hits = 0u64;
134
135    let mut on_triple = |raw: RawTriple| {
136        source_triple_count += 1;
137        *predicate_counts.entry(raw.predicate.clone()).or_default() += 1;
138        if gloss.contains(&q_hash(&raw.predicate)) {
139            comment_gloss_predicate_hits += 1;
140        }
141        let blank = raw.subject.starts_with("_:") || raw.object.starts_with("_:");
142        if blank {
143            blank_node_triples += 1;
144        }
145        for part in [&raw.subject, &raw.predicate, &raw.object] {
146            if part.len() > MAX_LEX_TERM_BYTES {
147                oversize_literal_count += 1;
148                if first_oversize_preview.is_none() {
149                    let preview: String = part.chars().take(120).collect();
150                    first_oversize_preview = Some(format!("{}… ({} bytes)", preview, part.len()));
151                }
152            }
153        }
154    };
155
156    if let Err(e) = parse_triples_format(fmt, reader, None, &mut on_triple) {
157        return Err(io::Error::new(io::ErrorKind::InvalidData, e));
158    }
159
160    let mut ranked: Vec<PredicateCount> = predicate_counts
161        .into_iter()
162        .map(|(iri, count)| PredicateCount { iri, count })
163        .collect();
164    ranked.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.iri.cmp(&b.iri)));
165    ranked.truncate(top_n.max(1));
166
167    let mut notes = Vec::new();
168    if oversize_literal_count > 0 {
169        notes.push(format!(
170            "{oversize_literal_count} term(s) exceed {MAX_LEX_TERM_BYTES} UTF-8 bytes (Complete ingest fails closed / TermTooLong)."
171        ));
172    }
173    if blank_node_triples > 0 {
174        notes.push(
175            "Source contains blank nodes; ground-graph proof will report BlankNodeCanonicalizationRequired when sets match."
176                .into(),
177        );
178    }
179
180    Ok(SourceAuditReport {
181        path: path_s,
182        format: format!("{fmt:?}"),
183        support,
184        source_triple_count,
185        skipped_or_filtered: 0,
186        oversize_literal_count,
187        first_oversize_preview,
188        blank_node_triples,
189        top_predicates: ranked,
190        comment_gloss_predicate_hits,
191        notes,
192    })
193}
194
195/// Walk a local ontology library root (files + datasources/*.zip) into a support matrix.
196pub fn audit_source_root(root: &Path) -> io::Result<LibraryAuditMatrix> {
197    let mut entries = Vec::new();
198    push_if_exists(&mut entries, root.join("chebi.owl"));
199    let oewn = root.join("english-wordnet-2025-plus");
200    if oewn.is_dir() {
201        for ent in std::fs::read_dir(&oewn)? {
202            let ent = ent?;
203            let p = ent.path();
204            if p.extension()
205                .and_then(|e| e.to_str())
206                .map(|e| e.eq_ignore_ascii_case("ttl"))
207                == Some(true)
208            {
209                push_file(&mut entries, &p)?;
210            }
211        }
212    }
213    let datasources = root.join("datasources");
214    if datasources.is_dir() {
215        for ent in std::fs::read_dir(&datasources)? {
216            let ent = ent?;
217            let p = ent.path();
218            if p.is_file() {
219                push_file(&mut entries, &p)?;
220            }
221        }
222    }
223    // Also list loose RDF next to root.
224    if root.is_dir() {
225        for ent in std::fs::read_dir(root)? {
226            let ent = ent?;
227            let p = ent.path();
228            if !p.is_file() {
229                continue;
230            }
231            let name = p.file_name().and_then(|n| n.to_str()).unwrap_or("");
232            if name.eq_ignore_ascii_case("chebi.owl") {
233                continue;
234            }
235            let lower = name.to_ascii_lowercase();
236            if lower.ends_with(".ttl")
237                || lower.ends_with(".nt")
238                || lower.ends_with(".owl")
239                || lower.ends_with(".rdf")
240            {
241                push_file(&mut entries, &p)?;
242            }
243        }
244    }
245    entries.sort_by(|a, b| a.path.cmp(&b.path));
246    Ok(LibraryAuditMatrix {
247        source_root: root.display().to_string(),
248        entries,
249    })
250}
251
252fn push_if_exists(entries: &mut Vec<LibraryEntry>, path: PathBuf) {
253    if path.exists() {
254        let _ = push_file(entries, &path);
255    }
256}
257
258fn push_file(entries: &mut Vec<LibraryEntry>, path: &Path) -> io::Result<()> {
259    let bytes = std::fs::metadata(path)?.len();
260    let (support, format_hint) = support_for_path(path);
261    let note = match support {
262        SourceSupport::RdfImportVerify => "import + verify-graph --encoder=import".into(),
263        SourceSupport::ZipBag => "unpack then audit members; RDF members are Tier-2".into(),
264        SourceSupport::UnsupportedTabular => {
265            "CSV/XLSX are not RDF graph import targets in this landing".into()
266        }
267        SourceSupport::UnsupportedOther => "format not supported for verify-graph".into(),
268        SourceSupport::Missing => "path missing".into(),
269    };
270    entries.push(LibraryEntry {
271        path: path.display().to_string(),
272        bytes,
273        support,
274        format_hint,
275        note,
276    });
277    Ok(())
278}