1use std::collections::HashMap;
5use std::fs::File;
6use std::io::{self, BufReader};
7use std::path::{Path, PathBuf};
8
9use serde::Serialize;
10
11use crate::q_hash;
12use crate::query::ingest_formats::{format_from_path, parse_triples_format, RawTriple};
13use crate::query::ingest_job::IngestRdfFormat;
14use crate::query::integrity_omit::MAX_LEX_TERM_BYTES;
15
16#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
18#[serde(rename_all = "snake_case")]
19pub enum SourceSupport {
20 RdfImportVerify,
21 ZipBag,
22 UnsupportedTabular,
23 UnsupportedOther,
24 Missing,
25}
26
27#[derive(Clone, Debug, Serialize)]
28pub struct PredicateCount {
29 pub iri: String,
30 pub count: u64,
31}
32
33#[derive(Clone, Debug, Serialize)]
34pub struct SourceAuditReport {
35 pub path: String,
36 pub format: String,
37 pub support: SourceSupport,
38 pub source_triple_count: u64,
39 pub skipped_or_filtered: u64,
40 pub oversize_literal_count: u64,
41 pub first_oversize_preview: Option<String>,
42 pub blank_node_triples: u64,
43 pub top_predicates: Vec<PredicateCount>,
44 pub comment_gloss_predicate_hits: u64,
45 pub notes: Vec<String>,
46}
47
48#[derive(Clone, Debug, Serialize)]
49pub struct LibraryEntry {
50 pub path: String,
51 pub bytes: u64,
52 pub support: SourceSupport,
53 pub format_hint: String,
54 pub note: String,
55}
56
57#[derive(Clone, Debug, Serialize)]
58pub struct LibraryAuditMatrix {
59 pub source_root: String,
60 pub entries: Vec<LibraryEntry>,
61}
62
63fn support_for_path(path: &Path) -> (SourceSupport, String) {
64 let name = path
65 .file_name()
66 .and_then(|n| n.to_str())
67 .unwrap_or("")
68 .to_ascii_lowercase();
69 if !path.exists() {
70 return (SourceSupport::Missing, "missing".into());
71 }
72 if name.ends_with(".zip") {
73 return (SourceSupport::ZipBag, "zip".into());
74 }
75 if name.ends_with(".csv") || name.ends_with(".xlsx") || name.ends_with(".xls") {
76 return (SourceSupport::UnsupportedTabular, "tabular".into());
77 }
78 let lower = path.to_string_lossy().to_ascii_lowercase();
79 match format_from_path(&lower) {
80 IngestRdfFormat::Auto
81 if !(lower.ends_with(".owl")
82 || lower.ends_with(".rdf")
83 || lower.ends_with(".ttl")
84 || lower.ends_with(".nt")
85 || lower.ends_with(".nq")
86 || lower.ends_with(".trig")
87 || lower.ends_with(".jsonld")
88 || lower.ends_with(".json-ld")) =>
89 {
90 (SourceSupport::UnsupportedOther, "unknown".into())
91 }
92 fmt => (SourceSupport::RdfImportVerify, format!("{fmt:?}")),
93 }
94}
95
96pub fn audit_rdf_source(path: &Path, top_n: usize) -> io::Result<SourceAuditReport> {
98 let (support, format_label) = support_for_path(path);
99 let path_s = path.display().to_string();
100 if support != SourceSupport::RdfImportVerify {
101 return Ok(SourceAuditReport {
102 path: path_s,
103 format: format_label,
104 support,
105 source_triple_count: 0,
106 skipped_or_filtered: 0,
107 oversize_literal_count: 0,
108 first_oversize_preview: None,
109 blank_node_triples: 0,
110 top_predicates: Vec::new(),
111 comment_gloss_predicate_hits: 0,
112 notes: vec![
113 "Not an RDF serialisation supported by import/verify-graph for Cross-Format Validation."
114 .into(),
115 ],
116 });
117 }
118
119 let lower = path.to_string_lossy().to_ascii_lowercase();
120 let fmt = format_from_path(&lower);
121 let file = File::open(path)?;
122 let reader = BufReader::new(file);
123 let mut predicate_counts: HashMap<String, u64> = HashMap::new();
124 let mut source_triple_count = 0u64;
125 let mut oversize_literal_count = 0u64;
126 let mut first_oversize_preview = None;
127 let mut blank_node_triples = 0u64;
128 let gloss: std::collections::HashSet<u64> =
129 crate::query::integrity_omit::COMMENT_GLOSS_PREDICATE_IRIS
130 .iter()
131 .map(|iri| q_hash(iri))
132 .collect();
133 let mut comment_gloss_predicate_hits = 0u64;
134
135 let mut on_triple = |raw: RawTriple| {
136 source_triple_count += 1;
137 *predicate_counts.entry(raw.predicate.clone()).or_default() += 1;
138 if gloss.contains(&q_hash(&raw.predicate)) {
139 comment_gloss_predicate_hits += 1;
140 }
141 let blank = raw.subject.starts_with("_:") || raw.object.starts_with("_:");
142 if blank {
143 blank_node_triples += 1;
144 }
145 for part in [&raw.subject, &raw.predicate, &raw.object] {
146 if part.len() > MAX_LEX_TERM_BYTES {
147 oversize_literal_count += 1;
148 if first_oversize_preview.is_none() {
149 let preview: String = part.chars().take(120).collect();
150 first_oversize_preview = Some(format!("{}… ({} bytes)", preview, part.len()));
151 }
152 }
153 }
154 };
155
156 if let Err(e) = parse_triples_format(fmt, reader, None, &mut on_triple) {
157 return Err(io::Error::new(io::ErrorKind::InvalidData, e));
158 }
159
160 let mut ranked: Vec<PredicateCount> = predicate_counts
161 .into_iter()
162 .map(|(iri, count)| PredicateCount { iri, count })
163 .collect();
164 ranked.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.iri.cmp(&b.iri)));
165 ranked.truncate(top_n.max(1));
166
167 let mut notes = Vec::new();
168 if oversize_literal_count > 0 {
169 notes.push(format!(
170 "{oversize_literal_count} term(s) exceed {MAX_LEX_TERM_BYTES} UTF-8 bytes (Complete ingest fails closed / TermTooLong)."
171 ));
172 }
173 if blank_node_triples > 0 {
174 notes.push(
175 "Source contains blank nodes; ground-graph proof will report BlankNodeCanonicalizationRequired when sets match."
176 .into(),
177 );
178 }
179
180 Ok(SourceAuditReport {
181 path: path_s,
182 format: format!("{fmt:?}"),
183 support,
184 source_triple_count,
185 skipped_or_filtered: 0,
186 oversize_literal_count,
187 first_oversize_preview,
188 blank_node_triples,
189 top_predicates: ranked,
190 comment_gloss_predicate_hits,
191 notes,
192 })
193}
194
195pub fn audit_source_root(root: &Path) -> io::Result<LibraryAuditMatrix> {
197 let mut entries = Vec::new();
198 push_if_exists(&mut entries, root.join("chebi.owl"));
199 let oewn = root.join("english-wordnet-2025-plus");
200 if oewn.is_dir() {
201 for ent in std::fs::read_dir(&oewn)? {
202 let ent = ent?;
203 let p = ent.path();
204 if p.extension()
205 .and_then(|e| e.to_str())
206 .map(|e| e.eq_ignore_ascii_case("ttl"))
207 == Some(true)
208 {
209 push_file(&mut entries, &p)?;
210 }
211 }
212 }
213 let datasources = root.join("datasources");
214 if datasources.is_dir() {
215 for ent in std::fs::read_dir(&datasources)? {
216 let ent = ent?;
217 let p = ent.path();
218 if p.is_file() {
219 push_file(&mut entries, &p)?;
220 }
221 }
222 }
223 if root.is_dir() {
225 for ent in std::fs::read_dir(root)? {
226 let ent = ent?;
227 let p = ent.path();
228 if !p.is_file() {
229 continue;
230 }
231 let name = p.file_name().and_then(|n| n.to_str()).unwrap_or("");
232 if name.eq_ignore_ascii_case("chebi.owl") {
233 continue;
234 }
235 let lower = name.to_ascii_lowercase();
236 if lower.ends_with(".ttl")
237 || lower.ends_with(".nt")
238 || lower.ends_with(".owl")
239 || lower.ends_with(".rdf")
240 {
241 push_file(&mut entries, &p)?;
242 }
243 }
244 }
245 entries.sort_by(|a, b| a.path.cmp(&b.path));
246 Ok(LibraryAuditMatrix {
247 source_root: root.display().to_string(),
248 entries,
249 })
250}
251
252fn push_if_exists(entries: &mut Vec<LibraryEntry>, path: PathBuf) {
253 if path.exists() {
254 let _ = push_file(entries, &path);
255 }
256}
257
258fn push_file(entries: &mut Vec<LibraryEntry>, path: &Path) -> io::Result<()> {
259 let bytes = std::fs::metadata(path)?.len();
260 let (support, format_hint) = support_for_path(path);
261 let note = match support {
262 SourceSupport::RdfImportVerify => "import + verify-graph --encoder=import".into(),
263 SourceSupport::ZipBag => "unpack then audit members; RDF members are Tier-2".into(),
264 SourceSupport::UnsupportedTabular => {
265 "CSV/XLSX are not RDF graph import targets in this landing".into()
266 }
267 SourceSupport::UnsupportedOther => "format not supported for verify-graph".into(),
268 SourceSupport::Missing => "path missing".into(),
269 };
270 entries.push(LibraryEntry {
271 path: path.display().to_string(),
272 bytes,
273 support,
274 format_hint,
275 note,
276 });
277 Ok(())
278}