1use crate::query::ingest_report::CountingReader;
14use crate::{q_hash, NQuin};
15
16pub use crate::query::ingest_job;
17pub use crate::query::ingest_report::{
18 format_bytes, format_count, IngestPhase, IngestReport, IngestSnapshot,
19};
20use log;
21
22use crate::query::ingest_formats::OBJECT_IRI_MASK as OBJECT_HASH_MASK;
23use crossbeam_channel::{bounded, Receiver, Sender};
24use std::collections::HashSet;
25use std::fs::File;
26use std::io::{self, BufReader, Cursor, Read};
27use std::path::Path;
28use std::thread;
29use std::time::Instant;
30use sysinfo::System;
31use tempfile::TempDir;
32
33pub fn catalog_base_iri(path: &Path) -> Option<oxiri::Iri<String>> {
35 let stem = path.file_stem()?.to_str()?.to_ascii_lowercase();
36 let iri = match stem.as_str() {
37 "earl" => "http://www.w3.org/ns/earl#".to_string(),
38 "music" => "http://purl.org/ontology/mo/".to_string(),
39 other => format!("https://www.w3.org/ns/{other}#"),
40 };
41 oxiri::Iri::parse(iri).ok()
42}
43
44pub fn expand_empty_turtle_prefixed_names(src: &str) -> String {
49 let mut prefixes: Vec<(String, String)> = Vec::new();
50 for line in src.lines() {
51 let trimmed = line.trim();
52 let rest = trimmed
53 .strip_prefix("@prefix ")
54 .or_else(|| trimmed.strip_prefix("PREFIX "));
55 let Some(rest) = rest else {
56 continue;
57 };
58 let rest = rest.trim();
59 let Some(colon) = rest.find(':') else {
60 continue;
61 };
62 let name = rest[..colon].trim();
63 let Some(start) = rest[colon + 1..].find('<') else {
64 continue;
65 };
66 let after = &rest[colon + 1 + start + 1..];
67 let Some(end) = after.find('>') else {
68 continue;
69 };
70 prefixes.push((name.to_string(), after[..end].to_string()));
71 }
72 prefixes.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
73 if prefixes.is_empty() {
74 return src.to_string();
75 }
76
77 let bytes = src.as_bytes();
78 let mut out = String::with_capacity(src.len() + 256);
79 let mut i = 0;
80 let mut in_iri = false;
81 let mut in_string = false;
82 let mut long_string = false;
83 let mut in_prefix_decl = false;
84 while i < bytes.len() {
85 let c = bytes[i] as char;
86 if !in_iri && !in_string && starts_with_keyword(bytes, i, b"@prefix") {
87 in_prefix_decl = true;
88 } else if !in_iri && !in_string && starts_with_keyword(bytes, i, b"PREFIX") {
89 in_prefix_decl = true;
90 }
91 if in_prefix_decl {
92 out.push(c);
93 if c == '>' {
94 in_prefix_decl = false;
95 }
96 i += 1;
97 continue;
98 }
99 if in_iri {
100 out.push(c);
101 if c == '>' {
102 in_iri = false;
103 }
104 i += 1;
105 continue;
106 }
107 if in_string {
108 out.push(c);
109 if c == '\\' && i + 1 < bytes.len() {
110 out.push(bytes[i + 1] as char);
111 i += 2;
112 continue;
113 }
114 if long_string
115 && c == '"'
116 && i + 2 < bytes.len()
117 && bytes[i + 1] == b'"'
118 && bytes[i + 2] == b'"'
119 {
120 out.push('"');
121 out.push('"');
122 i += 3;
123 in_string = false;
124 long_string = false;
125 continue;
126 }
127 if !long_string && c == '"' {
128 in_string = false;
129 }
130 i += 1;
131 continue;
132 }
133 if c == '<' {
134 in_iri = true;
135 out.push(c);
136 i += 1;
137 continue;
138 }
139 if c == '"' {
140 in_string = true;
141 long_string = i + 2 < bytes.len() && bytes[i + 1] == b'"' && bytes[i + 2] == b'"';
142 out.push(c);
143 i += 1;
144 continue;
145 }
146 if let Some((name, iri)) = prefixes.iter().find(|(name, _)| {
147 let start = i;
148 let end = start + name.len();
149 bytes.get(end) == Some(&b':')
150 && bytes.get(start..end) == Some(name.as_bytes())
151 && (start == 0 || !is_prefix_name_char(bytes[start - 1]))
152 }) {
153 let local_start = i + name.len() + 1;
154 if empty_local_name_follows(bytes, local_start) {
155 out.push('<');
156 out.push_str(iri);
157 out.push('>');
158 i = local_start;
159 continue;
160 }
161 if let Some(hash_at) = hash_terminated_local_name(bytes, local_start) {
162 out.push('<');
163 out.push_str(iri);
164 out.push_str(&src[local_start..=hash_at]);
165 out.push('>');
166 i = hash_at + 1;
167 continue;
168 }
169 }
170 out.push(c);
171 i += 1;
172 }
173 out
174}
175
176fn starts_with_keyword(bytes: &[u8], i: usize, keyword: &[u8]) -> bool {
177 bytes.get(i..i + keyword.len()) == Some(keyword)
178 && (i == 0 || !is_prefix_name_char(bytes[i - 1]))
179 && match bytes.get(i + keyword.len()) {
180 Some(b) if is_prefix_name_char(*b) => false,
181 _ => true,
182 }
183}
184
185fn is_prefix_name_char(b: u8) -> bool {
186 b.is_ascii_alphanumeric() || b == b'_' || b == b'-'
187}
188
189fn repair_rdfxml_empty_base(src: &str, base: &str) -> String {
190 crate::query::ingest_formats::repair_rdfxml_empty_base(src, base)
191}
192
193fn empty_local_name_follows(bytes: &[u8], i: usize) -> bool {
194 match bytes.get(i) {
195 None => true,
196 Some(b) => matches!(
197 *b,
198 b' ' | b'\t' | b'\r' | b'\n' | b',' | b';' | b'.' | b')' | b']'
199 ),
200 }
201}
202
203fn hash_terminated_local_name(bytes: &[u8], start: usize) -> Option<usize> {
204 let mut i = start;
205 if i >= bytes.len() || !is_prefix_name_char(bytes[i]) {
206 return None;
207 }
208 i += 1;
209 while i < bytes.len() && is_prefix_name_char(bytes[i]) {
210 i += 1;
211 }
212 if bytes.get(i) == Some(&b'#') {
213 Some(i)
214 } else {
215 None
216 }
217}
218
219fn ingest_scratch_dir() -> io::Result<TempDir> {
220 if let Some(parent) = std::env::var_os("QUALIA_INGEST_SCRATCH") {
221 std::fs::create_dir_all(&parent)?;
222 TempDir::new_in(parent)
223 } else {
224 TempDir::new()
225 }
226}
227
228#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
237pub enum IngestMode {
238 #[default]
243 Complete,
244 StripLiterals,
249}
250
251impl IngestMode {
252 pub fn label(self) -> &'static str {
254 match self {
255 IngestMode::Complete => "COMPLETE (lossless — all URIs & literals retained)",
256 IngestMode::StripLiterals => {
257 "STRIP-LITERALS (lossy — human-readable text discarded, structure only)"
258 }
259 }
260 }
261}
262
263pub use crate::query::ingest_formats::RawTriple;
264
265struct HashedAtom {
267 quin: NQuin,
268 terms: Vec<(u64, String)>,
269}
270
271pub fn streaming_import_rdf(in_path: &str, out_path: &str) -> std::io::Result<u64> {
273 streaming_import_rdf_with_mode(in_path, out_path, IngestMode::Complete)
274}
275
276pub fn streaming_import_rdf_with_mode(
278 in_path: &str,
279 out_path: &str,
280 mode: IngestMode,
281) -> std::io::Result<u64> {
282 streaming_import_rdf_with_report(in_path, out_path, mode, None, IngestReport::silent())
283}
284
285pub fn streaming_import_rdf_volume_set_with_mode(
288 in_path: &str,
289 root_path: &str,
290 mode: IngestMode,
291 max_segment_bytes: u64,
292) -> std::io::Result<u64> {
293 streaming_import_rdf_with_report(
294 in_path,
295 root_path,
296 mode,
297 Some(max_segment_bytes),
298 IngestReport::silent(),
299 )
300}
301
302pub fn streaming_import_rdf_with_report(
304 in_path: &str,
305 out_path: &str,
306 mode: IngestMode,
307 max_segment_bytes: Option<u64>,
308 report: IngestReport,
309) -> std::io::Result<u64> {
310 streaming_import_rdf_with_mode_inner(
311 in_path,
312 out_path,
313 mode,
314 max_segment_bytes,
315 report,
316 None,
317 &[],
318 )
319}
320
321pub fn streaming_import_rdf_with_job(job_dir: &Path, report: IngestReport) -> std::io::Result<u64> {
323 let job = crate::query::ingest_job::IngestJob::open(job_dir.to_path_buf())?;
324 let out = job.spec.output.clone();
325 let mode = if job.spec.mode == "strip_literals" {
326 IngestMode::StripLiterals
327 } else {
328 IngestMode::Complete
329 };
330 let segment = job.spec.segment_mib.map(|m| m.saturating_mul(1024 * 1024));
331 let locator = job.spec.source.locator().to_string();
332 streaming_import_rdf_with_mode_inner(
333 &locator,
334 &out,
335 mode,
336 segment,
337 report,
338 Some(job.dir.clone()),
339 &[],
340 )
341}
342
343pub fn streaming_import_rdf_with_report_filtered(
346 in_path: &str,
347 out_path: &str,
348 mode: IngestMode,
349 max_segment_bytes: Option<u64>,
350 report: IngestReport,
351 omit_predicate_hashes: &[u64],
352) -> std::io::Result<u64> {
353 streaming_import_rdf_with_mode_inner(
354 in_path,
355 out_path,
356 mode,
357 max_segment_bytes,
358 report,
359 None,
360 omit_predicate_hashes,
361 )
362}
363
364fn streaming_import_rdf_with_mode_inner(
365 in_path: &str,
366 out_path: &str,
367 mode: IngestMode,
368 max_segment_bytes: Option<u64>,
369 mut report: IngestReport,
370 job_dir: Option<std::path::PathBuf>,
371 omit_predicate_hashes: &[u64],
372) -> std::io::Result<u64> {
373 let start_time = Instant::now();
374 let parse_started = Instant::now();
375 report.emit(IngestPhase::Starting, "initializing ingest pipeline", None);
376
377 let mut sys = System::new_all();
379 sys.refresh_all();
380 let logical_cores = sys.cpus().len();
381
382 let target_workers = std::cmp::max(1, (logical_cores as f32 * 0.8).floor() as usize);
384 report.set_workers(target_workers as u32);
385 report.emit(
386 IngestPhase::Starting,
387 format!("{logical_cores} logical cores, {target_workers} hasher shards (80% cap)"),
388 Some(format!("mode={}", mode.label())),
389 );
390
391 let (tx_raw, rx_raw): (Sender<RawTriple>, Receiver<RawTriple>) = bounded(10_000);
394 let (tx_bin, rx_bin): (Sender<HashedAtom>, Receiver<HashedAtom>) = bounded(10_000);
395
396 let collect_lexicon = mode == IngestMode::Complete;
402 let mut worker_handles: Vec<thread::JoinHandle<u64>> = vec![];
403 for _worker_id in 0..target_workers {
404 let rx = rx_raw.clone();
405 let tx = tx_bin.clone();
406
407 let handle = thread::spawn(move || {
408 let mut local_count = 0u64;
409 let mut seen: HashSet<u64> = HashSet::new();
410 for triple in rx {
411 let s_hash = q_hash(&triple.subject);
412 let p_hash = q_hash(&triple.predicate);
413 let is_inline = triple.packed_object.is_some();
417 let o_hash = triple
418 .packed_object
419 .unwrap_or_else(|| q_hash(&triple.object) & OBJECT_HASH_MASK);
420 let context = triple.context;
421 let metadata = 0u64;
422 let parity = s_hash ^ p_hash ^ o_hash ^ context ^ metadata;
423
424 let quin = NQuin {
425 subject: s_hash,
426 predicate: p_hash,
427 object: o_hash,
428 context,
429 metadata,
430 parity,
431 };
432
433 let mut terms = Vec::new();
434 if collect_lexicon {
435 if seen.insert(s_hash) {
436 terms.push((s_hash, triple.subject));
437 }
438 if seen.insert(p_hash) {
439 terms.push((p_hash, triple.predicate));
440 }
441 if !is_inline && seen.insert(o_hash) {
442 terms.push((o_hash, triple.object));
443 }
444 }
445
446 if tx.send(HashedAtom { quin, terms }).is_err() {
447 break;
448 }
449 local_count += 1;
450 }
451 if local_count > 0 {
452 log::debug!(
453 "Ontology Ingest: worker shard finished {local_count} triples ({} first-seen hashes)",
454 seen.len()
455 );
456 }
457 local_count
458 });
459 worker_handles.push(handle);
460 }
461
462 drop(tx_bin);
464
465 let mut job = match job_dir.as_ref() {
469 Some(dir) => Some(crate::query::ingest_job::IngestJob::open(dir.clone())?),
470 None => None,
471 };
472 let skip_triples = job.as_ref().map(|j| j.checkpoint.triples).unwrap_or(0);
473 if let Some(j) = job.as_ref() {
474 report.set_quin_chunks(j.checkpoint.quin_chunks);
475 report.set_lex_runs(j.checkpoint.lex_runs);
476 report.set_skip_target(skip_triples);
477 report.attach_progress_file(j.dir.join(crate::query::ingest_job::JOB_PROGRESS));
478 if let crate::query::ingest_job::IngestSourceKind::File { path } = &j.spec.source {
479 if let Ok(meta) = std::fs::metadata(path) {
480 report.set_source_bytes(meta.len());
481 }
482 }
483 if skip_triples > 0 {
484 report.emit(
485 IngestPhase::Skipping,
486 format!(
487 "replaying first {} accepted triples (no new hash)",
488 crate::query::ingest_report::format_count(skip_triples)
489 ),
490 None,
491 );
492 }
493 }
494 let _sorter_temp;
495 let sorter_path = if let Some(j) = job.as_ref() {
496 let p = j.runs_dir();
497 std::fs::create_dir_all(&p)?;
498 p
499 } else {
500 _sorter_temp = Some(ingest_scratch_dir()?);
501 _sorter_temp.as_ref().unwrap().path().to_owned()
502 };
503 let adopt = job.is_some() && skip_triples > 0;
504 let collector_report = report.clone();
505 let collector_job_dir = job.as_ref().map(|j| j.dir.clone());
506 let collector_handle = thread::spawn(
507 move || -> std::io::Result<crate::external_sort::ExternalSorter> {
508 let mut sorter = if adopt {
509 crate::external_sort::ExternalSorter::adopt_existing(sorter_path)?
510 } else {
511 crate::external_sort::ExternalSorter::new(sorter_path)
512 };
513 sorter.set_note_sink(collector_report.note_sink());
514 let report = collector_report;
515 for atom in rx_bin {
516 for (hash, term) in atom.terms {
517 sorter.intern_term(hash, &term)?;
518 }
519 let chunks_before = sorter.quin_run_count();
520 sorter.push(atom.quin)?;
521 if sorter.quin_run_count() > chunks_before {
522 report.add_quin_chunk();
523 if let Some(dir) = collector_job_dir.as_ref() {
524 if let Ok(mut live) = crate::query::ingest_job::IngestJob::open(dir.clone())
525 {
526 let _ = live.record_progress(
527 sorter.quin_total(),
528 0,
529 sorter.quin_run_count() as u64,
530 sorter.lex_run_count() as u64,
531 );
532 }
533 }
534 }
535 report.set_interned(sorter.lex_interned_count());
536 }
537 Ok(sorter)
538 },
539 );
540
541 let digest_outcome = std::sync::Arc::new(crate::query::ingest_job::DigestOutcome::default());
545 let (source_reader, path_lower): (Box<dyn std::io::Read + Send>, String) =
546 if let Some(j) = job.as_ref() {
547 let opened = crate::query::ingest_job::open_ingest_source(
548 &j.spec.source,
549 Some(j.spec.encoding),
550 j.spec.format,
551 )?;
552 if let Some(len) = opened.content_length {
553 report.set_source_bytes(len);
554 }
555 let digesting = crate::query::ingest_job::DigestingReader::with_outcome(
556 opened.reader,
557 Some(j.windows_path()),
558 Some(digest_outcome.clone()),
559 );
560 let fake_name = format!("stream.{}", opened.format.file_extension());
561 (Box::new(digesting), fake_name.to_string())
562 } else {
563 let in_file = File::open(&in_path)?;
564 let src_len = in_file.metadata().map(|m| m.len()).unwrap_or(0);
565 report.set_source_bytes(src_len);
566 (Box::new(in_file), in_path.to_lowercase())
567 };
568 let fmt = job
569 .as_ref()
570 .map(|j| j.spec.format)
571 .filter(|f| *f != crate::query::ingest_job::IngestRdfFormat::Auto)
572 .unwrap_or_else(|| crate::query::ingest_formats::format_from_path(&path_lower));
573
574 let resume_cursor = job.as_ref().and_then(|j| {
575 crate::query::ingest_resume::ResumeCursor::load(&j.dir)
576 .ok()
577 .flatten()
578 });
579 let mut skip_triples = skip_triples;
580 let mut source_reader = source_reader;
581 if let (Some(j), Some(cur)) = (job.as_ref(), resume_cursor.as_ref()) {
582 if cur.seekable
583 && crate::query::ingest_resume::format_can_seek(fmt)
584 && matches!(
585 j.spec.source,
586 crate::query::ingest_job::IngestSourceKind::File { .. }
587 )
588 {
589 if let crate::query::ingest_job::IngestSourceKind::File { path } = &j.spec.source {
590 if let Ok((r, _off, _)) =
591 crate::query::ingest_resume::open_resumed_file(Path::new(path), Some(cur))
592 {
593 source_reader = r;
594 skip_triples = 0;
595 report.emit(
596 IngestPhase::Parsing,
597 format!(
598 "resumed at byte {} after {} triples",
599 cur.byte_offset, cur.triples
600 ),
601 None,
602 );
603 }
604 }
605 }
606 }
607
608 let triples_read = std::sync::atomic::AtomicU64::new(0);
609 let skip_left = std::sync::atomic::AtomicU64::new(skip_triples);
610 let omit: HashSet<u64> = omit_predicate_hashes.iter().copied().collect();
611 let mut accept_raw = |raw: RawTriple| {
612 use std::sync::atomic::Ordering::Relaxed;
613 if omit.contains(&q_hash(&raw.predicate)) {
614 return;
615 }
616 let skip = skip_left.load(Relaxed);
617 let seen = triples_read.load(Relaxed);
618 if seen < skip {
619 let n = triples_read.fetch_add(1, Relaxed) + 1;
620 report.maybe_tick_skip(n, skip);
621 return;
622 }
623 if seen == skip && skip > 0 {
624 report.emit(
625 IngestPhase::Parsing,
626 format!(
627 "skip complete ({} triples); hashing new statements",
628 crate::query::ingest_report::format_count(skip)
629 ),
630 None,
631 );
632 }
633 if tx_raw.send(raw).is_ok() {
634 let n = triples_read.fetch_add(1, Relaxed) + 1;
635 report.maybe_tick(IngestPhase::Parsing, n);
636 }
637 };
638
639 if skip_triples > 0 && crate::query::ingest_resume::format_is_line_oriented(fmt) {
641 let tick = |n, _b| report.maybe_tick_skip(n, skip_triples);
642 match crate::query::ingest_resume::skip_newlines(&mut source_reader, skip_triples, tick) {
643 Ok((bytes, tail)) => {
644 triples_read.store(skip_triples, std::sync::atomic::Ordering::Relaxed);
645 skip_left.store(0, std::sync::atomic::Ordering::Relaxed);
646 report.add_bytes_read(bytes);
647 source_reader = Box::new(std::io::Cursor::new(tail).chain(source_reader));
648 report.emit(
649 IngestPhase::Parsing,
650 format!(
651 "line-skip {} triples ({} bytes); hashing new statements",
652 crate::query::ingest_report::format_count(skip_triples),
653 crate::query::ingest_report::format_bytes(bytes)
654 ),
655 None,
656 );
657 if let Some(j) = job.as_ref() {
658 let _ = crate::query::ingest_resume::ResumeCursor {
659 schema: 1,
660 format: fmt,
661 byte_offset: bytes,
662 triples: triples_read.load(std::sync::atomic::Ordering::Relaxed),
663 prolog: String::new(),
664 seekable: true,
665 }
666 .store(&j.dir);
667 }
668 }
669 Err(e) => {
670 return Err(e);
671 }
672 }
673 }
674
675 log::info!(
676 "Ontology Ingest: streaming triples from {} ({fmt:?})",
677 in_path
678 );
679 let stream_ttl = job.is_some();
680 let base_iri = catalog_base_iri(Path::new(in_path));
681 let mut parse_error: Option<String> = None;
682 let mut captured_prolog = String::new();
683
684 if fmt == crate::query::ingest_job::IngestRdfFormat::N3 || path_lower.ends_with(".n3") {
685 log::info!("Ontology Ingest: parsing N3 source {}", in_path);
686 let text = std::fs::read_to_string(in_path).unwrap_or_default();
687 let mut parser = crate::modalities::logic::n3_parser::N3Parser::new(&text);
688 let mut webizen = crate::webizen::SlgArena::new();
689 let mut rules_parsed = 0;
690 let on_n3_event = |event: crate::modalities::logic::n3_parser::N3Event| -> Result<(), crate::modalities::logic::n3_parser::N3ParserError> {
691 match event {
692 crate::modalities::logic::n3_parser::N3Event::StaticTriple(triple) => {
693 let term = |t: crate::modalities::logic::n3_parser::Term| match t {
694 crate::modalities::logic::n3_parser::Term::Uri(s)
695 | crate::modalities::logic::n3_parser::Term::Variable(s)
696 | crate::modalities::logic::n3_parser::Term::Literal(s)
697 | crate::modalities::logic::n3_parser::Term::Formula(s) => s.to_string(),
698 };
699 accept_raw(RawTriple {
700 subject: term(triple.subject),
701 predicate: term(triple.predicate),
702 object: term(triple.object),
703 packed_object: None,
704 context: 0,
705 });
706 }
707 crate::modalities::logic::n3_parser::N3Event::LogicRule(rule) => {
708 webizen.register_rule(&rule);
709 rules_parsed += 1;
710 }
711 crate::modalities::logic::n3_parser::N3Event::AspBlock(_)
712 | crate::modalities::logic::n3_parser::N3Event::DiffuseBlock(_) => {}
713 }
714 Ok(())
715 };
716 if let Err(e) = parser.parse_all(on_n3_event) {
717 parse_error = Some(format!("N3: {e}"));
718 }
719 let fired = webizen.fire_registered_rules(crate::q_hash("q42:ingestSession"));
720 report.emit(
721 IngestPhase::Parsing,
722 format!("N3: {rules_parsed} rules registered, {fired} fired"),
723 None,
724 );
725 } else {
726 const STREAM_AFTER: u64 = 32 * 1024 * 1024;
727 let src_len = std::fs::metadata(in_path)
728 .map(|m| m.len())
729 .unwrap_or(u64::MAX);
730 if matches!(fmt, crate::query::ingest_job::IngestRdfFormat::Turtle)
731 && src_len <= STREAM_AFTER
732 && !stream_ttl
733 {
734 let raw = std::fs::read_to_string(in_path)?;
735 let expanded = expand_empty_turtle_prefixed_names(&raw);
736 if let Err(e) = crate::query::ingest_formats::parse_triples_format(
737 fmt,
738 Cursor::new(expanded),
739 base_iri.clone(),
740 &mut accept_raw,
741 ) {
742 parse_error = Some(e);
743 }
744 } else if matches!(fmt, crate::query::ingest_job::IngestRdfFormat::RdfXml)
745 && src_len <= STREAM_AFTER
746 && !stream_ttl
747 {
748 let raw = std::fs::read_to_string(in_path)?;
749 let repaired = match base_iri.as_ref() {
750 Some(base) => repair_rdfxml_empty_base(&raw, base.as_str()),
751 None => raw,
752 };
753 if let Err(e) = crate::query::ingest_formats::parse_triples_format(
754 fmt,
755 Cursor::new(repaired),
756 base_iri.clone(),
757 &mut accept_raw,
758 ) {
759 parse_error = Some(e);
760 }
761 } else if crate::query::ingest_resume::format_uses_prolog(fmt) {
762 let mut cap = crate::query::ingest_resume::PrefixCapture::new(source_reader);
763 {
764 let buf_reader = BufReader::new(CountingReader::new(&mut cap, report.clone()));
765 if let Err(e) = crate::query::ingest_formats::parse_triples_format(
766 fmt,
767 buf_reader,
768 base_iri.clone(),
769 &mut accept_raw,
770 ) {
771 parse_error = Some(e);
772 }
773 }
774 captured_prolog = cap.prolog;
775 } else {
776 let buf_reader = BufReader::new(CountingReader::new(source_reader, report.clone()));
777 if let Err(e) = crate::query::ingest_formats::parse_triples_format(
778 fmt,
779 buf_reader,
780 base_iri.clone(),
781 &mut accept_raw,
782 ) {
783 parse_error = Some(e);
784 }
785 }
786 }
787
788 if let Some(j) = job.as_ref() {
789 if !captured_prolog.is_empty() {
790 let _ = crate::query::ingest_resume::ResumeCursor {
791 schema: 1,
792 format: fmt,
793 byte_offset: 0,
794 triples: triples_read.load(std::sync::atomic::Ordering::Relaxed),
795 prolog: captured_prolog,
796 seekable: false,
797 }
798 .store(&j.dir);
799 }
800 }
801
802 let triples_read = triples_read.load(std::sync::atomic::Ordering::Relaxed);
803 if let Some(error) = parse_error {
804 drop(tx_raw);
805 return Err(io::Error::new(
806 io::ErrorKind::InvalidData,
807 format!("RDF parse failed after {triples_read} triples in {in_path}: {error}"),
808 ));
809 }
810
811 drop(tx_raw);
813
814 for handle in worker_handles {
817 let _ = handle
818 .join()
819 .map_err(|_| std::io::Error::other("Q42 ingest hasher shard panicked"))?;
820 }
821
822 let sorter = collector_handle
823 .join()
824 .map_err(|_| std::io::Error::other("Q42 external-sort collector thread panicked"))??;
825 report.set_parse_ms(parse_started.elapsed().as_millis() as u64);
826 let interned_terms = sorter.lex_interned_count();
827 let lex_runs = sorter.lex_run_count();
828 if let Some(j) = job.as_mut() {
829 let _ = j.record_progress(
830 triples_read,
831 0,
832 sorter.quin_run_count() as u64,
833 lex_runs as u64,
834 );
835 }
836 report.set_triples(triples_read);
837 report.set_interned(interned_terms);
838 report.emit(
839 IngestPhase::Sorting,
840 format!(
841 "parse finished: {triples_read} triples, {interned_terms} first-seen terms, {lex_runs} lexicon run(s)"
842 ),
843 None,
844 );
845 report.emit(
846 IngestPhase::Publishing,
847 "merging runs into Q42 volume",
848 None,
849 );
850 let publish_started = Instant::now();
851 let (total_written, total_superblocks) = match max_segment_bytes {
855 Some(cap) => {
856 let stats = sorter.merge_volume_set(std::path::Path::new(out_path), cap)?;
857 (stats.quins_written, stats.blocks_written)
858 }
859 None => {
860 let stats = sorter.merge(std::path::Path::new(out_path))?;
861 (stats.quins, stats.blocks)
862 }
863 };
864 report.set_publish_ms(publish_started.elapsed().as_millis() as u64);
865
866 let (lex_length, out_bytes) =
869 crate::q42_volume::Q42Volume::open(std::path::Path::new(out_path))
870 .ok()
871 .map(|root| {
872 let mut lex_bytes = root.header().lex_length;
873 let mut logical_bytes = std::fs::metadata(out_path).map(|m| m.len()).unwrap_or(0);
874 if let Ok(Some(manifest)) = root.volume_manifest() {
875 let parent = std::path::Path::new(out_path)
876 .parent()
877 .unwrap_or_else(|| std::path::Path::new("."));
878 for segment in manifest.segments {
879 logical_bytes = logical_bytes.saturating_add(segment.byte_length);
880 }
881 for segment in manifest.lexicon_segments {
882 logical_bytes = logical_bytes.saturating_add(segment.byte_length);
883 if let Ok(shard) =
884 crate::q42_volume::Q42Volume::open(&parent.join(&segment.locator))
885 {
886 lex_bytes = lex_bytes.saturating_add(shard.header().lex_length);
887 }
888 }
889 }
890 (lex_bytes, logical_bytes)
891 })
892 .unwrap_or((0, std::fs::metadata(out_path).map(|m| m.len()).unwrap_or(0)));
893
894 let duration = start_time.elapsed();
895
896 let src_bytes = std::fs::metadata(in_path).map(|m| m.len()).unwrap_or(0);
899 log::info!("Ontology Ingest: parsed {} triples", triples_read);
900 let summary = match mode {
901 IngestMode::Complete => format!(
902 "complete: {triples_read} triples, {total_written} Super-Quins, {interned_terms} first-seen terms, lexicon {lex_length} B, source {src_bytes} B → {out_bytes} B (lossless) in {duration:?}"
903 ),
904 IngestMode::StripLiterals => format!(
905 "complete STRIP-LITERALS (data loss, not compression): {triples_read} triples, {total_written} Super-Quins, source {src_bytes} B → {out_bytes} B in {duration:?}"
906 ),
907 };
908 report.emit(IngestPhase::Complete, summary, Some(out_path.to_string()));
909 if let Some(j) = job.as_mut() {
910 let full = digest_outcome.full.lock().ok().and_then(|g| *g);
911 let windows =
912 crate::query::ingest_job::read_window_hashes(&j.windows_path()).unwrap_or_default();
913 let commit = crate::query::ingest_job::window_commitment(&windows);
914 let att = crate::query::ingest_job::SourceAttestation {
915 locator: j.spec.source.locator().to_string(),
916 encoding: j.spec.encoding,
917 format: j.spec.format,
918 uncompressed_bytes: digest_outcome
919 .bytes
920 .load(std::sync::atomic::Ordering::Relaxed),
921 wire_bytes: None,
922 triples: triples_read,
923 uncompressed_sha256_hex: full.map(|d| crate::query::ingest_job::hex_encode(&d)),
924 wire_sha256_hex: None,
925 window_commitment_hex: if windows.is_empty() {
926 None
927 } else {
928 Some(crate::query::ingest_job::hex_encode(&commit))
929 },
930 window_bytes: crate::query::ingest_job::WINDOW_BYTES as u64,
931 window_count: windows.len() as u64,
932 etag: None,
933 content_length: None,
934 retrieved_unix: crate::query::ingest_job::unix_now(),
935 };
936 let _ = j.write_attestation(&att);
937 let sidecar = std::path::Path::new(out_path).with_extension("q42.source.json");
938 let _ = crate::query::ingest_job::write_json_atomic(&sidecar, &att);
939 let _ = j.set_phase(crate::query::ingest_job::IngestJobPhase::Complete);
940 }
941 log::info!(
942 "Ontology Ingest: Completed {} SuperBlocks ({} quins, mode {:?}) in {:?}",
943 total_superblocks,
944 total_written,
945 mode,
946 duration
947 );
948
949 Ok(total_written)
950}
951
952pub fn verify_integrity(
954 input_path: std::path::PathBuf,
955 dataset_path: std::path::PathBuf,
956) -> std::io::Result<bool> {
957 use crate::rdf_star::RdfStarParser;
958 use crate::sparql_library::parsers::turtle_star::TurtleStarParser;
959 use std::fs::File;
960 use std::io::BufReader;
961
962 let mut source_checksum: u64 = 0;
965 let mut source_records: u64 = 0;
966 let file = File::open(&input_path)?;
967 let mut reader = BufReader::new(file);
968 let mut parser = TurtleStarParser::new(0);
969
970 let mut buffer = Vec::new();
971 while {
972 buffer.clear();
973 std::io::BufRead::read_until(&mut reader, b'\n', &mut buffer)? > 0
974 } {
975 let mut slice = buffer.as_slice();
976 if slice.ends_with(b"\r\n") {
977 slice = &slice[..slice.len() - 2];
978 } else if slice.ends_with(b"\n") {
979 slice = &slice[..slice.len() - 1];
980 }
981
982 if slice.is_empty() || slice[0] == b'#' || slice.iter().all(|b| b.is_ascii_whitespace()) {
983 continue;
984 }
985
986 let (s, p, o) = parser.parse_triple(slice).map_err(|e| {
987 std::io::Error::new(
988 std::io::ErrorKind::InvalidData,
989 format!("source contains an unparseable RDF triple: {e}"),
990 )
991 })?;
992 source_checksum ^= s ^ p ^ o;
993 source_records += 1;
994 }
995
996 println!("Source Checksum: 0x{:016X}", source_checksum);
997
998 let mut dataset_checksum: u64 = 0;
1000 let mut dataset_records: u64 = 0;
1001
1002 let volume = match crate::q42_volume::Q42Volume::open(&dataset_path) {
1003 Ok(v) => v,
1004 Err(e) => {
1005 return Err(std::io::Error::new(
1006 std::io::ErrorKind::InvalidData,
1007 format!("Failed to open Q42 volume: {}", e),
1008 ))
1009 }
1010 };
1011
1012 let mut sb_buf = vec![0u8; crate::q42_volume::SUPERBLOCK_SIZE];
1013 for i in 0..volume.block_count() as usize {
1014 let _ = volume.read_superblock_into(i, &mut sb_buf)?;
1015 let quin_count = u64::from_le_bytes(sb_buf[16..24].try_into().unwrap()) as usize;
1016 let mut off = crate::q42_volume::SUPERBLOCK_HEADER;
1017 for _ in 0..quin_count {
1018 let quin: crate::NQuin =
1019 bytemuck::pod_read_unaligned(&sb_buf[off..off + crate::q42_volume::QUIN_SIZE]);
1020 if !quin.verify_ecc_parity() {
1021 return Err(std::io::Error::new(
1022 std::io::ErrorKind::InvalidData,
1023 format!("Q42 parity mismatch in block {i}"),
1024 ));
1025 }
1026 dataset_checksum ^= quin.parity;
1027 dataset_records += 1;
1028 off += crate::q42_volume::QUIN_SIZE;
1029 }
1030 }
1031
1032 println!("Dataset Checksum: 0x{:016X}", dataset_checksum);
1033
1034 println!("Source records: {source_records}");
1035 println!("Dataset records: {dataset_records}");
1036 Ok(source_checksum == dataset_checksum
1037 && source_records == dataset_records
1038 && source_records != 0)
1039}
1040
1041#[cfg(test)]
1042mod tests {
1043 use super::*;
1044 use tempfile::TempDir;
1045
1046 #[test]
1047 fn rdf_ingest_uses_external_runs_and_embeds_lossless_lexicon() {
1048 let dir = TempDir::new().unwrap();
1049 let input = dir.path().join("input.nt");
1050 let output = dir.path().join("output.q42");
1051 std::fs::write(
1052 &input,
1053 "<https://example.test/s> <https://example.test/p> \"value\" .\n",
1054 )
1055 .unwrap();
1056 assert_eq!(
1057 streaming_import_rdf_with_mode(
1058 input.to_str().unwrap(),
1059 output.to_str().unwrap(),
1060 IngestMode::Complete,
1061 )
1062 .unwrap(),
1063 1
1064 );
1065 let volume = crate::q42_volume::Q42Volume::open(&output).unwrap();
1066 assert_eq!(volume.block_count(), 1);
1067 assert!(volume.lex_view().unwrap().entry_count() >= 3);
1068 }
1069
1070 #[test]
1071 fn skip_n_replays_without_rehashing_and_ticks_skip_phase() {
1072 let dir = TempDir::new().unwrap();
1073 let input = dir.path().join("in.nt");
1074 let output = dir.path().join("out.q42");
1075 std::fs::write(
1076 &input,
1077 "<http://ex/s1> <http://ex/p> <http://ex/o1> .\n\
1078 <http://ex/s2> <http://ex/p> <http://ex/o2> .\n",
1079 )
1080 .unwrap();
1081 let job_dir = dir.path().join("job");
1082 let mut job = ingest_job::IngestJob::create(
1083 job_dir.clone(),
1084 ingest_job::IngestSourceKind::File {
1085 path: input.to_string_lossy().into_owned(),
1086 },
1087 ingest_job::IngestEncoding::Identity,
1088 ingest_job::IngestRdfFormat::NTriples,
1089 IngestMode::Complete,
1090 None,
1091 &output,
1092 )
1093 .unwrap();
1094 job.record_progress(1, 0, 0, 0).unwrap();
1095 let phases = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
1096 let slot = phases.clone();
1097 let report = IngestReport::new(1, move |s| slot.lock().unwrap().push(s.phase));
1098 let written = streaming_import_rdf_with_job(&job_dir, report).unwrap();
1099 assert_eq!(written, 1, "only the unskipped triple is hashed");
1100 let seen = phases.lock().unwrap().clone();
1101 assert!(
1102 seen.contains(&IngestPhase::Skipping),
1103 "skip must be visible in progress: {seen:?}"
1104 );
1105 assert!(job_dir.join("progress.json").is_file());
1106 }
1107
1108 #[test]
1109 fn multi_format_ingest_hashes_the_same_iri() {
1110 let dir = TempDir::new().unwrap();
1111 let samples: &[(&str, &str)] = &[
1112 ("in.nt", "<http://ex/s> <http://ex/p> <http://ex/keep> .\n"),
1113 (
1114 "in.ttl",
1115 "@prefix ex: <http://ex/> .\nex:s ex:p ex:keep .\n",
1116 ),
1117 (
1118 "in.jsonld",
1119 r#"{"@id":"http://ex/s","http://ex/p":{"@id":"http://ex/keep"}}"#,
1120 ),
1121 (
1122 "in.yamlld",
1123 "\"@id\": http://ex/s\n\"http://ex/p\":\n \"@id\": http://ex/keep\n",
1124 ),
1125 (
1126 "in.rj",
1127 r#"{"http://ex/s":{"http://ex/p":[{"type":"uri","value":"http://ex/keep"}]}}"#,
1128 ),
1129 (
1130 "in.rdf",
1131 r#"<?xml version="1.0"?>
1132<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
1133 xmlns:ex="http://ex/">
1134 <rdf:Description rdf:about="http://ex/s">
1135 <ex:p rdf:resource="http://ex/keep"/>
1136 </rdf:Description>
1137</rdf:RDF>"#,
1138 ),
1139 ];
1140 let needle = crate::query::ingest_formats::object_iri_hash("http://ex/keep");
1141 for (name, body) in samples {
1142 let input = dir.path().join(name);
1143 let output = dir.path().join(format!("{name}.q42"));
1144 std::fs::write(&input, body).unwrap();
1145 streaming_import_rdf_with_mode(
1146 input.to_str().unwrap(),
1147 output.to_str().unwrap(),
1148 IngestMode::Complete,
1149 )
1150 .unwrap();
1151 let mut scratch = vec![NQuin::default(); 8];
1152 let mut out = vec![NQuin::default(); 8];
1153 let (hit, scanned) = crate::query::graph_accel::sieve_volume_file(
1154 &output,
1155 crate::query::graph_accel::QuinField::Object,
1156 needle,
1157 &mut scratch,
1158 &mut out,
1159 )
1160 .unwrap();
1161 assert_eq!(scanned, 1, "{name} scanned");
1162 assert_eq!(hit.written, 1, "{name} missed object hash {needle:#x}");
1163 }
1164 }
1165
1166 #[test]
1167 fn empty_turtle_prefixed_names_expand_to_iris() {
1168 let src = "@prefix dc: <http://purl.org/dc/terms/> .\n@prefix ns1: <http://purl.org/ontology/mo/> .\n<http://ex/s> rdfs:seeAlso dc: ;\n rdfs:isDefinedBy ns1: .\n";
1169 let expanded = expand_empty_turtle_prefixed_names(src);
1170 assert!(expanded.contains("<http://purl.org/dc/terms/>"));
1171 assert!(expanded.contains("<http://purl.org/ontology/mo/>"));
1172 assert!(!expanded.contains(" dc: ;"));
1173 assert!(!expanded.contains(" ns1: ."));
1174 }
1175
1176 #[test]
1177 fn turtle_with_empty_prefix_names_ingests() {
1178 let dir = TempDir::new().unwrap();
1179 let input = dir.path().join("music-mini.ttl");
1180 let output = dir.path().join("music-mini.q42");
1181 std::fs::write(
1182 &input,
1183 "@prefix dc: <http://purl.org/dc/terms/> .\n\
1184 @prefix ns1: <http://purl.org/ontology/mo/> .\n\
1185 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .\n\
1186 <http://purl.org/ontology/mo/Track> rdfs:isDefinedBy ns1: .\n\
1187 <http://purl.org/ontology/mo/> dc:title \"The Music Ontology\" .\n",
1188 )
1189 .unwrap();
1190 let written =
1191 streaming_import_rdf(input.to_str().unwrap(), output.to_str().unwrap()).unwrap();
1192 assert!(
1193 written >= 1,
1194 "expected at least one SuperBlock, got {written}"
1195 );
1196 let report = crate::q42_volume::Q42InspectReport::from_path(&output).unwrap();
1197 assert!(!report.lexicon_has_no_terms);
1198 assert_eq!(report.flags & crate::q42_volume::FLAG_PERMISSIVE_COMMONS, 0);
1201 let volume = crate::q42_volume::Q42Volume::open(&output).unwrap();
1202 assert_eq!(volume.read_all_quins().unwrap().len(), 2);
1203 let object = crate::query::ingest_formats::object_iri_hash("http://purl.org/ontology/mo/");
1204 assert_eq!(volume.lex_view().unwrap().lookup_owned(object).as_deref(), Some("http://purl.org/ontology/mo/"));
1205 }
1206
1207 #[test]
1208 fn rdfxml_empty_xml_base_ingests_with_catalog_base() {
1209 let dir = TempDir::new().unwrap();
1210 let input = dir.path().join("earl.rdf");
1211 let output = dir.path().join("earl.q42");
1212 std::fs::write(
1213 &input,
1214 concat!(
1215 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
1216 "<rdf:RDF xml:base=\"\"\n",
1217 " xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\"\n",
1218 " xmlns:rdfs=\"http://www.w3.org/2000/01/rdf-schema#\">\n",
1219 " <rdf:Description rdf:about=\"#Assertion\">\n",
1220 " <rdfs:label xml:lang=\"en\">Assertion</rdfs:label>\n",
1221 " </rdf:Description>\n",
1222 "</rdf:RDF>\n",
1223 ),
1224 )
1225 .unwrap();
1226 let written = streaming_import_rdf(input.to_str().unwrap(), output.to_str().unwrap())
1227 .expect("empty xml:base must not fail closed after catalog base is applied");
1228 assert!(written >= 1);
1229 let report = crate::q42_volume::Q42InspectReport::from_path(&output).unwrap();
1230 assert!(!report.lexicon_has_no_terms);
1231 }
1232
1233 #[test]
1234 fn broken_turtle_fails_closed() {
1235 let dir = TempDir::new().unwrap();
1236 let input = dir.path().join("bad.ttl");
1237 let output = dir.path().join("bad.q42");
1238 std::fs::write(&input, "@prefix broken\n").unwrap();
1239 let err =
1240 streaming_import_rdf(input.to_str().unwrap(), output.to_str().unwrap()).unwrap_err();
1241 assert_eq!(err.kind(), io::ErrorKind::InvalidData);
1242 assert!(err.to_string().contains("parse failed"));
1243 }
1244}