Skip to main content

qualia_core_db/query/
ingest.rs

1//! Streaming import path for RDF text sources → canonical `.q42` volume.
2//!
3//! Pipeline: a Rio parser on the main thread streams triples into a bounded channel; a pool of worker
4//! shards hashes each triple into an `NQuin` and (in [`IngestMode::Complete`]) forwards first-seen
5//! terms; a collector interns those terms into a capped spilling lexicon and writes quin runs; merge
6//! publishes a volume set via [`crate::q42_volume::UnifiedVolumeBuilder`] — the GOVERNING `.q42`
7//! layout (160-byte SuperBlock headers, block directory, BIDX, Merkle-DAG, and Q42LEX shards).
8//!
9//! History: this path previously wrote headerless LZ4 blocks and an empty lexicon, so
10//! `Q42Volume::read_all_quins` could not read the graph back and all literal text was discarded while
11//! the shrunk file was reported as "compression". Both defects are fixed — see [`IngestMode`].
12
13use crate::query::ingest_report::CountingReader;
14use crate::{q_hash, NQuin};
15
16pub use crate::query::ingest_job;
17pub use crate::query::ingest_report::{
18    format_bytes, format_count, IngestPhase, IngestReport, IngestSnapshot,
19};
20use log;
21
22use crate::query::ingest_formats::OBJECT_IRI_MASK as OBJECT_HASH_MASK;
23use crossbeam_channel::{bounded, Receiver, Sender};
24use std::collections::HashSet;
25use std::fs::File;
26use std::io::{self, BufReader, Cursor, Read};
27use std::path::Path;
28use std::thread;
29use std::time::Instant;
30use sysinfo::System;
31use tempfile::TempDir;
32
33/// Base IRI for relative terms / empty `xml:base` in catalog RDF.
34pub fn catalog_base_iri(path: &Path) -> Option<oxiri::Iri<String>> {
35    let stem = path.file_stem()?.to_str()?.to_ascii_lowercase();
36    let iri = match stem.as_str() {
37        "earl" => "http://www.w3.org/ns/earl#".to_string(),
38        "music" => "http://purl.org/ontology/mo/".to_string(),
39        other => format!("https://www.w3.org/ns/{other}#"),
40    };
41    oxiri::Iri::parse(iri).ok()
42}
43
44/// Expand Turtle `prefix:` empty local names (`dc:`, `foaf:`, `ns1:`) to IRIs.
45///
46/// Rio rejects those tokens even though Turtle 1.1 allows them. Catalog
47/// ontologies (Music Ontology in particular) use them as namespace objects.
48pub fn expand_empty_turtle_prefixed_names(src: &str) -> String {
49    let mut prefixes: Vec<(String, String)> = Vec::new();
50    for line in src.lines() {
51        let trimmed = line.trim();
52        let rest = trimmed
53            .strip_prefix("@prefix ")
54            .or_else(|| trimmed.strip_prefix("PREFIX "));
55        let Some(rest) = rest else {
56            continue;
57        };
58        let rest = rest.trim();
59        let Some(colon) = rest.find(':') else {
60            continue;
61        };
62        let name = rest[..colon].trim();
63        let Some(start) = rest[colon + 1..].find('<') else {
64            continue;
65        };
66        let after = &rest[colon + 1 + start + 1..];
67        let Some(end) = after.find('>') else {
68            continue;
69        };
70        prefixes.push((name.to_string(), after[..end].to_string()));
71    }
72    prefixes.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
73    if prefixes.is_empty() {
74        return src.to_string();
75    }
76
77    let bytes = src.as_bytes();
78    let mut out = String::with_capacity(src.len() + 256);
79    let mut i = 0;
80    let mut in_iri = false;
81    let mut in_string = false;
82    let mut long_string = false;
83    let mut in_prefix_decl = false;
84    while i < bytes.len() {
85        let c = bytes[i] as char;
86        if !in_iri && !in_string && starts_with_keyword(bytes, i, b"@prefix") {
87            in_prefix_decl = true;
88        } else if !in_iri && !in_string && starts_with_keyword(bytes, i, b"PREFIX") {
89            in_prefix_decl = true;
90        }
91        if in_prefix_decl {
92            out.push(c);
93            if c == '>' {
94                in_prefix_decl = false;
95            }
96            i += 1;
97            continue;
98        }
99        if in_iri {
100            out.push(c);
101            if c == '>' {
102                in_iri = false;
103            }
104            i += 1;
105            continue;
106        }
107        if in_string {
108            out.push(c);
109            if c == '\\' && i + 1 < bytes.len() {
110                out.push(bytes[i + 1] as char);
111                i += 2;
112                continue;
113            }
114            if long_string
115                && c == '"'
116                && i + 2 < bytes.len()
117                && bytes[i + 1] == b'"'
118                && bytes[i + 2] == b'"'
119            {
120                out.push('"');
121                out.push('"');
122                i += 3;
123                in_string = false;
124                long_string = false;
125                continue;
126            }
127            if !long_string && c == '"' {
128                in_string = false;
129            }
130            i += 1;
131            continue;
132        }
133        if c == '<' {
134            in_iri = true;
135            out.push(c);
136            i += 1;
137            continue;
138        }
139        if c == '"' {
140            in_string = true;
141            long_string = i + 2 < bytes.len() && bytes[i + 1] == b'"' && bytes[i + 2] == b'"';
142            out.push(c);
143            i += 1;
144            continue;
145        }
146        if let Some((name, iri)) = prefixes.iter().find(|(name, _)| {
147            let start = i;
148            let end = start + name.len();
149            bytes.get(end) == Some(&b':')
150                && bytes.get(start..end) == Some(name.as_bytes())
151                && (start == 0 || !is_prefix_name_char(bytes[start - 1]))
152        }) {
153            let local_start = i + name.len() + 1;
154            if empty_local_name_follows(bytes, local_start) {
155                out.push('<');
156                out.push_str(iri);
157                out.push('>');
158                i = local_start;
159                continue;
160            }
161            if let Some(hash_at) = hash_terminated_local_name(bytes, local_start) {
162                out.push('<');
163                out.push_str(iri);
164                out.push_str(&src[local_start..=hash_at]);
165                out.push('>');
166                i = hash_at + 1;
167                continue;
168            }
169        }
170        out.push(c);
171        i += 1;
172    }
173    out
174}
175
176fn starts_with_keyword(bytes: &[u8], i: usize, keyword: &[u8]) -> bool {
177    bytes.get(i..i + keyword.len()) == Some(keyword)
178        && (i == 0 || !is_prefix_name_char(bytes[i - 1]))
179        && match bytes.get(i + keyword.len()) {
180            Some(b) if is_prefix_name_char(*b) => false,
181            _ => true,
182        }
183}
184
185fn is_prefix_name_char(b: u8) -> bool {
186    b.is_ascii_alphanumeric() || b == b'_' || b == b'-'
187}
188
189fn repair_rdfxml_empty_base(src: &str, base: &str) -> String {
190    crate::query::ingest_formats::repair_rdfxml_empty_base(src, base)
191}
192
193fn empty_local_name_follows(bytes: &[u8], i: usize) -> bool {
194    match bytes.get(i) {
195        None => true,
196        Some(b) => matches!(
197            *b,
198            b' ' | b'\t' | b'\r' | b'\n' | b',' | b';' | b'.' | b')' | b']'
199        ),
200    }
201}
202
203fn hash_terminated_local_name(bytes: &[u8], start: usize) -> Option<usize> {
204    let mut i = start;
205    if i >= bytes.len() || !is_prefix_name_char(bytes[i]) {
206        return None;
207    }
208    i += 1;
209    while i < bytes.len() && is_prefix_name_char(bytes[i]) {
210        i += 1;
211    }
212    if bytes.get(i) == Some(&b'#') {
213        Some(i)
214    } else {
215        None
216    }
217}
218
219fn ingest_scratch_dir() -> io::Result<TempDir> {
220    if let Some(parent) = std::env::var_os("QUALIA_INGEST_SCRATCH") {
221        std::fs::create_dir_all(&parent)?;
222        TempDir::new_in(parent)
223    } else {
224        TempDir::new()
225    }
226}
227
228/// How much of the source graph the `.q42` retains.
229///
230/// The historical ingest hashed every subject/predicate/object into a 48-byte quin and wrote an
231/// **empty** lexicon (`lex_length: 0`). That threw away every URI and every literal — the source text
232/// was irrecoverable — while the shrunk output was presented as "compression". It was not compression;
233/// it was data loss reported as a size win. That is exactly the kind of claim-vs-reality gap this
234/// project's integrity rules (CLAUDE.md §15) forbid. This enum makes the choice explicit and the
235/// reporting honest.
236#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
237pub enum IngestMode {
238    /// **Lossless.** Intern every subject/predicate URI and every literal (full UTF-8 / Unicode) into
239    /// the q42 lexicon, so `Q42LexMmap::lookup_hash(quin.field)` recovers the original term. The `.q42`
240    /// is a faithful, reversible representation of the source graph. This is the default — honesty over
241    /// a smaller file.
242    #[default]
243    Complete,
244    /// **Lossy, structure-only.** Store only the 48-byte hash quins; discard all human-readable text
245    /// (URIs and literals). Smaller on disk, but the original strings CANNOT be recovered. The size
246    /// reduction is data loss, not compression, and is reported as such. Use only when the graph
247    /// structure alone is wanted and the term strings are available elsewhere.
248    StripLiterals,
249}
250
251impl IngestMode {
252    /// Label.
253    pub fn label(self) -> &'static str {
254        match self {
255            IngestMode::Complete => "COMPLETE (lossless — all URIs & literals retained)",
256            IngestMode::StripLiterals => {
257                "STRIP-LITERALS (lossy — human-readable text discarded, structure only)"
258            }
259        }
260    }
261}
262
263pub use crate::query::ingest_formats::RawTriple;
264
265/// Hashed quin plus first-seen lexical terms from one worker (Complete mode).
266struct HashedAtom {
267    quin: NQuin,
268    terms: Vec<(u64, String)>,
269}
270
271/// Back-compat entry point: ingests losslessly ([`IngestMode::Complete`]) — the honest default.
272pub fn streaming_import_rdf(in_path: &str, out_path: &str) -> std::io::Result<u64> {
273    streaming_import_rdf_with_mode(in_path, out_path, IngestMode::Complete)
274}
275
276/// Stream-ingest an RDF source into a `.q42` volume under an explicit [`IngestMode`].
277pub fn streaming_import_rdf_with_mode(
278    in_path: &str,
279    out_path: &str,
280    mode: IngestMode,
281) -> std::io::Result<u64> {
282    streaming_import_rdf_with_report(in_path, out_path, mode, None, IngestReport::silent())
283}
284
285/// Stream-ingest RDF into a front-embedded logical volume root and immutable,
286/// size-capped child Q42 segments.
287pub fn streaming_import_rdf_volume_set_with_mode(
288    in_path: &str,
289    root_path: &str,
290    mode: IngestMode,
291    max_segment_bytes: u64,
292) -> std::io::Result<u64> {
293    streaming_import_rdf_with_report(
294        in_path,
295        root_path,
296        mode,
297        Some(max_segment_bytes),
298        IngestReport::silent(),
299    )
300}
301
302/// Same as the volume-set / single-file import, with a progress sink for CLI / UI.
303pub fn streaming_import_rdf_with_report(
304    in_path: &str,
305    out_path: &str,
306    mode: IngestMode,
307    max_segment_bytes: Option<u64>,
308    report: IngestReport,
309) -> std::io::Result<u64> {
310    streaming_import_rdf_with_mode_inner(
311        in_path,
312        out_path,
313        mode,
314        max_segment_bytes,
315        report,
316        None,
317        &[],
318    )
319}
320
321/// Resume or run a durable job directory (`job.json` + `runs/`).
322pub fn streaming_import_rdf_with_job(job_dir: &Path, report: IngestReport) -> std::io::Result<u64> {
323    let job = crate::query::ingest_job::IngestJob::open(job_dir.to_path_buf())?;
324    let out = job.spec.output.clone();
325    let mode = if job.spec.mode == "strip_literals" {
326        IngestMode::StripLiterals
327    } else {
328        IngestMode::Complete
329    };
330    let segment = job.spec.segment_mib.map(|m| m.saturating_mul(1024 * 1024));
331    let locator = job.spec.source.locator().to_string();
332    streaming_import_rdf_with_mode_inner(
333        &locator,
334        &out,
335        mode,
336        segment,
337        report,
338        Some(job.dir.clone()),
339        &[],
340    )
341}
342
343/// Same as [`streaming_import_rdf_with_report`], with omit-predicate hashes
344/// applied before hashing (must match `verify-graph --omit-predicates`).
345pub fn streaming_import_rdf_with_report_filtered(
346    in_path: &str,
347    out_path: &str,
348    mode: IngestMode,
349    max_segment_bytes: Option<u64>,
350    report: IngestReport,
351    omit_predicate_hashes: &[u64],
352) -> std::io::Result<u64> {
353    streaming_import_rdf_with_mode_inner(
354        in_path,
355        out_path,
356        mode,
357        max_segment_bytes,
358        report,
359        None,
360        omit_predicate_hashes,
361    )
362}
363
364fn streaming_import_rdf_with_mode_inner(
365    in_path: &str,
366    out_path: &str,
367    mode: IngestMode,
368    max_segment_bytes: Option<u64>,
369    mut report: IngestReport,
370    job_dir: Option<std::path::PathBuf>,
371    omit_predicate_hashes: &[u64],
372) -> std::io::Result<u64> {
373    let start_time = Instant::now();
374    let parse_started = Instant::now();
375    report.emit(IngestPhase::Starting, "initializing ingest pipeline", None);
376
377    // 1. Hardware Detection & Scaling
378    let mut sys = System::new_all();
379    sys.refresh_all();
380    let logical_cores = sys.cpus().len();
381
382    // Constraint: Use no more than 80% of available CPU resources
383    let target_workers = std::cmp::max(1, (logical_cores as f32 * 0.8).floor() as usize);
384    report.set_workers(target_workers as u32);
385    report.emit(
386        IngestPhase::Starting,
387        format!("{logical_cores} logical cores, {target_workers} hasher shards (80% cap)"),
388        Some(format!("mode={}", mode.label())),
389    );
390
391    // 2. Channel Setup
392    // Use bounded channels to strictly enforce the 512MB RAM floor (backpressure)
393    let (tx_raw, rx_raw): (Sender<RawTriple>, Receiver<RawTriple>) = bounded(10_000);
394    let (tx_bin, rx_bin): (Sender<HashedAtom>, Receiver<HashedAtom>) = bounded(10_000);
395
396    // 3. Spawn Parallel Hasher Shards (Workers)
397    // Workers keep only a hash set of first-seen tokens (8 bytes each). Strings ride
398    // with the quin to the collector, which interns into a capped spilling lexicon.
399    // Holding every WKT literal in per-shard HashMaps is what wedged the AU OSM ingest
400    // at ~80 GiB commit.
401    let collect_lexicon = mode == IngestMode::Complete;
402    let mut worker_handles: Vec<thread::JoinHandle<u64>> = vec![];
403    for _worker_id in 0..target_workers {
404        let rx = rx_raw.clone();
405        let tx = tx_bin.clone();
406
407        let handle = thread::spawn(move || {
408            let mut local_count = 0u64;
409            let mut seen: HashSet<u64> = HashSet::new();
410            for triple in rx {
411                let s_hash = q_hash(&triple.subject);
412                let p_hash = q_hash(&triple.predicate);
413                // Objects that pack into the quin inline (typed int/decimal/bool) carry their full value
414                // in the field itself — no string needed. Everything else is stored under the same
415                // masked hash that lands in `quin.object`, so `lookup_hash(quin.object)` resolves it.
416                let is_inline = triple.packed_object.is_some();
417                let o_hash = triple
418                    .packed_object
419                    .unwrap_or_else(|| q_hash(&triple.object) & OBJECT_HASH_MASK);
420                let context = triple.context;
421                let metadata = 0u64;
422                let parity = s_hash ^ p_hash ^ o_hash ^ context ^ metadata;
423
424                let quin = NQuin {
425                    subject: s_hash,
426                    predicate: p_hash,
427                    object: o_hash,
428                    context,
429                    metadata,
430                    parity,
431                };
432
433                let mut terms = Vec::new();
434                if collect_lexicon {
435                    if seen.insert(s_hash) {
436                        terms.push((s_hash, triple.subject));
437                    }
438                    if seen.insert(p_hash) {
439                        terms.push((p_hash, triple.predicate));
440                    }
441                    if !is_inline && seen.insert(o_hash) {
442                        terms.push((o_hash, triple.object));
443                    }
444                }
445
446                if tx.send(HashedAtom { quin, terms }).is_err() {
447                    break;
448                }
449                local_count += 1;
450            }
451            if local_count > 0 {
452                log::debug!(
453                    "Ontology Ingest: worker shard finished {local_count} triples ({} first-seen hashes)",
454                    seen.len()
455                );
456            }
457            local_count
458        });
459        worker_handles.push(handle);
460    }
461
462    // Drop the extra transmitters so channels close correctly
463    drop(tx_bin);
464
465    // 4. Drain hashed Quins into bounded external-sort runs instead of one
466    // whole-graph Vec. A durable job directory keeps runs across crashes;
467    // otherwise TempDir owns every run and cleans it on success, error, or unwind.
468    let mut job = match job_dir.as_ref() {
469        Some(dir) => Some(crate::query::ingest_job::IngestJob::open(dir.clone())?),
470        None => None,
471    };
472    let skip_triples = job.as_ref().map(|j| j.checkpoint.triples).unwrap_or(0);
473    if let Some(j) = job.as_ref() {
474        report.set_quin_chunks(j.checkpoint.quin_chunks);
475        report.set_lex_runs(j.checkpoint.lex_runs);
476        report.set_skip_target(skip_triples);
477        report.attach_progress_file(j.dir.join(crate::query::ingest_job::JOB_PROGRESS));
478        if let crate::query::ingest_job::IngestSourceKind::File { path } = &j.spec.source {
479            if let Ok(meta) = std::fs::metadata(path) {
480                report.set_source_bytes(meta.len());
481            }
482        }
483        if skip_triples > 0 {
484            report.emit(
485                IngestPhase::Skipping,
486                format!(
487                    "replaying first {} accepted triples (no new hash)",
488                    crate::query::ingest_report::format_count(skip_triples)
489                ),
490                None,
491            );
492        }
493    }
494    let _sorter_temp;
495    let sorter_path = if let Some(j) = job.as_ref() {
496        let p = j.runs_dir();
497        std::fs::create_dir_all(&p)?;
498        p
499    } else {
500        _sorter_temp = Some(ingest_scratch_dir()?);
501        _sorter_temp.as_ref().unwrap().path().to_owned()
502    };
503    let adopt = job.is_some() && skip_triples > 0;
504    let collector_report = report.clone();
505    let collector_job_dir = job.as_ref().map(|j| j.dir.clone());
506    let collector_handle = thread::spawn(
507        move || -> std::io::Result<crate::external_sort::ExternalSorter> {
508            let mut sorter = if adopt {
509                crate::external_sort::ExternalSorter::adopt_existing(sorter_path)?
510            } else {
511                crate::external_sort::ExternalSorter::new(sorter_path)
512            };
513            sorter.set_note_sink(collector_report.note_sink());
514            let report = collector_report;
515            for atom in rx_bin {
516                for (hash, term) in atom.terms {
517                    sorter.intern_term(hash, &term)?;
518                }
519                let chunks_before = sorter.quin_run_count();
520                sorter.push(atom.quin)?;
521                if sorter.quin_run_count() > chunks_before {
522                    report.add_quin_chunk();
523                    if let Some(dir) = collector_job_dir.as_ref() {
524                        if let Ok(mut live) = crate::query::ingest_job::IngestJob::open(dir.clone())
525                        {
526                            let _ = live.record_progress(
527                                sorter.quin_total(),
528                                0,
529                                sorter.quin_run_count() as u64,
530                                sorter.lex_run_count() as u64,
531                            );
532                        }
533                    }
534                }
535                report.set_interned(sorter.lex_interned_count());
536            }
537            Ok(sorter)
538        },
539    );
540
541    // 5. The Streaming Sieve (Main Thread)
542    // Uses Rio to read the file sequentially without loading the whole graph into RAM.
543    // A job URL/gzip source is opened as a stream (no second copy of the ontology).
544    let digest_outcome = std::sync::Arc::new(crate::query::ingest_job::DigestOutcome::default());
545    let (source_reader, path_lower): (Box<dyn std::io::Read + Send>, String) =
546        if let Some(j) = job.as_ref() {
547            let opened = crate::query::ingest_job::open_ingest_source(
548                &j.spec.source,
549                Some(j.spec.encoding),
550                j.spec.format,
551            )?;
552            if let Some(len) = opened.content_length {
553                report.set_source_bytes(len);
554            }
555            let digesting = crate::query::ingest_job::DigestingReader::with_outcome(
556                opened.reader,
557                Some(j.windows_path()),
558                Some(digest_outcome.clone()),
559            );
560            let fake_name = format!("stream.{}", opened.format.file_extension());
561            (Box::new(digesting), fake_name.to_string())
562        } else {
563            let in_file = File::open(&in_path)?;
564            let src_len = in_file.metadata().map(|m| m.len()).unwrap_or(0);
565            report.set_source_bytes(src_len);
566            (Box::new(in_file), in_path.to_lowercase())
567        };
568    let fmt = job
569        .as_ref()
570        .map(|j| j.spec.format)
571        .filter(|f| *f != crate::query::ingest_job::IngestRdfFormat::Auto)
572        .unwrap_or_else(|| crate::query::ingest_formats::format_from_path(&path_lower));
573
574    let resume_cursor = job.as_ref().and_then(|j| {
575        crate::query::ingest_resume::ResumeCursor::load(&j.dir)
576            .ok()
577            .flatten()
578    });
579    let mut skip_triples = skip_triples;
580    let mut source_reader = source_reader;
581    if let (Some(j), Some(cur)) = (job.as_ref(), resume_cursor.as_ref()) {
582        if cur.seekable
583            && crate::query::ingest_resume::format_can_seek(fmt)
584            && matches!(
585                j.spec.source,
586                crate::query::ingest_job::IngestSourceKind::File { .. }
587            )
588        {
589            if let crate::query::ingest_job::IngestSourceKind::File { path } = &j.spec.source {
590                if let Ok((r, _off, _)) =
591                    crate::query::ingest_resume::open_resumed_file(Path::new(path), Some(cur))
592                {
593                    source_reader = r;
594                    skip_triples = 0;
595                    report.emit(
596                        IngestPhase::Parsing,
597                        format!(
598                            "resumed at byte {} after {} triples",
599                            cur.byte_offset, cur.triples
600                        ),
601                        None,
602                    );
603                }
604            }
605        }
606    }
607
608    let triples_read = std::sync::atomic::AtomicU64::new(0);
609    let skip_left = std::sync::atomic::AtomicU64::new(skip_triples);
610    let omit: HashSet<u64> = omit_predicate_hashes.iter().copied().collect();
611    let mut accept_raw = |raw: RawTriple| {
612        use std::sync::atomic::Ordering::Relaxed;
613        if omit.contains(&q_hash(&raw.predicate)) {
614            return;
615        }
616        let skip = skip_left.load(Relaxed);
617        let seen = triples_read.load(Relaxed);
618        if seen < skip {
619            let n = triples_read.fetch_add(1, Relaxed) + 1;
620            report.maybe_tick_skip(n, skip);
621            return;
622        }
623        if seen == skip && skip > 0 {
624            report.emit(
625                IngestPhase::Parsing,
626                format!(
627                    "skip complete ({} triples); hashing new statements",
628                    crate::query::ingest_report::format_count(skip)
629                ),
630                None,
631            );
632        }
633        if tx_raw.send(raw).is_ok() {
634            let n = triples_read.fetch_add(1, Relaxed) + 1;
635            report.maybe_tick(IngestPhase::Parsing, n);
636        }
637    };
638
639    // Line-oriented skip: do not Rio-parse 1.8B N-Triples/N-Quads just to drop them.
640    if skip_triples > 0 && crate::query::ingest_resume::format_is_line_oriented(fmt) {
641        let tick = |n, _b| report.maybe_tick_skip(n, skip_triples);
642        match crate::query::ingest_resume::skip_newlines(&mut source_reader, skip_triples, tick) {
643            Ok((bytes, tail)) => {
644                triples_read.store(skip_triples, std::sync::atomic::Ordering::Relaxed);
645                skip_left.store(0, std::sync::atomic::Ordering::Relaxed);
646                report.add_bytes_read(bytes);
647                source_reader = Box::new(std::io::Cursor::new(tail).chain(source_reader));
648                report.emit(
649                    IngestPhase::Parsing,
650                    format!(
651                        "line-skip {} triples ({} bytes); hashing new statements",
652                        crate::query::ingest_report::format_count(skip_triples),
653                        crate::query::ingest_report::format_bytes(bytes)
654                    ),
655                    None,
656                );
657                if let Some(j) = job.as_ref() {
658                    let _ = crate::query::ingest_resume::ResumeCursor {
659                        schema: 1,
660                        format: fmt,
661                        byte_offset: bytes,
662                        triples: triples_read.load(std::sync::atomic::Ordering::Relaxed),
663                        prolog: String::new(),
664                        seekable: true,
665                    }
666                    .store(&j.dir);
667                }
668            }
669            Err(e) => {
670                return Err(e);
671            }
672        }
673    }
674
675    log::info!(
676        "Ontology Ingest: streaming triples from {} ({fmt:?})",
677        in_path
678    );
679    let stream_ttl = job.is_some();
680    let base_iri = catalog_base_iri(Path::new(in_path));
681    let mut parse_error: Option<String> = None;
682    let mut captured_prolog = String::new();
683
684    if fmt == crate::query::ingest_job::IngestRdfFormat::N3 || path_lower.ends_with(".n3") {
685        log::info!("Ontology Ingest: parsing N3 source {}", in_path);
686        let text = std::fs::read_to_string(in_path).unwrap_or_default();
687        let mut parser = crate::modalities::logic::n3_parser::N3Parser::new(&text);
688        let mut webizen = crate::webizen::SlgArena::new();
689        let mut rules_parsed = 0;
690        let on_n3_event = |event: crate::modalities::logic::n3_parser::N3Event| -> Result<(), crate::modalities::logic::n3_parser::N3ParserError> {
691            match event {
692                crate::modalities::logic::n3_parser::N3Event::StaticTriple(triple) => {
693                    let term = |t: crate::modalities::logic::n3_parser::Term| match t {
694                        crate::modalities::logic::n3_parser::Term::Uri(s)
695                        | crate::modalities::logic::n3_parser::Term::Variable(s)
696                        | crate::modalities::logic::n3_parser::Term::Literal(s)
697                        | crate::modalities::logic::n3_parser::Term::Formula(s) => s.to_string(),
698                    };
699                    accept_raw(RawTriple {
700                        subject: term(triple.subject),
701                        predicate: term(triple.predicate),
702                        object: term(triple.object),
703                        packed_object: None,
704                        context: 0,
705                    });
706                }
707                crate::modalities::logic::n3_parser::N3Event::LogicRule(rule) => {
708                    webizen.register_rule(&rule);
709                    rules_parsed += 1;
710                }
711                crate::modalities::logic::n3_parser::N3Event::AspBlock(_)
712                | crate::modalities::logic::n3_parser::N3Event::DiffuseBlock(_) => {}
713            }
714            Ok(())
715        };
716        if let Err(e) = parser.parse_all(on_n3_event) {
717            parse_error = Some(format!("N3: {e}"));
718        }
719        let fired = webizen.fire_registered_rules(crate::q_hash("q42:ingestSession"));
720        report.emit(
721            IngestPhase::Parsing,
722            format!("N3: {rules_parsed} rules registered, {fired} fired"),
723            None,
724        );
725    } else {
726        const STREAM_AFTER: u64 = 32 * 1024 * 1024;
727        let src_len = std::fs::metadata(in_path)
728            .map(|m| m.len())
729            .unwrap_or(u64::MAX);
730        if matches!(fmt, crate::query::ingest_job::IngestRdfFormat::Turtle)
731            && src_len <= STREAM_AFTER
732            && !stream_ttl
733        {
734            let raw = std::fs::read_to_string(in_path)?;
735            let expanded = expand_empty_turtle_prefixed_names(&raw);
736            if let Err(e) = crate::query::ingest_formats::parse_triples_format(
737                fmt,
738                Cursor::new(expanded),
739                base_iri.clone(),
740                &mut accept_raw,
741            ) {
742                parse_error = Some(e);
743            }
744        } else if matches!(fmt, crate::query::ingest_job::IngestRdfFormat::RdfXml)
745            && src_len <= STREAM_AFTER
746            && !stream_ttl
747        {
748            let raw = std::fs::read_to_string(in_path)?;
749            let repaired = match base_iri.as_ref() {
750                Some(base) => repair_rdfxml_empty_base(&raw, base.as_str()),
751                None => raw,
752            };
753            if let Err(e) = crate::query::ingest_formats::parse_triples_format(
754                fmt,
755                Cursor::new(repaired),
756                base_iri.clone(),
757                &mut accept_raw,
758            ) {
759                parse_error = Some(e);
760            }
761        } else if crate::query::ingest_resume::format_uses_prolog(fmt) {
762            let mut cap = crate::query::ingest_resume::PrefixCapture::new(source_reader);
763            {
764                let buf_reader = BufReader::new(CountingReader::new(&mut cap, report.clone()));
765                if let Err(e) = crate::query::ingest_formats::parse_triples_format(
766                    fmt,
767                    buf_reader,
768                    base_iri.clone(),
769                    &mut accept_raw,
770                ) {
771                    parse_error = Some(e);
772                }
773            }
774            captured_prolog = cap.prolog;
775        } else {
776            let buf_reader = BufReader::new(CountingReader::new(source_reader, report.clone()));
777            if let Err(e) = crate::query::ingest_formats::parse_triples_format(
778                fmt,
779                buf_reader,
780                base_iri.clone(),
781                &mut accept_raw,
782            ) {
783                parse_error = Some(e);
784            }
785        }
786    }
787
788    if let Some(j) = job.as_ref() {
789        if !captured_prolog.is_empty() {
790            let _ = crate::query::ingest_resume::ResumeCursor {
791                schema: 1,
792                format: fmt,
793                byte_offset: 0,
794                triples: triples_read.load(std::sync::atomic::Ordering::Relaxed),
795                prolog: captured_prolog,
796                seekable: false,
797            }
798            .store(&j.dir);
799        }
800    }
801
802    let triples_read = triples_read.load(std::sync::atomic::Ordering::Relaxed);
803    if let Some(error) = parse_error {
804        drop(tx_raw);
805        return Err(io::Error::new(
806            io::ErrorKind::InvalidData,
807            format!("RDF parse failed after {triples_read} triples in {in_path}: {error}"),
808        ));
809    }
810
811    // Drop the main sender so workers know to terminate
812    drop(tx_raw);
813
814    // 6. Join the workers first (this closes the collector's channel). Lexicon
815    // strings are already interned on the collector via spilling runs.
816    for handle in worker_handles {
817        let _ = handle
818            .join()
819            .map_err(|_| std::io::Error::other("Q42 ingest hasher shard panicked"))?;
820    }
821
822    let sorter = collector_handle
823        .join()
824        .map_err(|_| std::io::Error::other("Q42 external-sort collector thread panicked"))??;
825    report.set_parse_ms(parse_started.elapsed().as_millis() as u64);
826    let interned_terms = sorter.lex_interned_count();
827    let lex_runs = sorter.lex_run_count();
828    if let Some(j) = job.as_mut() {
829        let _ = j.record_progress(
830            triples_read,
831            0,
832            sorter.quin_run_count() as u64,
833            lex_runs as u64,
834        );
835    }
836    report.set_triples(triples_read);
837    report.set_interned(interned_terms);
838    report.emit(
839        IngestPhase::Sorting,
840        format!(
841            "parse finished: {triples_read} triples, {interned_terms} first-seen terms, {lex_runs} lexicon run(s)"
842        ),
843        None,
844    );
845    report.emit(
846        IngestPhase::Publishing,
847        "merging runs into Q42 volume",
848        None,
849    );
850    let publish_started = Instant::now();
851    // `total_written` is the Super-Quin count ("five triples in, five quins
852    // out"); `total_superblocks` is the SuperBlock count. The two were
853    // conflated before MergeStats existed (native merge returned blocks).
854    let (total_written, total_superblocks) = match max_segment_bytes {
855        Some(cap) => {
856            let stats = sorter.merge_volume_set(std::path::Path::new(out_path), cap)?;
857            (stats.quins_written, stats.blocks_written)
858        }
859        None => {
860            let stats = sorter.merge(std::path::Path::new(out_path))?;
861            (stats.quins, stats.blocks)
862        }
863    };
864    report.set_publish_ms(publish_started.elapsed().as_millis() as u64);
865
866    // Lexicon byte size actually written — read back cheaply from the finished header (mmap, no
867    // re-serialize) so the report reflects what is really on disk.
868    let (lex_length, out_bytes) =
869        crate::q42_volume::Q42Volume::open(std::path::Path::new(out_path))
870            .ok()
871            .map(|root| {
872                let mut lex_bytes = root.header().lex_length;
873                let mut logical_bytes = std::fs::metadata(out_path).map(|m| m.len()).unwrap_or(0);
874                if let Ok(Some(manifest)) = root.volume_manifest() {
875                    let parent = std::path::Path::new(out_path)
876                        .parent()
877                        .unwrap_or_else(|| std::path::Path::new("."));
878                    for segment in manifest.segments {
879                        logical_bytes = logical_bytes.saturating_add(segment.byte_length);
880                    }
881                    for segment in manifest.lexicon_segments {
882                        logical_bytes = logical_bytes.saturating_add(segment.byte_length);
883                        if let Ok(shard) =
884                            crate::q42_volume::Q42Volume::open(&parent.join(&segment.locator))
885                        {
886                            lex_bytes = lex_bytes.saturating_add(shard.header().lex_length);
887                        }
888                    }
889                }
890                (lex_bytes, logical_bytes)
891            })
892            .unwrap_or((0, std::fs::metadata(out_path).map(|m| m.len()).unwrap_or(0)));
893
894    let duration = start_time.elapsed();
895
896    // 8. Honest reporting — state the mode and, for the lossy mode, that the size reduction is data
897    // loss, NOT compression (CLAUDE.md §15: no claim-vs-reality gap).
898    let src_bytes = std::fs::metadata(in_path).map(|m| m.len()).unwrap_or(0);
899    log::info!("Ontology Ingest: parsed {} triples", triples_read);
900    let summary = match mode {
901        IngestMode::Complete => format!(
902            "complete: {triples_read} triples, {total_written} Super-Quins, {interned_terms} first-seen terms, lexicon {lex_length} B, source {src_bytes} B → {out_bytes} B (lossless) in {duration:?}"
903        ),
904        IngestMode::StripLiterals => format!(
905            "complete STRIP-LITERALS (data loss, not compression): {triples_read} triples, {total_written} Super-Quins, source {src_bytes} B → {out_bytes} B in {duration:?}"
906        ),
907    };
908    report.emit(IngestPhase::Complete, summary, Some(out_path.to_string()));
909    if let Some(j) = job.as_mut() {
910        let full = digest_outcome.full.lock().ok().and_then(|g| *g);
911        let windows =
912            crate::query::ingest_job::read_window_hashes(&j.windows_path()).unwrap_or_default();
913        let commit = crate::query::ingest_job::window_commitment(&windows);
914        let att = crate::query::ingest_job::SourceAttestation {
915            locator: j.spec.source.locator().to_string(),
916            encoding: j.spec.encoding,
917            format: j.spec.format,
918            uncompressed_bytes: digest_outcome
919                .bytes
920                .load(std::sync::atomic::Ordering::Relaxed),
921            wire_bytes: None,
922            triples: triples_read,
923            uncompressed_sha256_hex: full.map(|d| crate::query::ingest_job::hex_encode(&d)),
924            wire_sha256_hex: None,
925            window_commitment_hex: if windows.is_empty() {
926                None
927            } else {
928                Some(crate::query::ingest_job::hex_encode(&commit))
929            },
930            window_bytes: crate::query::ingest_job::WINDOW_BYTES as u64,
931            window_count: windows.len() as u64,
932            etag: None,
933            content_length: None,
934            retrieved_unix: crate::query::ingest_job::unix_now(),
935        };
936        let _ = j.write_attestation(&att);
937        let sidecar = std::path::Path::new(out_path).with_extension("q42.source.json");
938        let _ = crate::query::ingest_job::write_json_atomic(&sidecar, &att);
939        let _ = j.set_phase(crate::query::ingest_job::IngestJobPhase::Complete);
940    }
941    log::info!(
942        "Ontology Ingest: Completed {} SuperBlocks ({} quins, mode {:?}) in {:?}",
943        total_superblocks,
944        total_written,
945        mode,
946        duration
947    );
948
949    Ok(total_written)
950}
951
952/// Verifies `integrity`.
953pub fn verify_integrity(
954    input_path: std::path::PathBuf,
955    dataset_path: std::path::PathBuf,
956) -> std::io::Result<bool> {
957    use crate::rdf_star::RdfStarParser;
958    use crate::sparql_library::parsers::turtle_star::TurtleStarParser;
959    use std::fs::File;
960    use std::io::BufReader;
961
962    // Retained only as a fast legacy diagnostic. XOR has compensating-error
963    // collisions; `verify-graph` performs the bounded encoded-set proof.
964    let mut source_checksum: u64 = 0;
965    let mut source_records: u64 = 0;
966    let file = File::open(&input_path)?;
967    let mut reader = BufReader::new(file);
968    let mut parser = TurtleStarParser::new(0);
969
970    let mut buffer = Vec::new();
971    while {
972        buffer.clear();
973        std::io::BufRead::read_until(&mut reader, b'\n', &mut buffer)? > 0
974    } {
975        let mut slice = buffer.as_slice();
976        if slice.ends_with(b"\r\n") {
977            slice = &slice[..slice.len() - 2];
978        } else if slice.ends_with(b"\n") {
979            slice = &slice[..slice.len() - 1];
980        }
981
982        if slice.is_empty() || slice[0] == b'#' || slice.iter().all(|b| b.is_ascii_whitespace()) {
983            continue;
984        }
985
986        let (s, p, o) = parser.parse_triple(slice).map_err(|e| {
987            std::io::Error::new(
988                std::io::ErrorKind::InvalidData,
989                format!("source contains an unparseable RDF triple: {e}"),
990            )
991        })?;
992        source_checksum ^= s ^ p ^ o;
993        source_records += 1;
994    }
995
996    println!("Source Checksum: 0x{:016X}", source_checksum);
997
998    // Dataset calculation
999    let mut dataset_checksum: u64 = 0;
1000    let mut dataset_records: u64 = 0;
1001
1002    let volume = match crate::q42_volume::Q42Volume::open(&dataset_path) {
1003        Ok(v) => v,
1004        Err(e) => {
1005            return Err(std::io::Error::new(
1006                std::io::ErrorKind::InvalidData,
1007                format!("Failed to open Q42 volume: {}", e),
1008            ))
1009        }
1010    };
1011
1012    let mut sb_buf = vec![0u8; crate::q42_volume::SUPERBLOCK_SIZE];
1013    for i in 0..volume.block_count() as usize {
1014        let _ = volume.read_superblock_into(i, &mut sb_buf)?;
1015        let quin_count = u64::from_le_bytes(sb_buf[16..24].try_into().unwrap()) as usize;
1016        let mut off = crate::q42_volume::SUPERBLOCK_HEADER;
1017        for _ in 0..quin_count {
1018            let quin: crate::NQuin =
1019                bytemuck::pod_read_unaligned(&sb_buf[off..off + crate::q42_volume::QUIN_SIZE]);
1020            if !quin.verify_ecc_parity() {
1021                return Err(std::io::Error::new(
1022                    std::io::ErrorKind::InvalidData,
1023                    format!("Q42 parity mismatch in block {i}"),
1024                ));
1025            }
1026            dataset_checksum ^= quin.parity;
1027            dataset_records += 1;
1028            off += crate::q42_volume::QUIN_SIZE;
1029        }
1030    }
1031
1032    println!("Dataset Checksum: 0x{:016X}", dataset_checksum);
1033
1034    println!("Source records: {source_records}");
1035    println!("Dataset records: {dataset_records}");
1036    Ok(source_checksum == dataset_checksum
1037        && source_records == dataset_records
1038        && source_records != 0)
1039}
1040
1041#[cfg(test)]
1042mod tests {
1043    use super::*;
1044    use tempfile::TempDir;
1045
1046    #[test]
1047    fn rdf_ingest_uses_external_runs_and_embeds_lossless_lexicon() {
1048        let dir = TempDir::new().unwrap();
1049        let input = dir.path().join("input.nt");
1050        let output = dir.path().join("output.q42");
1051        std::fs::write(
1052            &input,
1053            "<https://example.test/s> <https://example.test/p> \"value\" .\n",
1054        )
1055        .unwrap();
1056        assert_eq!(
1057            streaming_import_rdf_with_mode(
1058                input.to_str().unwrap(),
1059                output.to_str().unwrap(),
1060                IngestMode::Complete,
1061            )
1062            .unwrap(),
1063            1
1064        );
1065        let volume = crate::q42_volume::Q42Volume::open(&output).unwrap();
1066        assert_eq!(volume.block_count(), 1);
1067        assert!(volume.lex_view().unwrap().entry_count() >= 3);
1068    }
1069
1070    #[test]
1071    fn skip_n_replays_without_rehashing_and_ticks_skip_phase() {
1072        let dir = TempDir::new().unwrap();
1073        let input = dir.path().join("in.nt");
1074        let output = dir.path().join("out.q42");
1075        std::fs::write(
1076            &input,
1077            "<http://ex/s1> <http://ex/p> <http://ex/o1> .\n\
1078             <http://ex/s2> <http://ex/p> <http://ex/o2> .\n",
1079        )
1080        .unwrap();
1081        let job_dir = dir.path().join("job");
1082        let mut job = ingest_job::IngestJob::create(
1083            job_dir.clone(),
1084            ingest_job::IngestSourceKind::File {
1085                path: input.to_string_lossy().into_owned(),
1086            },
1087            ingest_job::IngestEncoding::Identity,
1088            ingest_job::IngestRdfFormat::NTriples,
1089            IngestMode::Complete,
1090            None,
1091            &output,
1092        )
1093        .unwrap();
1094        job.record_progress(1, 0, 0, 0).unwrap();
1095        let phases = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
1096        let slot = phases.clone();
1097        let report = IngestReport::new(1, move |s| slot.lock().unwrap().push(s.phase));
1098        let written = streaming_import_rdf_with_job(&job_dir, report).unwrap();
1099        assert_eq!(written, 1, "only the unskipped triple is hashed");
1100        let seen = phases.lock().unwrap().clone();
1101        assert!(
1102            seen.contains(&IngestPhase::Skipping),
1103            "skip must be visible in progress: {seen:?}"
1104        );
1105        assert!(job_dir.join("progress.json").is_file());
1106    }
1107
1108    #[test]
1109    fn multi_format_ingest_hashes_the_same_iri() {
1110        let dir = TempDir::new().unwrap();
1111        let samples: &[(&str, &str)] = &[
1112            ("in.nt", "<http://ex/s> <http://ex/p> <http://ex/keep> .\n"),
1113            (
1114                "in.ttl",
1115                "@prefix ex: <http://ex/> .\nex:s ex:p ex:keep .\n",
1116            ),
1117            (
1118                "in.jsonld",
1119                r#"{"@id":"http://ex/s","http://ex/p":{"@id":"http://ex/keep"}}"#,
1120            ),
1121            (
1122                "in.yamlld",
1123                "\"@id\": http://ex/s\n\"http://ex/p\":\n  \"@id\": http://ex/keep\n",
1124            ),
1125            (
1126                "in.rj",
1127                r#"{"http://ex/s":{"http://ex/p":[{"type":"uri","value":"http://ex/keep"}]}}"#,
1128            ),
1129            (
1130                "in.rdf",
1131                r#"<?xml version="1.0"?>
1132<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
1133         xmlns:ex="http://ex/">
1134  <rdf:Description rdf:about="http://ex/s">
1135    <ex:p rdf:resource="http://ex/keep"/>
1136  </rdf:Description>
1137</rdf:RDF>"#,
1138            ),
1139        ];
1140        let needle = crate::query::ingest_formats::object_iri_hash("http://ex/keep");
1141        for (name, body) in samples {
1142            let input = dir.path().join(name);
1143            let output = dir.path().join(format!("{name}.q42"));
1144            std::fs::write(&input, body).unwrap();
1145            streaming_import_rdf_with_mode(
1146                input.to_str().unwrap(),
1147                output.to_str().unwrap(),
1148                IngestMode::Complete,
1149            )
1150            .unwrap();
1151            let mut scratch = vec![NQuin::default(); 8];
1152            let mut out = vec![NQuin::default(); 8];
1153            let (hit, scanned) = crate::query::graph_accel::sieve_volume_file(
1154                &output,
1155                crate::query::graph_accel::QuinField::Object,
1156                needle,
1157                &mut scratch,
1158                &mut out,
1159            )
1160            .unwrap();
1161            assert_eq!(scanned, 1, "{name} scanned");
1162            assert_eq!(hit.written, 1, "{name} missed object hash {needle:#x}");
1163        }
1164    }
1165
1166    #[test]
1167    fn empty_turtle_prefixed_names_expand_to_iris() {
1168        let src = "@prefix dc: <http://purl.org/dc/terms/> .\n@prefix ns1: <http://purl.org/ontology/mo/> .\n<http://ex/s> rdfs:seeAlso dc: ;\n  rdfs:isDefinedBy ns1: .\n";
1169        let expanded = expand_empty_turtle_prefixed_names(src);
1170        assert!(expanded.contains("<http://purl.org/dc/terms/>"));
1171        assert!(expanded.contains("<http://purl.org/ontology/mo/>"));
1172        assert!(!expanded.contains(" dc: ;"));
1173        assert!(!expanded.contains(" ns1: ."));
1174    }
1175
1176    #[test]
1177    fn turtle_with_empty_prefix_names_ingests() {
1178        let dir = TempDir::new().unwrap();
1179        let input = dir.path().join("music-mini.ttl");
1180        let output = dir.path().join("music-mini.q42");
1181        std::fs::write(
1182            &input,
1183            "@prefix dc: <http://purl.org/dc/terms/> .\n\
1184             @prefix ns1: <http://purl.org/ontology/mo/> .\n\
1185             @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .\n\
1186             <http://purl.org/ontology/mo/Track> rdfs:isDefinedBy ns1: .\n\
1187             <http://purl.org/ontology/mo/> dc:title \"The Music Ontology\" .\n",
1188        )
1189        .unwrap();
1190        let written =
1191            streaming_import_rdf(input.to_str().unwrap(), output.to_str().unwrap()).unwrap();
1192        assert!(
1193            written >= 1,
1194            "expected at least one SuperBlock, got {written}"
1195        );
1196        let report = crate::q42_volume::Q42InspectReport::from_path(&output).unwrap();
1197        assert!(!report.lexicon_has_no_terms);
1198        // Successful parsing is not a reviewed redistribution decision. The
1199        // native writer's QW-10 contract keeps default ingest unmarked.
1200        assert_eq!(report.flags & crate::q42_volume::FLAG_PERMISSIVE_COMMONS, 0);
1201        let volume = crate::q42_volume::Q42Volume::open(&output).unwrap();
1202        assert_eq!(volume.read_all_quins().unwrap().len(), 2);
1203        let object = crate::query::ingest_formats::object_iri_hash("http://purl.org/ontology/mo/");
1204        assert_eq!(volume.lex_view().unwrap().lookup_owned(object).as_deref(), Some("http://purl.org/ontology/mo/"));
1205    }
1206
1207    #[test]
1208    fn rdfxml_empty_xml_base_ingests_with_catalog_base() {
1209        let dir = TempDir::new().unwrap();
1210        let input = dir.path().join("earl.rdf");
1211        let output = dir.path().join("earl.q42");
1212        std::fs::write(
1213            &input,
1214            concat!(
1215                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
1216                "<rdf:RDF xml:base=\"\"\n",
1217                "         xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\"\n",
1218                "         xmlns:rdfs=\"http://www.w3.org/2000/01/rdf-schema#\">\n",
1219                "  <rdf:Description rdf:about=\"#Assertion\">\n",
1220                "    <rdfs:label xml:lang=\"en\">Assertion</rdfs:label>\n",
1221                "  </rdf:Description>\n",
1222                "</rdf:RDF>\n",
1223            ),
1224        )
1225        .unwrap();
1226        let written = streaming_import_rdf(input.to_str().unwrap(), output.to_str().unwrap())
1227            .expect("empty xml:base must not fail closed after catalog base is applied");
1228        assert!(written >= 1);
1229        let report = crate::q42_volume::Q42InspectReport::from_path(&output).unwrap();
1230        assert!(!report.lexicon_has_no_terms);
1231    }
1232
1233    #[test]
1234    fn broken_turtle_fails_closed() {
1235        let dir = TempDir::new().unwrap();
1236        let input = dir.path().join("bad.ttl");
1237        let output = dir.path().join("bad.q42");
1238        std::fs::write(&input, "@prefix broken\n").unwrap();
1239        let err =
1240            streaming_import_rdf(input.to_str().unwrap(), output.to_str().unwrap()).unwrap_err();
1241        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
1242        assert!(err.to_string().contains("parse failed"));
1243    }
1244}