Skip to main content

qualia_cli/ingest/
mod.rs

1//! N-Triples → unified `.q42` v2 volume (embedded lex + bidx + LZ4 SuperBlocks).
2//!
3//! Reads N-Triples line-by-line, buffers all [`NQuin`] records, sorts by
4//! object hash, then writes a single v2 volume via [`qualia_core_db::q42_volume`].
5//!
6//! Legacy v1 sidecars (`.q42.lex`, `.q42.bidx`) remain readable via
7//! [`qualia_core_db::q42_lex::Q42Lexicon::load_for_q42`] but are no longer emitted.
8
9use std::collections::HashMap;
10use std::fs::File;
11use std::io::{BufRead, BufReader};
12use std::path::Path;
13
14use qualia_core_db::external_sort::ExternalSorter;
15use qualia_core_db::mini_parser::hash_token;
16use qualia_core_db::{NQuin, QUINS_PER_BLOCK};
17use rio_api::parser::TriplesParser;
18use rio_xml::RdfXmlParser;
19
20pub mod agent_intent;
21pub mod csv_mapper;
22pub mod detect;
23pub mod json_mapper;
24pub mod mapper;
25pub mod pipeline;
26pub mod writer;
27
28#[derive(Debug)]
29pub enum IngestError {
30    Io(std::io::Error),
31    Other(String),
32}
33
34impl From<std::io::Error> for IngestError {
35    fn from(err: std::io::Error) -> Self {
36        IngestError::Io(err)
37    }
38}
39
40impl std::fmt::Display for IngestError {
41    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
42        match self {
43            IngestError::Io(e) => write!(f, "IO Error: {}", e),
44            IngestError::Other(e) => write!(f, "{}", e),
45        }
46    }
47}
48impl std::error::Error for IngestError {}
49
50// ---------------------------------------------------------------------------
51// Public API
52// ---------------------------------------------------------------------------
53
54/// Statistics returned after a successful ingest.
55#[derive(Debug)]
56pub struct IngestStats {
57    pub triples_ingested: u64,
58    pub blocks_written: u64,
59    pub lex_entries: u64,
60    pub lines_skipped: u64,
61    pub bidx_written: bool,
62}
63
64/// Ingest an N-Triples file at `input` and write a unified v2 `.q42` to `output`.
65///
66/// Uses ExternalSorter (out-of-core K-way merge, ~48 MB peak) to stay within
67/// the 512 MB RAM floor on arbitrarily large inputs.
68pub fn ingest_ntriples(
69    input: &Path,
70    output: &Path,
71) -> Result<IngestStats, Box<dyn std::error::Error>> {
72    let reader = BufReader::new(File::open(input)?);
73
74    let temp_dir = std::env::temp_dir().join("qualia_sort_nt");
75    let mut sorter = ExternalSorter::new(temp_dir);
76    let mut triples: u64 = 0;
77    let mut skipped: u64 = 0;
78
79    for raw_line in reader.lines() {
80        let line = raw_line?;
81        let line = line.trim();
82        if line.is_empty() || line.starts_with('#') {
83            skipped += 1;
84            continue;
85        }
86        let Some((s, p, o)) = parse_nt_line(line) else {
87            skipped += 1;
88            continue;
89        };
90        let sh = hash_token(s);
91        let ph = hash_token(p);
92        let oh = hash_token(o);
93
94        sorter.push_lex(sh, s);
95        sorter.push_lex(ph, p);
96        sorter.push_lex(oh, o);
97        sorter.push(NQuin {
98            subject: sh,
99            predicate: ph,
100            object: oh,
101            context: 0,
102            metadata: 0,
103            parity: NQuin::calculate_parity(sh, ph, oh, 0, 0),
104        })?;
105        triples += 1;
106    }
107
108    let stats = sorter.merge_with_access_policy(
109        output,
110        qualia_core_db::q42_volume::IngestAccessPolicy::Restricted,
111        None,
112    )?;
113
114    let lex_entries = qualia_core_db::q42_volume::Q42Volume::open(output)
115        .ok()
116        .and_then(|v| v.lex_view().ok().map(|l| l.entry_count() as u64))
117        .unwrap_or(0);
118
119    Ok(IngestStats {
120        triples_ingested: triples,
121        blocks_written: stats.blocks,
122        lex_entries,
123        lines_skipped: skipped,
124        bidx_written: true,
125    })
126}
127
128/// Ingest an RDF/XML file at `input` — same output format as [`ingest_ntriples`].
129///
130/// Streams triples into ExternalSorter so peak RAM stays bounded at ~48 MB
131/// regardless of input size.
132pub fn ingest_rdf_xml(
133    input: &Path,
134    output: &Path,
135) -> Result<IngestStats, Box<dyn std::error::Error>> {
136    let reader = BufReader::new(File::open(input)?);
137
138    let temp_dir = std::env::temp_dir().join("qualia_sort_xml");
139    let mut sorter = ExternalSorter::new(temp_dir);
140    let mut triples: u64 = 0;
141    let mut parse_err: Option<std::io::Error> = None;
142
143    let mut parser = RdfXmlParser::new(reader, None);
144    let _ = parser.parse_all(
145        &mut |t: rio_api::model::Triple| -> Result<(), std::io::Error> {
146            let s = t.subject.to_string();
147            let p = t.predicate.to_string();
148            let o = t.object.to_string();
149
150            let sh = hash_token(&s);
151            let ph = hash_token(&p);
152
153            let mut oh = None;
154
155            if let rio_api::model::Term::Literal(rio_api::model::Literal::Typed {
156                value,
157                datatype,
158            }) = t.object
159            {
160                let dt = datatype.iri;
161                if dt == "http://www.w3.org/2001/XMLSchema#integer" {
162                    if let Ok(num) = value.parse::<i64>() {
163                        let max_val = (1i64 << 59) - 1;
164                        let min_val = -(1i64 << 59);
165                        if num >= min_val && num <= max_val {
166                            let unsigned =
167                                (num as u64) & qualia_core_db::resolver::INLINE_VALUE_MASK;
168                            oh = Some(qualia_core_db::resolver::INLINE_TAG_INTEGER | unsigned);
169                        }
170                    }
171                } else if dt == "http://www.w3.org/2001/XMLSchema#decimal" {
172                    if let Ok(num) = value.parse::<f64>() {
173                        let scaled = num * 1_000_000.0;
174                        let max_val = ((1i64 << 59) - 1) as f64;
175                        let min_val = (-(1i64 << 59)) as f64;
176                        if scaled >= min_val && scaled <= max_val {
177                            let num_i64 = scaled.round() as i64;
178                            let unsigned =
179                                (num_i64 as u64) & qualia_core_db::resolver::INLINE_VALUE_MASK;
180                            oh = Some(qualia_core_db::resolver::INLINE_TAG_DECIMAL | unsigned);
181                        }
182                    }
183                } else if dt == "http://www.w3.org/2001/XMLSchema#boolean" {
184                    if value == "true" || value == "1" {
185                        oh = Some(qualia_core_db::resolver::INLINE_TAG_BOOLEAN | 1);
186                    } else if value == "false" || value == "0" {
187                        oh = Some(qualia_core_db::resolver::INLINE_TAG_BOOLEAN | 0);
188                    }
189                }
190            }
191
192            let oh = oh.unwrap_or_else(|| hash_token(&o) & 0x0FFF_FFFF_FFFF_FFFF);
193
194            sorter
195                .push(NQuin {
196                    subject: sh,
197                    predicate: ph,
198                    object: oh,
199                    context: 0,
200                    metadata: 0,
201                    parity: NQuin::calculate_parity(sh, ph, oh, 0, 0),
202                })
203                .map_err(|e| {
204                    let io_err = std::io::Error::new(std::io::ErrorKind::Other, e.to_string());
205                    parse_err = Some(std::io::Error::new(
206                        std::io::ErrorKind::Other,
207                        e.to_string(),
208                    ));
209                    io_err
210                })?;
211            triples += 1;
212            Ok(())
213        },
214    );
215
216    if let Some(e) = parse_err {
217        return Err(Box::new(e));
218    }
219
220    let stats = sorter.merge(output)?;
221
222    Ok(IngestStats {
223        triples_ingested: triples,
224        blocks_written: stats.blocks,
225        lex_entries: 0,
226        lines_skipped: 0,
227        bidx_written: true,
228    })
229}
230
231// ---------------------------------------------------------------------------
232// N-Triples line parser
233// ---------------------------------------------------------------------------
234
235fn parse_nt_line(line: &str) -> Option<(&str, &str, &str)> {
236    let mut tokens = line.split_ascii_whitespace();
237    let s = tokens.next()?;
238    let p = tokens.next()?;
239    let o = tokens.next()?;
240    Some((s, p, o))
241}
242
243/// Ingests a checkpoint.
244pub fn ingest_chk(input: &Path, output: &Path) -> Result<IngestStats, Box<dyn std::error::Error>> {
245    let reader = File::open(input)?;
246    let temp_dir = std::env::temp_dir().join("qualia_sort_chk");
247    let mut sorter = ExternalSorter::new(temp_dir);
248
249    // .chk format does not use a lexicon currently
250    let triples = qualia_core_db::parsers::chk_parser::parse_chk_stream(reader, 0, &mut sorter)?;
251
252    let stats = sorter.merge(output)?;
253
254    Ok(IngestStats {
255        triples_ingested: triples,
256        blocks_written: stats.blocks,
257        lex_entries: 0,
258        lines_skipped: 0,
259        bidx_written: true,
260    })
261}
262
263/// Ingests CBOR.
264pub fn ingest_cbor(input: &Path, output: &Path) -> Result<IngestStats, Box<dyn std::error::Error>> {
265    let file = File::open(input)?;
266    let file_size = file.metadata()?.len();
267    if file_size > 256 * 1024 * 1024 {
268        return Err(format!(
269            "CBOR input is {} MB — exceeds 256 MB guard. Split into smaller files.",
270            file_size / (1024 * 1024)
271        )
272        .into());
273    }
274    // Safety: file size checked above; mmap avoids heap copy of raw bytes.
275    let mmap = unsafe { memmap2::Mmap::map(&file)? };
276    let buffer: &[u8] = &mmap;
277
278    let temp_dir = std::env::temp_dir().join("qualia_sort_cbor");
279    let mut sorter = ExternalSorter::new(temp_dir);
280
281    let triples =
282        qualia_core_db::parsers::cbor_parser::parse_cbor_ld_stream(&buffer, 0, &mut sorter)?;
283
284    let stats = sorter.merge(output)?;
285
286    Ok(IngestStats {
287        triples_ingested: triples,
288        blocks_written: stats.blocks,
289        lex_entries: 0,
290        lines_skipped: 0,
291        bidx_written: true,
292    })
293}
294
295/// Ingest a Turtle-Star file with SPARQL-Star embedded triples.
296///
297/// This function uses the new LexiconEntry type to support embedded triples
298/// in addition to regular string lexicon entries.
299pub fn ingest_turtle_star(
300    input: &Path,
301    output: &Path,
302) -> Result<IngestStats, Box<dyn std::error::Error>> {
303    let parent_dir = output.parent().unwrap_or(Path::new("."));
304    let ingestor = pipeline::IncrementalIngestor::new(parent_dir, 256 * 1024 * 1024);
305
306    // We map the custom IngestError to Box<dyn std::error::Error>
307    ingestor
308        .execute_stream_compilation(input, output)
309        .map_err(|e| Box::new(e) as Box<dyn std::error::Error>)?;
310
311    // For now we return empty stats since we didn't track them in the pipeline struct.
312    Ok(IngestStats {
313        triples_ingested: 0,
314        blocks_written: 0,
315        lex_entries: 0,
316        lines_skipped: 0,
317        bidx_written: true,
318    })
319}
320
321/// Ingest a KML file into a `.q42` volume via `kml_bridge::import_kml`.
322///
323/// Each `<Placemark>` becomes a set of GeoSPARQL + PROV-O quins.  The string
324/// lexicon returned by the bridge is merged into the volume's embedded lexicon.
325pub fn ingest_kml(input: &Path, output: &Path) -> Result<IngestStats, Box<dyn std::error::Error>> {
326    let file = File::open(input)?;
327    let file_size = file.metadata()?.len();
328    if file_size > 256 * 1024 * 1024 {
329        return Err(format!(
330            "KML input is {} MB — exceeds 256 MB guard. Split into smaller files.",
331            file_size / (1024 * 1024)
332        )
333        .into());
334    }
335    // Safety: file size checked above; OS maps pages on-demand, no heap copy.
336    let mmap = unsafe { memmap2::Mmap::map(&file)? };
337    let bytes: &[u8] = &mmap;
338
339    let (quins, str_lex) = qualia_core_db::kml_bridge::import_kml(bytes)
340        .map_err(|e| format!("KML parse error: {e}"))?;
341
342    // Convert the string lexicon into `LexiconEntry::String` entries.
343    let lex: HashMap<u64, qualia_core_db::q42_lex::LexiconEntry> = str_lex
344        .into_iter()
345        .map(|(k, v)| (k, qualia_core_db::q42_lex::LexiconEntry::String(v)))
346        .collect();
347
348    let mut all_quins = quins;
349    all_quins.sort_unstable_by_key(|q| q.object);
350
351    let mut blocks: Vec<Vec<NQuin>> = Vec::new();
352    let mut block_ranges: Vec<(u64, u64)> = Vec::new();
353    for chunk in all_quins.chunks(QUINS_PER_BLOCK) {
354        let min_hash = chunk.iter().map(|q| q.object).min().unwrap_or(0);
355        let max_hash = chunk.iter().map(|q| q.object).max().unwrap_or(0);
356        block_ranges.push((min_hash, max_hash));
357        blocks.push(chunk.to_vec());
358    }
359
360    let triples_ingested = all_quins.len() as u64;
361    let block_seq = blocks.len() as u64;
362    let lex_entries = lex.len() as u64;
363
364    qualia_core_db::q42_volume::write_unified_volume_with_entries(
365        output,
366        &lex,
367        &block_ranges,
368        &blocks,
369    )?;
370
371    Ok(IngestStats {
372        triples_ingested,
373        blocks_written: block_seq,
374        lex_entries,
375        lines_skipped: 0,
376        bidx_written: true,
377    })
378}
379
380/// Ingest a 3D asset (OBJ / STL / GLB) into a `.q42` volume via `asset_bridge`.
381///
382/// The geometry is parsed to a bounding-boxed `Mesh`; `mesh_to_nquins` emits the *semantic* quins
383/// (type, vertex/triangle counts, bounding box, centroid, source format) which are written to the
384/// volume — the artefact is *semantically known*, not just drawn. (Raw vertex/index buffers feed
385/// the GPU renderer separately — Phase 1.2, RENDERER_IMPLEMENTATION_PLAN.)
386pub fn ingest_asset(
387    input: &Path,
388    output: &Path,
389) -> Result<IngestStats, Box<dyn std::error::Error>> {
390    let file = File::open(input)?;
391    let file_size = file.metadata()?.len();
392    if file_size > 256 * 1024 * 1024 {
393        return Err(format!(
394            "asset input is {} MB — exceeds 256 MB guard. Split into smaller files.",
395            file_size / (1024 * 1024)
396        )
397        .into());
398    }
399    // Safety: file size checked above; OS maps pages on-demand, no heap copy.
400    let mmap = unsafe { memmap2::Mmap::map(&file)? };
401    let bytes: &[u8] = &mmap;
402
403    let hint = input.extension().and_then(|e| e.to_str());
404    let mesh = qualia_core_db::render::assets::import_asset(bytes, hint)
405        .map_err(|e| format!("asset parse error: {e}"))?;
406
407    let asset_uri = format!(
408        "urn:qualia:asset:{}",
409        input.file_name().and_then(|s| s.to_str()).unwrap_or("mesh")
410    );
411    let (quins, str_lex) =
412        qualia_core_db::render::assets::mesh_to_nquins(&mesh, &asset_uri, hint.unwrap_or("mesh"));
413
414    // Convert the string lexicon into `LexiconEntry::String` entries.
415    let lex: HashMap<u64, qualia_core_db::q42_lex::LexiconEntry> = str_lex
416        .into_iter()
417        .map(|(k, v)| (k, qualia_core_db::q42_lex::LexiconEntry::String(v)))
418        .collect();
419
420    let mut all_quins = quins;
421    all_quins.sort_unstable_by_key(|q| q.object);
422
423    let mut blocks: Vec<Vec<NQuin>> = Vec::new();
424    let mut block_ranges: Vec<(u64, u64)> = Vec::new();
425    for chunk in all_quins.chunks(QUINS_PER_BLOCK) {
426        let min_hash = chunk.iter().map(|q| q.object).min().unwrap_or(0);
427        let max_hash = chunk.iter().map(|q| q.object).max().unwrap_or(0);
428        block_ranges.push((min_hash, max_hash));
429        blocks.push(chunk.to_vec());
430    }
431
432    let triples_ingested = all_quins.len() as u64;
433    let block_seq = blocks.len() as u64;
434    let lex_entries = lex.len() as u64;
435
436    qualia_core_db::q42_volume::write_unified_volume_with_entries(
437        output,
438        &lex,
439        &block_ranges,
440        &blocks,
441    )?;
442
443    Ok(IngestStats {
444        triples_ingested,
445        blocks_written: block_seq,
446        lex_entries,
447        lines_skipped: 0,
448        bidx_written: true,
449    })
450}
451
452// ──────────────────────────────────────────────────────────────────────────────
453// Phase 2: wrappers for core-db RDF-Star parsers (all streaming via ExternalSorter)
454// ──────────────────────────────────────────────────────────────────────────────
455
456macro_rules! stream_ingest {
457    ($name:ident, $parse_fn:path, $temp_suffix:literal) => {
458        pub fn $name(
459            input: &Path,
460            output: &Path,
461        ) -> Result<IngestStats, Box<dyn std::error::Error>> {
462            let reader = File::open(input)?;
463            let temp_dir = std::env::temp_dir().join($temp_suffix);
464            let mut sorter = ExternalSorter::new(temp_dir);
465            let triples = $parse_fn(reader, 0, &mut sorter)?;
466            // QW-10: default Restricted — never mint permissive-commons here.
467            let stats = sorter.merge_with_access_policy(
468                output,
469                qualia_core_db::q42_volume::IngestAccessPolicy::Restricted,
470                None,
471            )?;
472            let lex_entries = qualia_core_db::q42_volume::Q42Volume::open(output)
473                .ok()
474                .and_then(|v| v.lex_view().ok().map(|l| l.entry_count() as u64))
475                .unwrap_or(0);
476            Ok(IngestStats {
477                triples_ingested: triples,
478                blocks_written: stats.blocks,
479                lex_entries,
480                lines_skipped: 0,
481                bidx_written: true,
482            })
483        }
484    };
485}
486
487stream_ingest!(
488    ingest_ntriples_star,
489    qualia_core_db::parsers::ntriples_star::parse_ntriples_star_stream,
490    "qualia_sort_nts"
491);
492
493stream_ingest!(
494    ingest_nquads,
495    qualia_core_db::parsers::nquads_star::parse_nquads_star_stream,
496    "qualia_sort_nq"
497);
498
499stream_ingest!(
500    ingest_nquads_star,
501    qualia_core_db::parsers::nquads_star::parse_nquads_star_stream,
502    "qualia_sort_nqs"
503);
504
505stream_ingest!(
506    ingest_turtle,
507    qualia_core_db::parsers::turtle_doc::parse_turtle_doc_stream,
508    "qualia_sort_ttl"
509);
510
511stream_ingest!(
512    ingest_trig,
513    qualia_core_db::parsers::trig_star::parse_trig_star_stream,
514    "qualia_sort_trig"
515);
516
517stream_ingest!(
518    ingest_trig_star,
519    qualia_core_db::parsers::trig_star::parse_trig_star_stream,
520    "qualia_sort_trigs"
521);
522
523stream_ingest!(
524    ingest_n3,
525    qualia_core_db::parsers::turtle_doc::parse_turtle_doc_stream,
526    "qualia_sort_n3"
527);
528
529stream_ingest!(
530    ingest_json_ld,
531    qualia_core_db::parsers::json_ld_stream::parse_json_ld_stream,
532    "qualia_sort_jsonld"
533);
534
535/// Ingests JSON-LD*.
536pub fn ingest_json_ld_star(
537    input: &Path,
538    output: &Path,
539) -> Result<IngestStats, Box<dyn std::error::Error>> {
540    let reader = File::open(input)?;
541    let temp_dir = std::env::temp_dir().join("qualia_sort_jsonlds");
542    let mut sorter = ExternalSorter::new(temp_dir);
543    let triples = qualia_core_db::parsers::json_ld_stream::parse_json_ld_star_stream(
544        reader,
545        0,
546        &mut sorter,
547        true,
548    )?;
549    let stats = sorter.merge(output)?;
550    Ok(IngestStats {
551        triples_ingested: triples,
552        blocks_written: stats.blocks,
553        lex_entries: 0,
554        lines_skipped: 0,
555        bidx_written: true,
556    })
557}
558
559/// Dispatch to the correct ingest function based on auto-detected format.
560/// Defaults to Restricted access policy (QW-10). Prefer [`ingest_auto_with_policy`].
561pub fn ingest_auto(
562    input: &Path,
563    output: &Path,
564) -> Result<(IngestStats, detect::SemanticFormat), Box<dyn std::error::Error>> {
565    ingest_auto_with_policy(
566        input,
567        output,
568        qualia_core_db::q42_volume::IngestAccessPolicy::Restricted,
569        None,
570        None,
571        None,
572    )
573}
574
575/// Auto-detect format, ingest, write provenance companion (QW-10).
576/// `public-redistributable` is refused here — use the catalog/Pages publish path.
577pub fn ingest_auto_with_policy(
578    input: &Path,
579    output: &Path,
580    access_policy: qualia_core_db::q42_volume::IngestAccessPolicy,
581    mapping_version: Option<&str>,
582    context_digest: Option<&str>,
583    compiler_build_digest: Option<&str>,
584) -> Result<(IngestStats, detect::SemanticFormat), Box<dyn std::error::Error>> {
585    if access_policy.may_declare_permissive_commons() {
586        return Err(
587            "public-redistributable semantic ingest requires the catalog publish path; use restricted | project-internal | public-not-for-redistribution for review-gated Civics releases"
588                .into(),
589        );
590    }
591    let fmt = detect::detect_format(input).ok_or_else(|| {
592        format!(
593            "Cannot auto-detect format for '{}'. Use --format to specify.",
594            input.display()
595        )
596    })?;
597
598    let stats = match fmt {
599        detect::SemanticFormat::NTriples => ingest_ntriples(input, output)?,
600        detect::SemanticFormat::NTriplesStar => ingest_ntriples_star(input, output)?,
601        detect::SemanticFormat::NQuads => ingest_nquads(input, output)?,
602        detect::SemanticFormat::NQuadsStar => ingest_nquads_star(input, output)?,
603        detect::SemanticFormat::Turtle => ingest_turtle(input, output)?,
604        detect::SemanticFormat::TurtleStar => ingest_turtle_star(input, output)?,
605        detect::SemanticFormat::TriG => ingest_trig(input, output)?,
606        detect::SemanticFormat::TriGStar => ingest_trig_star(input, output)?,
607        detect::SemanticFormat::N3 => ingest_n3(input, output)?,
608        detect::SemanticFormat::RdfXml => ingest_rdf_xml(input, output)?,
609        detect::SemanticFormat::JsonLd => ingest_json_ld(input, output)?,
610        detect::SemanticFormat::JsonLdStar => ingest_json_ld_star(input, output)?,
611        detect::SemanticFormat::AgentIntentJsonl => {
612            agent_intent::ingest_agent_intent(input, output)?
613        }
614        detect::SemanticFormat::CborLd => ingest_cbor(input, output)?,
615        detect::SemanticFormat::Kml => ingest_kml(input, output)?,
616        detect::SemanticFormat::Mesh => ingest_asset(input, output)?,
617        detect::SemanticFormat::Chk => ingest_chk(input, output)?,
618        detect::SemanticFormat::Q42 => {
619            return Err("Q42 vaults are already in native format — no ingestion needed.".into())
620        }
621    };
622
623    let source_sha = {
624        use sha2::{Digest, Sha256};
625        let bytes = std::fs::read(input)?;
626        hex::encode(Sha256::digest(&bytes))
627    };
628    let prov = qualia_core_db::q42_volume::IngestProvenanceRecord::new(
629        access_policy,
630        source_sha,
631        mapping_version.unwrap_or("unspecified"),
632        context_digest.unwrap_or("unspecified"),
633        compiler_build_digest.unwrap_or(env!("CARGO_PKG_VERSION")),
634    );
635    prov.write_beside_volume(output)?;
636    Ok((stats, fmt))
637}