1use std::collections::HashMap;
10use std::fs::File;
11use std::io::{BufRead, BufReader};
12use std::path::Path;
13
14use qualia_core_db::external_sort::ExternalSorter;
15use qualia_core_db::mini_parser::hash_token;
16use qualia_core_db::{NQuin, QUINS_PER_BLOCK};
17use rio_api::parser::TriplesParser;
18use rio_xml::RdfXmlParser;
19
20pub mod agent_intent;
21pub mod csv_mapper;
22pub mod detect;
23pub mod json_mapper;
24pub mod mapper;
25pub mod pipeline;
26pub mod writer;
27
28#[derive(Debug)]
29pub enum IngestError {
30 Io(std::io::Error),
31 Other(String),
32}
33
34impl From<std::io::Error> for IngestError {
35 fn from(err: std::io::Error) -> Self {
36 IngestError::Io(err)
37 }
38}
39
40impl std::fmt::Display for IngestError {
41 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
42 match self {
43 IngestError::Io(e) => write!(f, "IO Error: {}", e),
44 IngestError::Other(e) => write!(f, "{}", e),
45 }
46 }
47}
48impl std::error::Error for IngestError {}
49
50#[derive(Debug)]
56pub struct IngestStats {
57 pub triples_ingested: u64,
58 pub blocks_written: u64,
59 pub lex_entries: u64,
60 pub lines_skipped: u64,
61 pub bidx_written: bool,
62}
63
64pub fn ingest_ntriples(
69 input: &Path,
70 output: &Path,
71) -> Result<IngestStats, Box<dyn std::error::Error>> {
72 let reader = BufReader::new(File::open(input)?);
73
74 let temp_dir = std::env::temp_dir().join("qualia_sort_nt");
75 let mut sorter = ExternalSorter::new(temp_dir);
76 let mut triples: u64 = 0;
77 let mut skipped: u64 = 0;
78
79 for raw_line in reader.lines() {
80 let line = raw_line?;
81 let line = line.trim();
82 if line.is_empty() || line.starts_with('#') {
83 skipped += 1;
84 continue;
85 }
86 let Some((s, p, o)) = parse_nt_line(line) else {
87 skipped += 1;
88 continue;
89 };
90 let sh = hash_token(s);
91 let ph = hash_token(p);
92 let oh = hash_token(o);
93
94 sorter.push_lex(sh, s);
95 sorter.push_lex(ph, p);
96 sorter.push_lex(oh, o);
97 sorter.push(NQuin {
98 subject: sh,
99 predicate: ph,
100 object: oh,
101 context: 0,
102 metadata: 0,
103 parity: NQuin::calculate_parity(sh, ph, oh, 0, 0),
104 })?;
105 triples += 1;
106 }
107
108 let stats = sorter.merge_with_access_policy(
109 output,
110 qualia_core_db::q42_volume::IngestAccessPolicy::Restricted,
111 None,
112 )?;
113
114 let lex_entries = qualia_core_db::q42_volume::Q42Volume::open(output)
115 .ok()
116 .and_then(|v| v.lex_view().ok().map(|l| l.entry_count() as u64))
117 .unwrap_or(0);
118
119 Ok(IngestStats {
120 triples_ingested: triples,
121 blocks_written: stats.blocks,
122 lex_entries,
123 lines_skipped: skipped,
124 bidx_written: true,
125 })
126}
127
128pub fn ingest_rdf_xml(
133 input: &Path,
134 output: &Path,
135) -> Result<IngestStats, Box<dyn std::error::Error>> {
136 let reader = BufReader::new(File::open(input)?);
137
138 let temp_dir = std::env::temp_dir().join("qualia_sort_xml");
139 let mut sorter = ExternalSorter::new(temp_dir);
140 let mut triples: u64 = 0;
141 let mut parse_err: Option<std::io::Error> = None;
142
143 let mut parser = RdfXmlParser::new(reader, None);
144 let _ = parser.parse_all(
145 &mut |t: rio_api::model::Triple| -> Result<(), std::io::Error> {
146 let s = t.subject.to_string();
147 let p = t.predicate.to_string();
148 let o = t.object.to_string();
149
150 let sh = hash_token(&s);
151 let ph = hash_token(&p);
152
153 let mut oh = None;
154
155 if let rio_api::model::Term::Literal(rio_api::model::Literal::Typed {
156 value,
157 datatype,
158 }) = t.object
159 {
160 let dt = datatype.iri;
161 if dt == "http://www.w3.org/2001/XMLSchema#integer" {
162 if let Ok(num) = value.parse::<i64>() {
163 let max_val = (1i64 << 59) - 1;
164 let min_val = -(1i64 << 59);
165 if num >= min_val && num <= max_val {
166 let unsigned =
167 (num as u64) & qualia_core_db::resolver::INLINE_VALUE_MASK;
168 oh = Some(qualia_core_db::resolver::INLINE_TAG_INTEGER | unsigned);
169 }
170 }
171 } else if dt == "http://www.w3.org/2001/XMLSchema#decimal" {
172 if let Ok(num) = value.parse::<f64>() {
173 let scaled = num * 1_000_000.0;
174 let max_val = ((1i64 << 59) - 1) as f64;
175 let min_val = (-(1i64 << 59)) as f64;
176 if scaled >= min_val && scaled <= max_val {
177 let num_i64 = scaled.round() as i64;
178 let unsigned =
179 (num_i64 as u64) & qualia_core_db::resolver::INLINE_VALUE_MASK;
180 oh = Some(qualia_core_db::resolver::INLINE_TAG_DECIMAL | unsigned);
181 }
182 }
183 } else if dt == "http://www.w3.org/2001/XMLSchema#boolean" {
184 if value == "true" || value == "1" {
185 oh = Some(qualia_core_db::resolver::INLINE_TAG_BOOLEAN | 1);
186 } else if value == "false" || value == "0" {
187 oh = Some(qualia_core_db::resolver::INLINE_TAG_BOOLEAN | 0);
188 }
189 }
190 }
191
192 let oh = oh.unwrap_or_else(|| hash_token(&o) & 0x0FFF_FFFF_FFFF_FFFF);
193
194 sorter
195 .push(NQuin {
196 subject: sh,
197 predicate: ph,
198 object: oh,
199 context: 0,
200 metadata: 0,
201 parity: NQuin::calculate_parity(sh, ph, oh, 0, 0),
202 })
203 .map_err(|e| {
204 let io_err = std::io::Error::new(std::io::ErrorKind::Other, e.to_string());
205 parse_err = Some(std::io::Error::new(
206 std::io::ErrorKind::Other,
207 e.to_string(),
208 ));
209 io_err
210 })?;
211 triples += 1;
212 Ok(())
213 },
214 );
215
216 if let Some(e) = parse_err {
217 return Err(Box::new(e));
218 }
219
220 let stats = sorter.merge(output)?;
221
222 Ok(IngestStats {
223 triples_ingested: triples,
224 blocks_written: stats.blocks,
225 lex_entries: 0,
226 lines_skipped: 0,
227 bidx_written: true,
228 })
229}
230
231fn parse_nt_line(line: &str) -> Option<(&str, &str, &str)> {
236 let mut tokens = line.split_ascii_whitespace();
237 let s = tokens.next()?;
238 let p = tokens.next()?;
239 let o = tokens.next()?;
240 Some((s, p, o))
241}
242
243pub fn ingest_chk(input: &Path, output: &Path) -> Result<IngestStats, Box<dyn std::error::Error>> {
245 let reader = File::open(input)?;
246 let temp_dir = std::env::temp_dir().join("qualia_sort_chk");
247 let mut sorter = ExternalSorter::new(temp_dir);
248
249 let triples = qualia_core_db::parsers::chk_parser::parse_chk_stream(reader, 0, &mut sorter)?;
251
252 let stats = sorter.merge(output)?;
253
254 Ok(IngestStats {
255 triples_ingested: triples,
256 blocks_written: stats.blocks,
257 lex_entries: 0,
258 lines_skipped: 0,
259 bidx_written: true,
260 })
261}
262
263pub fn ingest_cbor(input: &Path, output: &Path) -> Result<IngestStats, Box<dyn std::error::Error>> {
265 let file = File::open(input)?;
266 let file_size = file.metadata()?.len();
267 if file_size > 256 * 1024 * 1024 {
268 return Err(format!(
269 "CBOR input is {} MB — exceeds 256 MB guard. Split into smaller files.",
270 file_size / (1024 * 1024)
271 )
272 .into());
273 }
274 let mmap = unsafe { memmap2::Mmap::map(&file)? };
276 let buffer: &[u8] = &mmap;
277
278 let temp_dir = std::env::temp_dir().join("qualia_sort_cbor");
279 let mut sorter = ExternalSorter::new(temp_dir);
280
281 let triples =
282 qualia_core_db::parsers::cbor_parser::parse_cbor_ld_stream(&buffer, 0, &mut sorter)?;
283
284 let stats = sorter.merge(output)?;
285
286 Ok(IngestStats {
287 triples_ingested: triples,
288 blocks_written: stats.blocks,
289 lex_entries: 0,
290 lines_skipped: 0,
291 bidx_written: true,
292 })
293}
294
295pub fn ingest_turtle_star(
300 input: &Path,
301 output: &Path,
302) -> Result<IngestStats, Box<dyn std::error::Error>> {
303 let parent_dir = output.parent().unwrap_or(Path::new("."));
304 let ingestor = pipeline::IncrementalIngestor::new(parent_dir, 256 * 1024 * 1024);
305
306 ingestor
308 .execute_stream_compilation(input, output)
309 .map_err(|e| Box::new(e) as Box<dyn std::error::Error>)?;
310
311 Ok(IngestStats {
313 triples_ingested: 0,
314 blocks_written: 0,
315 lex_entries: 0,
316 lines_skipped: 0,
317 bidx_written: true,
318 })
319}
320
321pub fn ingest_kml(input: &Path, output: &Path) -> Result<IngestStats, Box<dyn std::error::Error>> {
326 let file = File::open(input)?;
327 let file_size = file.metadata()?.len();
328 if file_size > 256 * 1024 * 1024 {
329 return Err(format!(
330 "KML input is {} MB — exceeds 256 MB guard. Split into smaller files.",
331 file_size / (1024 * 1024)
332 )
333 .into());
334 }
335 let mmap = unsafe { memmap2::Mmap::map(&file)? };
337 let bytes: &[u8] = &mmap;
338
339 let (quins, str_lex) = qualia_core_db::kml_bridge::import_kml(bytes)
340 .map_err(|e| format!("KML parse error: {e}"))?;
341
342 let lex: HashMap<u64, qualia_core_db::q42_lex::LexiconEntry> = str_lex
344 .into_iter()
345 .map(|(k, v)| (k, qualia_core_db::q42_lex::LexiconEntry::String(v)))
346 .collect();
347
348 let mut all_quins = quins;
349 all_quins.sort_unstable_by_key(|q| q.object);
350
351 let mut blocks: Vec<Vec<NQuin>> = Vec::new();
352 let mut block_ranges: Vec<(u64, u64)> = Vec::new();
353 for chunk in all_quins.chunks(QUINS_PER_BLOCK) {
354 let min_hash = chunk.iter().map(|q| q.object).min().unwrap_or(0);
355 let max_hash = chunk.iter().map(|q| q.object).max().unwrap_or(0);
356 block_ranges.push((min_hash, max_hash));
357 blocks.push(chunk.to_vec());
358 }
359
360 let triples_ingested = all_quins.len() as u64;
361 let block_seq = blocks.len() as u64;
362 let lex_entries = lex.len() as u64;
363
364 qualia_core_db::q42_volume::write_unified_volume_with_entries(
365 output,
366 &lex,
367 &block_ranges,
368 &blocks,
369 )?;
370
371 Ok(IngestStats {
372 triples_ingested,
373 blocks_written: block_seq,
374 lex_entries,
375 lines_skipped: 0,
376 bidx_written: true,
377 })
378}
379
380pub fn ingest_asset(
387 input: &Path,
388 output: &Path,
389) -> Result<IngestStats, Box<dyn std::error::Error>> {
390 let file = File::open(input)?;
391 let file_size = file.metadata()?.len();
392 if file_size > 256 * 1024 * 1024 {
393 return Err(format!(
394 "asset input is {} MB — exceeds 256 MB guard. Split into smaller files.",
395 file_size / (1024 * 1024)
396 )
397 .into());
398 }
399 let mmap = unsafe { memmap2::Mmap::map(&file)? };
401 let bytes: &[u8] = &mmap;
402
403 let hint = input.extension().and_then(|e| e.to_str());
404 let mesh = qualia_core_db::render::assets::import_asset(bytes, hint)
405 .map_err(|e| format!("asset parse error: {e}"))?;
406
407 let asset_uri = format!(
408 "urn:qualia:asset:{}",
409 input.file_name().and_then(|s| s.to_str()).unwrap_or("mesh")
410 );
411 let (quins, str_lex) =
412 qualia_core_db::render::assets::mesh_to_nquins(&mesh, &asset_uri, hint.unwrap_or("mesh"));
413
414 let lex: HashMap<u64, qualia_core_db::q42_lex::LexiconEntry> = str_lex
416 .into_iter()
417 .map(|(k, v)| (k, qualia_core_db::q42_lex::LexiconEntry::String(v)))
418 .collect();
419
420 let mut all_quins = quins;
421 all_quins.sort_unstable_by_key(|q| q.object);
422
423 let mut blocks: Vec<Vec<NQuin>> = Vec::new();
424 let mut block_ranges: Vec<(u64, u64)> = Vec::new();
425 for chunk in all_quins.chunks(QUINS_PER_BLOCK) {
426 let min_hash = chunk.iter().map(|q| q.object).min().unwrap_or(0);
427 let max_hash = chunk.iter().map(|q| q.object).max().unwrap_or(0);
428 block_ranges.push((min_hash, max_hash));
429 blocks.push(chunk.to_vec());
430 }
431
432 let triples_ingested = all_quins.len() as u64;
433 let block_seq = blocks.len() as u64;
434 let lex_entries = lex.len() as u64;
435
436 qualia_core_db::q42_volume::write_unified_volume_with_entries(
437 output,
438 &lex,
439 &block_ranges,
440 &blocks,
441 )?;
442
443 Ok(IngestStats {
444 triples_ingested,
445 blocks_written: block_seq,
446 lex_entries,
447 lines_skipped: 0,
448 bidx_written: true,
449 })
450}
451
452macro_rules! stream_ingest {
457 ($name:ident, $parse_fn:path, $temp_suffix:literal) => {
458 pub fn $name(
459 input: &Path,
460 output: &Path,
461 ) -> Result<IngestStats, Box<dyn std::error::Error>> {
462 let reader = File::open(input)?;
463 let temp_dir = std::env::temp_dir().join($temp_suffix);
464 let mut sorter = ExternalSorter::new(temp_dir);
465 let triples = $parse_fn(reader, 0, &mut sorter)?;
466 let stats = sorter.merge_with_access_policy(
468 output,
469 qualia_core_db::q42_volume::IngestAccessPolicy::Restricted,
470 None,
471 )?;
472 let lex_entries = qualia_core_db::q42_volume::Q42Volume::open(output)
473 .ok()
474 .and_then(|v| v.lex_view().ok().map(|l| l.entry_count() as u64))
475 .unwrap_or(0);
476 Ok(IngestStats {
477 triples_ingested: triples,
478 blocks_written: stats.blocks,
479 lex_entries,
480 lines_skipped: 0,
481 bidx_written: true,
482 })
483 }
484 };
485}
486
487stream_ingest!(
488 ingest_ntriples_star,
489 qualia_core_db::parsers::ntriples_star::parse_ntriples_star_stream,
490 "qualia_sort_nts"
491);
492
493stream_ingest!(
494 ingest_nquads,
495 qualia_core_db::parsers::nquads_star::parse_nquads_star_stream,
496 "qualia_sort_nq"
497);
498
499stream_ingest!(
500 ingest_nquads_star,
501 qualia_core_db::parsers::nquads_star::parse_nquads_star_stream,
502 "qualia_sort_nqs"
503);
504
505stream_ingest!(
506 ingest_turtle,
507 qualia_core_db::parsers::turtle_doc::parse_turtle_doc_stream,
508 "qualia_sort_ttl"
509);
510
511stream_ingest!(
512 ingest_trig,
513 qualia_core_db::parsers::trig_star::parse_trig_star_stream,
514 "qualia_sort_trig"
515);
516
517stream_ingest!(
518 ingest_trig_star,
519 qualia_core_db::parsers::trig_star::parse_trig_star_stream,
520 "qualia_sort_trigs"
521);
522
523stream_ingest!(
524 ingest_n3,
525 qualia_core_db::parsers::turtle_doc::parse_turtle_doc_stream,
526 "qualia_sort_n3"
527);
528
529stream_ingest!(
530 ingest_json_ld,
531 qualia_core_db::parsers::json_ld_stream::parse_json_ld_stream,
532 "qualia_sort_jsonld"
533);
534
535pub fn ingest_json_ld_star(
537 input: &Path,
538 output: &Path,
539) -> Result<IngestStats, Box<dyn std::error::Error>> {
540 let reader = File::open(input)?;
541 let temp_dir = std::env::temp_dir().join("qualia_sort_jsonlds");
542 let mut sorter = ExternalSorter::new(temp_dir);
543 let triples = qualia_core_db::parsers::json_ld_stream::parse_json_ld_star_stream(
544 reader,
545 0,
546 &mut sorter,
547 true,
548 )?;
549 let stats = sorter.merge(output)?;
550 Ok(IngestStats {
551 triples_ingested: triples,
552 blocks_written: stats.blocks,
553 lex_entries: 0,
554 lines_skipped: 0,
555 bidx_written: true,
556 })
557}
558
559pub fn ingest_auto(
562 input: &Path,
563 output: &Path,
564) -> Result<(IngestStats, detect::SemanticFormat), Box<dyn std::error::Error>> {
565 ingest_auto_with_policy(
566 input,
567 output,
568 qualia_core_db::q42_volume::IngestAccessPolicy::Restricted,
569 None,
570 None,
571 None,
572 )
573}
574
575pub fn ingest_auto_with_policy(
578 input: &Path,
579 output: &Path,
580 access_policy: qualia_core_db::q42_volume::IngestAccessPolicy,
581 mapping_version: Option<&str>,
582 context_digest: Option<&str>,
583 compiler_build_digest: Option<&str>,
584) -> Result<(IngestStats, detect::SemanticFormat), Box<dyn std::error::Error>> {
585 if access_policy.may_declare_permissive_commons() {
586 return Err(
587 "public-redistributable semantic ingest requires the catalog publish path; use restricted | project-internal | public-not-for-redistribution for review-gated Civics releases"
588 .into(),
589 );
590 }
591 let fmt = detect::detect_format(input).ok_or_else(|| {
592 format!(
593 "Cannot auto-detect format for '{}'. Use --format to specify.",
594 input.display()
595 )
596 })?;
597
598 let stats = match fmt {
599 detect::SemanticFormat::NTriples => ingest_ntriples(input, output)?,
600 detect::SemanticFormat::NTriplesStar => ingest_ntriples_star(input, output)?,
601 detect::SemanticFormat::NQuads => ingest_nquads(input, output)?,
602 detect::SemanticFormat::NQuadsStar => ingest_nquads_star(input, output)?,
603 detect::SemanticFormat::Turtle => ingest_turtle(input, output)?,
604 detect::SemanticFormat::TurtleStar => ingest_turtle_star(input, output)?,
605 detect::SemanticFormat::TriG => ingest_trig(input, output)?,
606 detect::SemanticFormat::TriGStar => ingest_trig_star(input, output)?,
607 detect::SemanticFormat::N3 => ingest_n3(input, output)?,
608 detect::SemanticFormat::RdfXml => ingest_rdf_xml(input, output)?,
609 detect::SemanticFormat::JsonLd => ingest_json_ld(input, output)?,
610 detect::SemanticFormat::JsonLdStar => ingest_json_ld_star(input, output)?,
611 detect::SemanticFormat::AgentIntentJsonl => {
612 agent_intent::ingest_agent_intent(input, output)?
613 }
614 detect::SemanticFormat::CborLd => ingest_cbor(input, output)?,
615 detect::SemanticFormat::Kml => ingest_kml(input, output)?,
616 detect::SemanticFormat::Mesh => ingest_asset(input, output)?,
617 detect::SemanticFormat::Chk => ingest_chk(input, output)?,
618 detect::SemanticFormat::Q42 => {
619 return Err("Q42 vaults are already in native format — no ingestion needed.".into())
620 }
621 };
622
623 let source_sha = {
624 use sha2::{Digest, Sha256};
625 let bytes = std::fs::read(input)?;
626 hex::encode(Sha256::digest(&bytes))
627 };
628 let prov = qualia_core_db::q42_volume::IngestProvenanceRecord::new(
629 access_policy,
630 source_sha,
631 mapping_version.unwrap_or("unspecified"),
632 context_digest.unwrap_or("unspecified"),
633 compiler_build_digest.unwrap_or(env!("CARGO_PKG_VERSION")),
634 );
635 prov.write_beside_volume(output)?;
636 Ok((stats, fmt))
637}