1use std::io::{BufRead, Cursor, Read};
5
6use rio_api::model::{GraphName, Literal, Quad, Subject, Term, Triple};
7use rio_api::parser::{QuadsParser, TriplesParser};
8use rio_turtle::{NQuadsParser, NTriplesParser, TriGParser, TurtleParser};
9use rio_xml::RdfXmlParser;
10
11use crate::modalities::logic::n3_parser::{
12 N3Event, N3Parser, RuleType, Term as N3Term, Triple as N3Triple,
13};
14use crate::q_hash;
15use crate::query::cbor_compiler::parse_cbor_ld_to_quin_sized;
16use crate::query::ingest_job::IngestRdfFormat;
17
18pub const OBJECT_IRI_MASK: u64 = 0x0FFF_FFFF_FFFF_FFFF;
20
21pub fn object_iri_hash(iri: &str) -> u64 {
23 q_hash(iri) & OBJECT_IRI_MASK
24}
25
26#[derive(Debug)]
28pub struct RawTriple {
29 pub subject: String,
30 pub predicate: String,
31 pub object: String,
32 pub packed_object: Option<u64>,
33 pub context: u64,
34}
35
36pub fn format_from_path(path_lower: &str) -> IngestRdfFormat {
38 crate::query::ingest_job::infer_rdf_format(path_lower)
39}
40
41pub fn pack_rio_triple(t: Triple<'_>, context: u64) -> RawTriple {
43 let (object, packed_object) = pack_term(t.object);
44 RawTriple {
45 subject: pack_subject(t.subject),
46 predicate: t.predicate.iri.to_string(),
47 object,
48 packed_object,
49 context,
50 }
51}
52
53fn pack_subject(s: Subject<'_>) -> String {
54 match s {
55 Subject::NamedNode(n) => n.iri.to_string(),
56 Subject::BlankNode(b) => format!("_:{}", b.id),
57 Subject::Triple(inner) => inner.to_string(),
58 }
59}
60
61fn pack_term(term: Term<'_>) -> (String, Option<u64>) {
62 match term {
63 Term::NamedNode(n) => (n.iri.to_string(), None),
64 Term::BlankNode(b) => (format!("_:{}", b.id), None),
65 Term::Literal(Literal::Simple { value }) => (value.to_string(), None),
66 Term::Literal(Literal::LanguageTaggedString { value, language }) => {
67 (format!("{value}@{language}"), None)
68 }
69 Term::Literal(Literal::Typed { value, datatype }) => {
70 if datatype.iri == crate::resolver::typed_double::DATATYPE {
71 return (Literal::Typed { value, datatype }.to_string(), None);
75 }
76 (value.to_string(), pack_typed_literal(value, datatype.iri))
77 }
78 Term::Triple(inner) => (inner.to_string(), None),
79 }
80}
81
82pub fn repair_rdfxml_empty_base(src: &str, base: &str) -> String {
84 src.replace("xml:base=\"\"", &format!("xml:base=\"{base}\""))
85 .replace("xml:base=''", &format!("xml:base='{base}'"))
86}
87
88pub fn pack_rio_quad(q: Quad<'_>) -> RawTriple {
90 let context = match q.graph_name {
91 Some(GraphName::NamedNode(n)) => q_hash(n.iri),
92 Some(GraphName::BlankNode(b)) => q_hash(b.id),
93 None => 0,
94 };
95 pack_rio_triple(
96 Triple {
97 subject: q.subject,
98 predicate: q.predicate,
99 object: q.object,
100 },
101 context,
102 )
103}
104
105fn pack_typed_literal(value: &str, dt: &str) -> Option<u64> {
106 crate::resolver::pack_typed_literal(value, dt)
107}
108
109pub fn parse_triples_format<R, F>(
111 format: IngestRdfFormat,
112 reader: R,
113 base_iri: Option<oxiri::Iri<String>>,
114 on_triple: &mut F,
115) -> Result<(), String>
116where
117 R: BufRead,
118 F: FnMut(RawTriple),
119{
120 match format {
121 IngestRdfFormat::NTriples => {
122 let mut p = NTriplesParser::new(reader);
123 p.parse_all(&mut |t| {
124 on_triple(pack_rio_triple(t, 0));
125 Ok(()) as Result<(), rio_turtle::TurtleError>
126 })
127 .map_err(|e| format!("N-Triples: {e}"))
128 }
129 IngestRdfFormat::Turtle | IngestRdfFormat::Auto => {
130 let mut p = TurtleParser::new(reader, base_iri);
131 p.parse_all(&mut |t| {
132 on_triple(pack_rio_triple(t, 0));
133 Ok(()) as Result<(), rio_turtle::TurtleError>
134 })
135 .map_err(|e| format!("Turtle: {e}"))
136 }
137 IngestRdfFormat::NQuads => {
138 let mut p = NQuadsParser::new(reader);
139 p.parse_all(&mut |q| {
140 on_triple(pack_rio_quad(q));
141 Ok(()) as Result<(), rio_turtle::TurtleError>
142 })
143 .map_err(|e| format!("N-Quads: {e}"))
144 }
145 IngestRdfFormat::TriG => {
146 let mut p = TriGParser::new(reader, base_iri);
147 p.parse_all(&mut |q| {
148 on_triple(pack_rio_quad(q));
149 Ok(()) as Result<(), rio_turtle::TurtleError>
150 })
151 .map_err(|e| format!("TriG: {e}"))
152 }
153 IngestRdfFormat::RdfXml => {
154 let mut p = RdfXmlParser::new(reader, base_iri);
155 p.parse_all(&mut |t| {
156 on_triple(pack_rio_triple(t, 0));
157 Ok(()) as Result<(), rio_xml::RdfXmlError>
158 })
159 .map_err(|e| format!("RDF/XML: {e}"))
160 }
161 IngestRdfFormat::JsonLd => parse_jsonld(reader, on_triple),
162 IngestRdfFormat::YamlLd => parse_yamlld(reader, on_triple),
163 IngestRdfFormat::RdfJson => parse_rdfjson(reader, on_triple),
164 IngestRdfFormat::CborLd => parse_cbor_ld_stream(reader, on_triple),
165 IngestRdfFormat::N3 => parse_n3_stream(reader, on_triple),
166 }
167}
168
169fn n3_term_str(t: N3Term<'_>) -> String {
170 match t {
171 N3Term::Uri(s) | N3Term::Variable(s) | N3Term::Formula(s) => {
172 s.trim_start_matches('<').trim_end_matches('>').to_string()
173 }
174 N3Term::Literal(s) => s.trim_matches('"').to_string(),
175 }
176}
177
178fn pack_n3_triple(t: N3Triple<'_>, context: u64) -> RawTriple {
179 RawTriple {
180 subject: n3_term_str(t.subject),
181 predicate: n3_term_str(t.predicate),
182 object: n3_term_str(t.object),
183 packed_object: None,
184 context,
185 }
186}
187
188fn parse_n3_stream<R, F>(mut reader: R, on_triple: &mut F) -> Result<(), String>
189where
190 R: Read,
191 F: FnMut(RawTriple),
192{
193 let mut text = String::new();
194 reader
195 .read_to_string(&mut text)
196 .map_err(|e| format!("N3 read: {e}"))?;
197 let mut parser = N3Parser::new(&text);
198 let mut rule_idx = 0usize;
199 parser
200 .parse_all(|event| {
201 match event {
202 N3Event::StaticTriple(t) => {
203 on_triple(pack_n3_triple(t, 0));
204 }
205 N3Event::LogicRule(rule) => {
206 let rule_ctx = q_hash(&format!("urn:qualia:n3:rule:{rule_idx}"));
207 for &pt in &rule.premise.triples {
208 on_triple(pack_n3_triple(pt, rule_ctx));
209 }
210 for &ct in &rule.conclusion.triples {
211 on_triple(pack_n3_triple(ct, rule_ctx));
212 }
213 let rule_pred = match rule.rule_type {
214 RuleType::Strict => "http://www.w3.org/2000/10/swap/log#implies",
215 RuleType::Defeasible => "urn:qualia:n3:defeasiblyImplies",
216 RuleType::Defeater => "urn:qualia:n3:defeats",
217 RuleType::Linear => "urn:qualia:n3:linearImplies",
218 };
219 on_triple(RawTriple {
220 subject: format!("_:n3_premise_{rule_idx}"),
221 predicate: rule_pred.to_string(),
222 object: format!("_:n3_conclusion_{rule_idx}"),
223 packed_object: None,
224 context: rule_ctx,
225 });
226 rule_idx += 1;
227 }
228 N3Event::AspBlock(_) | N3Event::DiffuseBlock(_) => {}
229 }
230 Ok(())
231 })
232 .map_err(|e| format!("N3: {e}"))
233}
234
235fn parse_jsonld<R, F>(mut reader: R, on_triple: &mut F) -> Result<(), String>
236where
237 R: Read,
238 F: FnMut(RawTriple),
239{
240 let v = read_json_document(&mut reader, "JSON-LD")?;
241 emit_jsonld_value(&v, on_triple)
242}
243
244fn parse_yamlld<R, F>(mut reader: R, on_triple: &mut F) -> Result<(), String>
245where
246 R: Read,
247 F: FnMut(RawTriple),
248{
249 let mut bytes = Vec::new();
250 reader
251 .read_to_end(&mut bytes)
252 .map_err(|e| format!("YAML-LD read: {e}"))?;
253 if bytes.len() > 32 * 1024 * 1024 {
254 return Err(
255 "YAML-LD ingest is limited to 32 MiB (use Turtle/N-Triples/N-Quads for dumps)".into(),
256 );
257 }
258 let v: serde_json::Value =
259 serde_yaml::from_slice(&bytes).map_err(|e| format!("YAML-LD: {e}"))?;
260 emit_jsonld_value(&v, on_triple)
261}
262
263fn parse_rdfjson<R, F>(mut reader: R, on_triple: &mut F) -> Result<(), String>
264where
265 R: Read,
266 F: FnMut(RawTriple),
267{
268 let v = read_json_document(&mut reader, "RDF/JSON")?;
269 emit_rdfjson_value(&v, on_triple)
270}
271
272fn read_json_document<R: Read>(reader: &mut R, label: &str) -> Result<serde_json::Value, String> {
273 let mut bytes = Vec::new();
274 reader
275 .read_to_end(&mut bytes)
276 .map_err(|e| format!("{label} read: {e}"))?;
277 if bytes.len() > 32 * 1024 * 1024 {
278 return Err(format!(
279 "{label} ingest is limited to 32 MiB (use Turtle/N-Triples/N-Quads for dumps)"
280 ));
281 }
282 serde_json::from_slice(&bytes).map_err(|e| format!("{label}: {e}"))
283}
284
285fn emit_rdfjson_value<F>(v: &serde_json::Value, on_triple: &mut F) -> Result<(), String>
286where
287 F: FnMut(RawTriple),
288{
289 let Some(subjects) = v.as_object() else {
290 return Err("RDF/JSON must be an object of subject → predicate maps".into());
291 };
292 for (subj, preds) in subjects {
293 let Some(pred_map) = preds.as_object() else {
294 continue;
295 };
296 for (pred, objects) in pred_map {
297 let items = if let Some(arr) = objects.as_array() {
298 arr.as_slice()
299 } else {
300 std::slice::from_ref(objects)
301 };
302 for item in items {
303 let Some(obj) = item.as_object() else {
304 continue;
305 };
306 let kind = obj
307 .get("type")
308 .and_then(|t| t.as_str())
309 .unwrap_or("literal");
310 let value = obj
311 .get("value")
312 .and_then(|t| t.as_str())
313 .unwrap_or("")
314 .to_string();
315 if value.is_empty() {
316 continue;
317 }
318 let packed_object = match kind {
319 "uri" | "iri" | "bnode" => None,
320 _ => obj
321 .get("datatype")
322 .and_then(|t| t.as_str())
323 .and_then(|dt| pack_typed_literal(&value, dt)),
324 };
325 let object = if !matches!(kind, "uri" | "iri" | "bnode")
326 && obj.get("datatype").and_then(|t| t.as_str())
327 == Some(crate::resolver::typed_double::DATATYPE)
328 {
329 Literal::Typed {
330 value: &value,
331 datatype: rio_api::model::NamedNode {
332 iri: crate::resolver::typed_double::DATATYPE,
333 },
334 }
335 .to_string()
336 } else if kind == "bnode" && !value.starts_with("_:") {
337 format!("_:{value}")
338 } else if let Some(lang) = obj.get("lang").and_then(|t| t.as_str()) {
339 format!("{value}@{lang}")
340 } else {
341 value
342 };
343 on_triple(RawTriple {
344 subject: subj.clone(),
345 predicate: pred.clone(),
346 object,
347 packed_object,
348 context: 0,
349 });
350 }
351 }
352 }
353 Ok(())
354}
355
356fn emit_jsonld_value<F>(v: &serde_json::Value, on_triple: &mut F) -> Result<(), String>
357where
358 F: FnMut(RawTriple),
359{
360 if let Ok(flat) = serde_json::from_value::<Vec<FlatTriple>>(v.clone()) {
361 for t in flat {
362 on_triple(RawTriple {
363 subject: t.s,
364 predicate: t.p,
365 object: t.o,
366 packed_object: None,
367 context: t.g.map(|g| q_hash(&g)).unwrap_or(0),
368 });
369 }
370 return Ok(());
371 }
372 let nodes = if let Some(g) = v.get("@graph").and_then(|g| g.as_array()) {
373 g.clone()
374 } else if v.is_array() {
375 v.as_array().cloned().unwrap_or_default()
376 } else {
377 vec![v.clone()]
378 };
379 for node in nodes {
380 let Some(subj) = node.get("@id").and_then(|x| x.as_str()).map(str::to_string) else {
381 continue;
382 };
383 if let Some(obj) = &node.as_object() {
384 for (k, val) in *obj {
385 if k.starts_with('@') {
386 continue;
387 }
388 emit_jsonld_pred(&subj, k, val, 0, on_triple)?;
389 }
390 }
391 }
392 Ok(())
393}
394
395fn emit_jsonld_pred<F>(
396 subj: &str,
397 pred: &str,
398 val: &serde_json::Value,
399 context: u64,
400 on_triple: &mut F,
401) -> Result<(), String>
402where
403 F: FnMut(RawTriple),
404{
405 match val {
406 serde_json::Value::Array(items) => {
407 for item in items {
408 emit_jsonld_pred(subj, pred, item, context, on_triple)?;
409 }
410 }
411 serde_json::Value::Object(map) => {
412 let obj = map
413 .get("@id")
414 .and_then(|x| x.as_str())
415 .or_else(|| map.get("@value").and_then(|x| x.as_str()))
416 .unwrap_or("")
417 .to_string();
418 if !obj.is_empty() {
419 on_triple(RawTriple {
420 subject: subj.to_string(),
421 predicate: pred.to_string(),
422 object: obj,
423 packed_object: None,
424 context,
425 });
426 }
427 }
428 serde_json::Value::String(s) => {
429 on_triple(RawTriple {
430 subject: subj.to_string(),
431 predicate: pred.to_string(),
432 object: s.clone(),
433 packed_object: None,
434 context,
435 });
436 }
437 serde_json::Value::Number(n) => {
438 on_triple(RawTriple {
439 subject: subj.to_string(),
440 predicate: pred.to_string(),
441 object: n.to_string(),
442 packed_object: None,
443 context,
444 });
445 }
446 serde_json::Value::Bool(b) => {
447 on_triple(RawTriple {
448 subject: subj.to_string(),
449 predicate: pred.to_string(),
450 object: b.to_string(),
451 packed_object: Some(crate::resolver::INLINE_TAG_BOOLEAN | u64::from(*b)),
452 context,
453 });
454 }
455 _ => {}
456 }
457 Ok(())
458}
459
460#[derive(serde::Deserialize)]
461struct FlatTriple {
462 s: String,
463 p: String,
464 o: String,
465 g: Option<String>,
466}
467
468fn parse_cbor_ld_stream<R, F>(mut reader: R, on_triple: &mut F) -> Result<(), String>
469where
470 R: Read,
471 F: FnMut(RawTriple),
472{
473 let mut bytes = Vec::new();
474 reader
475 .read_to_end(&mut bytes)
476 .map_err(|e| format!("CBOR-LD read: {e}"))?;
477 let mut rest = bytes.as_slice();
478 while !rest.is_empty() {
479 match parse_cbor_ld_to_quin_sized(rest) {
480 Ok((quin, consumed)) if consumed > 0 => {
481 on_triple(RawTriple {
482 subject: quin.subject.to_string(),
483 predicate: quin.predicate.to_string(),
484 object: quin.object.to_string(),
485 packed_object: Some(quin.object),
486 context: quin.context,
487 });
488 rest = &rest[consumed..];
489 }
490 _ => break,
491 }
492 }
493 Ok(())
494}
495
496pub fn parse_bytes_to_raw(format: IngestRdfFormat, bytes: &[u8]) -> Result<Vec<RawTriple>, String> {
498 let mut out = Vec::new();
499 parse_triples_format(format, Cursor::new(bytes), None, &mut |t| {
500 out.push(t);
501 })?;
502 Ok(out)
503}
504
505#[cfg(test)]
506mod tests {
507 use super::*;
508
509 #[test]
510 fn ntriples_and_nquads_parse() {
511 let nt = b"<http://ex/s> <http://ex/p> <http://ex/o> .\n";
512 let got = parse_bytes_to_raw(IngestRdfFormat::NTriples, nt).unwrap();
513 assert_eq!(got.len(), 1);
514 assert_eq!(got[0].subject, "http://ex/s");
515 assert_eq!(got[0].predicate, "http://ex/p");
516 assert_eq!(got[0].object, "http://ex/o");
517 let nq = b"<http://ex/s> <http://ex/p> <http://ex/o> <http://ex/g> .\n";
518 let got = parse_bytes_to_raw(IngestRdfFormat::NQuads, nq).unwrap();
519 assert_eq!(got.len(), 1);
520 assert_ne!(got[0].context, 0);
521 }
522
523 #[test]
524 fn turtle_and_trig_parse() {
525 let ttl = b"@prefix ex: <http://ex/> .\nex:s ex:p ex:o .\n";
526 let got = parse_bytes_to_raw(IngestRdfFormat::Turtle, ttl).unwrap();
527 assert_eq!(got.len(), 1);
528 let trig = b"@prefix ex: <http://ex/> .\ngraph ex:g { ex:s ex:p ex:o . }\n";
529 let got = parse_bytes_to_raw(IngestRdfFormat::TriG, trig).unwrap();
530 assert_eq!(got.len(), 1);
531 assert_ne!(got[0].context, 0);
532 }
533
534 #[test]
535 fn jsonld_flat_and_graph() {
536 let flat = br#"[{"s":"http://ex/s","p":"http://ex/p","o":"http://ex/o"}]"#;
537 let got = parse_bytes_to_raw(IngestRdfFormat::JsonLd, flat).unwrap();
538 assert_eq!(got.len(), 1);
539 let g = br#"{"@graph":[{"@id":"http://ex/s","http://ex/p":{"@id":"http://ex/o"}}]}"#;
540 let got = parse_bytes_to_raw(IngestRdfFormat::JsonLd, g).unwrap();
541 assert_eq!(got.len(), 1);
542 assert_eq!(got[0].object, "http://ex/o");
543 }
544
545 #[test]
546 fn rdfxml_yamlld_and_rdfjson_parse() {
547 let xml = br#"<?xml version="1.0"?>
548<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
549 xmlns:ex="http://ex/">
550 <rdf:Description rdf:about="http://ex/s">
551 <ex:p rdf:resource="http://ex/o"/>
552 </rdf:Description>
553</rdf:RDF>"#;
554 let got = parse_bytes_to_raw(IngestRdfFormat::RdfXml, xml).unwrap();
555 assert_eq!(got.len(), 1);
556 assert_eq!(got[0].subject, "http://ex/s");
557 assert_eq!(got[0].object, "http://ex/o");
558
559 let yaml = br#""@graph":
560 - "@id": http://ex/s
561 "http://ex/p":
562 "@id": http://ex/o
563"#;
564 let got = parse_bytes_to_raw(IngestRdfFormat::YamlLd, yaml).unwrap();
565 assert_eq!(got.len(), 1);
566 assert_eq!(got[0].object, "http://ex/o");
567
568 let rj = br#"{"http://ex/s":{"http://ex/p":[{"type":"uri","value":"http://ex/o"}]}}"#;
569 let got = parse_bytes_to_raw(IngestRdfFormat::RdfJson, rj).unwrap();
570 assert_eq!(got.len(), 1);
571 assert_eq!(got[0].subject, "http://ex/s");
572 assert_eq!(got[0].object, "http://ex/o");
573 }
574
575 #[test]
576 fn formats_share_the_same_iri_spelling() {
577 let nt = parse_bytes_to_raw(
578 IngestRdfFormat::NTriples,
579 b"<http://ex/s> <http://ex/p> <http://ex/o> .\n",
580 )
581 .unwrap();
582 let ttl = parse_bytes_to_raw(
583 IngestRdfFormat::Turtle,
584 b"@prefix ex: <http://ex/> .\nex:s ex:p ex:o .\n",
585 )
586 .unwrap();
587 assert_eq!(nt[0].subject, ttl[0].subject);
588 assert_eq!(nt[0].object, ttl[0].object);
589 assert_eq!(
590 object_iri_hash(&nt[0].object),
591 object_iri_hash("http://ex/o")
592 );
593 }
594
595 #[test]
596 fn typed_double_rdfjson_matches_complete_rio_term_without_f32_rounding() {
597 let nt = parse_bytes_to_raw(IngestRdfFormat::NTriples,
598 b"<urn:s> <urn:p> \"1.0000000000000002\"^^<http://www.w3.org/2001/XMLSchema#double> .\n").unwrap();
599 let json = parse_bytes_to_raw(IngestRdfFormat::RdfJson,
600 br#"{"urn:s":{"urn:p":[{"type":"literal","value":"1.0000000000000002","datatype":"http://www.w3.org/2001/XMLSchema#double"}]}}"#).unwrap();
601 assert_eq!(nt[0].object, json[0].object);
602 assert!(nt[0].packed_object.is_none());
603 assert!(json[0].packed_object.is_none());
604 assert_eq!(
605 crate::resolver::typed_double::decode(&nt[0].object)
606 .unwrap()
607 .1
608 .to_bits(),
609 1.0000000000000002f64.to_bits()
610 );
611 }
612
613 #[test]
614 fn n3_and_cbor_ld_stream_parse() {
615 let n3 = b":alice a :NaturalAgent .\n{ ?x a :CorporatePerson } => { ?x :flag :Deny } .\n";
616 let got_n3 = parse_bytes_to_raw(IngestRdfFormat::N3, n3).unwrap();
617 assert!(
618 got_n3.len() >= 4,
619 "must emit static triple + reified rule triples"
620 );
621
622 let frame1: [u8; 13] = [
625 0x84, 0x19, 0x00, 0x84, 0x19, 0x07, 0xD0, 0x19, 0x0B, 0xB8, 0x19, 0x0F, 0xA0,
626 ];
627 let mut two_frames = Vec::new();
628 two_frames.extend_from_slice(&frame1);
629 two_frames.extend_from_slice(&frame1);
630 let got_cbor = parse_bytes_to_raw(IngestRdfFormat::CborLd, &two_frames).unwrap();
631 assert_eq!(got_cbor.len(), 2);
632 assert_eq!(got_cbor[0].subject, "132");
633 assert_eq!(got_cbor[1].subject, "132");
634 }
635}