qualia_core_db/sparql_library/parsers/
turtle_doc.rs

1//! Turtle-document parser for QualiaDB ingest.
2//!
3//! The other parsers in this directory (`n3_star`, `turtle_star`) are line-oriented:
4//! they split each physical line on whitespace and keep the first three tokens. That
5//! is fine for N-Triples-shaped input but **shreds real Turtle** — multi-line
6//! predicate-object lists (`;`), object lists (`,`), and multi-word quoted literals
7//! all break, and `@prefix` is never expanded (CURIEs get hashed verbatim, so two
8//! documents' `doc:` terms collide).
9//!
10//! This parser handles the Turtle subset the values-credentials corpus actually uses:
11//!
12//! * `@prefix pfx: <iri> .` / `@base <iri> .` directives, expanded into full IRIs
13//!   before hashing (so a query with `PREFIX`/full `<IRI>` matches the stored hash,
14//!   and each instrument's `doc:` namespace is unique).
15//! * statements spanning multiple lines, terminated by `.`
16//! * `;` (repeat subject) and `,` (repeat subject + predicate) lists
17//! * `a` as a synonym for `rdf:type`
18//! * quoted literals — including spaces, escapes, `"""…"""`, and trailing
19//!   `@lang` / `^^datatype` tags (the tag is dropped; the lexical value is hashed)
20//!
21//! Terms are hashed with [`generate_60bit_token`] over the **expanded IRI** (or the
22//! literal's lexical form), matching the SPARQL query path.
23
24use crate::lexicon::generate_60bit_token;
25use crate::sparql_library::quin_sink::QuinSink;
26use crate::NQuin;
27use std::collections::HashMap;
28use std::io::Read;
29
30const RDF_TYPE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
31
32#[derive(Debug, Clone)]
33enum Tok {
34    Iri(String),       // <…>  (content, brackets stripped)
35    Pname(String),     // prefixed name or bare word (may end in ':')
36    A,                 // the `a` keyword (rdf:type)
37    Lit(String),       // quoted literal lexical value
38    Directive(String), // @prefix / @base (keyword without '@')
39    Semi,
40    Comma,
41    Dot,
42}
43
44fn tokenize(s: &str) -> Vec<Tok> {
45    let b = s.as_bytes();
46    let mut i = 0;
47    let mut out = Vec::new();
48    while i < b.len() {
49        let c = b[i];
50        // Classify on the raw byte, never `byte as char`: a 0xA0 continuation byte is
51        // whitespace under Latin-1 and would wrongly split a multibyte char. UTF-8 is
52        // self-synchronising, so ASCII delimiters never occur inside a multibyte sequence.
53        if c.is_ascii_whitespace() {
54            i += 1;
55            continue;
56        }
57        match c {
58            b'#' => {
59                while i < b.len() && b[i] != b'\n' {
60                    i += 1;
61                }
62            }
63            b'<' => {
64                let st = i + 1;
65                let mut j = st;
66                while j < b.len() && b[j] != b'>' {
67                    j += 1;
68                }
69                out.push(Tok::Iri(s[st..j].to_string()));
70                i = j + 1;
71            }
72            b'"' => {
73                let (lit, next) = read_literal(s, i);
74                out.push(Tok::Lit(lit));
75                i = next;
76                // Drop an optional language tag (@en) or datatype (^^xsd:string / ^^<iri>).
77                while i < b.len() {
78                    let d = b[i];
79                    if d == b'@'
80                        || d == b'^'
81                        || d == b':'
82                        || d.is_ascii_alphanumeric()
83                        || d == b'-'
84                        || d == b'<'
85                        || d == b'>'
86                        || d == b'/'
87                        || d == b'.'
88                        || d == b'#'
89                    {
90                        i += 1;
91                    } else {
92                        break;
93                    }
94                }
95            }
96            b';' => {
97                out.push(Tok::Semi);
98                i += 1;
99            }
100            b',' => {
101                out.push(Tok::Comma);
102                i += 1;
103            }
104            b'.' => {
105                out.push(Tok::Dot);
106                i += 1;
107            }
108            b'@' => {
109                let st = i + 1;
110                let mut j = st;
111                while j < b.len() && b[j].is_ascii_alphabetic() {
112                    j += 1;
113                }
114                out.push(Tok::Directive(s[st..j].to_string()));
115                i = j;
116            }
117            _ => {
118                let st = i;
119                while i < b.len() {
120                    let d = b[i];
121                    if d.is_ascii_whitespace()
122                        || d == b';'
123                        || d == b','
124                        || d == b'<'
125                        || d == b'"'
126                        || d == b'#'
127                    {
128                        break;
129                    }
130                    // A '.' terminates a word only when trailing (followed by ws/EOF/term);
131                    // CURIE local names here contain no internal dots.
132                    if d == b'.' {
133                        let nxt = b.get(i + 1).copied();
134                        if nxt.is_none()
135                            || nxt.unwrap().is_ascii_whitespace()
136                            || matches!(nxt, Some(b';') | Some(b',') | Some(b'#'))
137                        {
138                            break;
139                        }
140                    }
141                    i += 1;
142                }
143                let w = &s[st..i];
144                if w == "a" {
145                    out.push(Tok::A);
146                } else if !w.is_empty() {
147                    out.push(Tok::Pname(w.to_string()));
148                }
149            }
150        }
151    }
152    out
153}
154
155/// Read a `"…"` or `"""…"""` literal starting at `start` (the opening quote). Returns the
156/// unescaped lexical value and the byte index just past the closing quote(s).
157///
158/// The closing delimiter (`"`, escaped by ASCII `\`) is found by a byte scan — safe under
159/// UTF-8 self-synchronisation — and the inner text is then taken as a `&str` SLICE, never
160/// reconstructed byte-by-byte, so non-ASCII content (Arabic, CJK, Ge'ez, em-dashes, …) is
161/// preserved exactly. Slice bounds land on ASCII quote positions, i.e. valid char boundaries.
162fn read_literal(s: &str, start: usize) -> (String, usize) {
163    let b = s.as_bytes();
164    // Triple-quoted: """ … """
165    if b[start..].starts_with(b"\"\"\"") {
166        let inner = start + 3;
167        let mut j = inner;
168        while j + 3 <= b.len() && !(b[j] == b'"' && b[j + 1] == b'"' && b[j + 2] == b'"') {
169            j += 1;
170        }
171        let end = if j + 3 <= b.len() { j } else { b.len() };
172        return (unescape(&s[inner..end]), (end + 3).min(b.len()));
173    }
174    // Single-quoted: " … " — scan to the first unescaped closing quote.
175    let inner = start + 1;
176    let mut j = inner;
177    while j < b.len() {
178        match b[j] {
179            b'\\' if j + 1 < b.len() => j += 2, // skip the (ASCII) escape pair
180            b'"' => break,
181            _ => j += 1,
182        }
183    }
184    (unescape(&s[inner..j]), (j + 1).min(b.len()))
185}
186
187/// Unicode-safe unescape over a `&str` — iterates chars, never bytes.
188fn unescape(s: &str) -> String {
189    if !s.contains('\\') {
190        return s.to_string();
191    }
192    let mut out = String::with_capacity(s.len());
193    let mut chars = s.chars();
194    while let Some(c) = chars.next() {
195        if c == '\\' {
196            match chars.next() {
197                Some('n') => out.push('\n'),
198                Some('t') => out.push('\t'),
199                Some('r') => out.push('\r'),
200                Some('"') => out.push('"'),
201                Some('\\') => out.push('\\'),
202                Some(other) => out.push(other),
203                None => out.push('\\'),
204            }
205        } else {
206            out.push(c);
207        }
208    }
209    out
210}
211
212fn expand(pname: &str, prefixes: &HashMap<String, String>) -> String {
213    if let Some((pfx, local)) = pname.split_once(':') {
214        if let Some(base) = prefixes.get(pfx) {
215            return format!("{base}{local}");
216        }
217    }
218    pname.to_string()
219}
220
221/// Resolve a non-directive token to `(hash, canonical lexical string)`. `None` for
222/// punctuation. The string is what gets recorded in the lexicon for recovery: the
223/// expanded full IRI, the rdf:type IRI for `a`, or the literal's lexical value.
224fn resolve(tok: &Tok, prefixes: &HashMap<String, String>, base: &str) -> Option<(u64, String)> {
225    let s = match tok {
226        Tok::A => RDF_TYPE.to_string(),
227        Tok::Iri(iri) if !base.is_empty() && !iri.contains("://") && !iri.is_empty() => {
228            format!("{base}{iri}")
229        }
230        Tok::Iri(iri) => iri.clone(),
231        Tok::Pname(p) => expand(p, prefixes),
232        Tok::Lit(l) => l.clone(),
233        _ => return None,
234    };
235    Some((generate_60bit_token(s.as_bytes()), s))
236}
237
238/// Parse a Turtle document into `sink`, returning the number of triples emitted.
239pub fn parse_turtle_doc_into<R: Read, S: QuinSink>(
240    mut reader: R,
241    context_hash: u64,
242    sink: &mut S,
243) -> Result<u64, Box<dyn std::error::Error>> {
244    let mut text = String::new();
245    reader.read_to_string(&mut text)?;
246    let toks = tokenize(&text);
247
248    let mut prefixes: HashMap<String, String> = HashMap::new();
249    let mut base = String::new();
250    let mut subject: Option<u64> = None;
251    let mut predicate: Option<u64> = None;
252    let mut count = 0u64;
253
254    let mut i = 0;
255    while i < toks.len() {
256        match &toks[i] {
257            Tok::Directive(kw) => {
258                if kw.eq_ignore_ascii_case("prefix") {
259                    if let (Some(Tok::Pname(lbl)), Some(Tok::Iri(iri))) =
260                        (toks.get(i + 1), toks.get(i + 2))
261                    {
262                        prefixes.insert(lbl.trim_end_matches(':').to_string(), iri.clone());
263                    }
264                } else if kw.eq_ignore_ascii_case("base") {
265                    if let Some(Tok::Iri(iri)) = toks.get(i + 1) {
266                        base = iri.clone();
267                    }
268                }
269                // Skip to the directive-terminating Dot.
270                while i < toks.len() && !matches!(toks[i], Tok::Dot) {
271                    i += 1;
272                }
273                i += 1;
274                subject = None;
275                predicate = None;
276            }
277            Tok::Dot => {
278                subject = None;
279                predicate = None;
280                i += 1;
281            }
282            Tok::Semi => {
283                predicate = None;
284                i += 1;
285            }
286            Tok::Comma => {
287                i += 1;
288            }
289            tok => {
290                let (h, lex) = match resolve(tok, &prefixes, &base) {
291                    Some(pair) => pair,
292                    None => {
293                        i += 1;
294                        continue;
295                    }
296                };
297                sink.push_lex(h, &lex);
298                if subject.is_none() {
299                    subject = Some(h);
300                } else if predicate.is_none() {
301                    predicate = Some(h);
302                } else {
303                    let (s, p) = (subject.unwrap(), predicate.unwrap());
304                    sink.push(NQuin {
305                        subject: s,
306                        predicate: p,
307                        object: h,
308                        context: context_hash,
309                        metadata: 0,
310                        parity: s ^ p ^ h ^ context_hash,
311                    })?;
312                    count += 1;
313                }
314                i += 1;
315            }
316        }
317    }
318    Ok(count)
319}
320
321/// Streaming entry point used by the CLI ingest pipeline (writes via `ExternalSorter`).
322pub fn parse_turtle_doc_stream<R: Read>(
323    reader: R,
324    context_hash: u64,
325    sorter: &mut crate::external_sort::ExternalSorter,
326) -> Result<u64, Box<dyn std::error::Error>> {
327    parse_turtle_doc_into(reader, context_hash, sorter)
328}
329
330#[cfg(test)]
331mod tests {
332    use super::*;
333    use crate::lexicon::generate_60bit_token as h;
334
335    /// Collect every emitted triple into a Vec (test sink).
336    #[derive(Default)]
337    struct VecSink(Vec<NQuin>);
338    impl QuinSink for VecSink {
339        fn push(&mut self, q: NQuin) -> std::io::Result<()> {
340            self.0.push(q);
341            Ok(())
342        }
343    }
344
345    fn parse(doc: &str) -> Vec<NQuin> {
346        let mut sink = VecSink::default();
347        parse_turtle_doc_into(doc.as_bytes(), 0, &mut sink).unwrap();
348        sink.0
349    }
350
351    #[test]
352    fn multiline_predicate_object_list_with_prefix_and_literals() {
353        let doc = r#"
354@prefix dc:     <http://purl.org/dc/terms/> .
355@prefix values: <https://ns.webcivics.net/values/> .
356@prefix doc:    <https://ns.webcivics.net/values/inst#> .
357
358doc:article-1 a values:Undertaking ;
359    dc:title "Article 1" ;
360    values:partOf doc:Instrument ;
361    values:originalText "Each Member undertakes to suppress forced labour." .
362"#;
363        let quins = parse(doc);
364        assert_eq!(quins.len(), 4, "subject reused across `;` → four triples");
365
366        let art1 = h(b"https://ns.webcivics.net/values/inst#article-1");
367        let rdf_type = h(b"http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
368        let undertaking = h(b"https://ns.webcivics.net/values/Undertaking");
369        let dc_title = h(b"http://purl.org/dc/terms/title");
370        let part_of = h(b"https://ns.webcivics.net/values/partOf");
371
372        // `a` expands to rdf:type; CURIEs expand via @prefix.
373        assert!(quins
374            .iter()
375            .any(|q| q.subject == art1 && q.predicate == rdf_type && q.object == undertaking));
376        // dc:title is a PREDICATE on a continuation line, subject carried over — the bug we fixed.
377        assert!(quins
378            .iter()
379            .any(|q| q.subject == art1 && q.predicate == dc_title));
380        // Multi-word literal hashes as ONE object (not shredded into words).
381        let title = h(b"Article 1");
382        assert!(quins
383            .iter()
384            .any(|q| q.predicate == dc_title && q.object == title));
385        // partOf links to the doc:-expanded Instrument (now namespace-unique).
386        let instrument = h(b"https://ns.webcivics.net/values/inst#Instrument");
387        assert!(quins
388            .iter()
389            .any(|q| q.subject == art1 && q.predicate == part_of && q.object == instrument));
390    }
391
392    #[test]
393    fn object_list_comma_repeats_subject_and_predicate() {
394        let doc = r#"
395@prefix values: <https://ns.webcivics.net/values/> .
396values:State values:bears values:DutyA , values:DutyB , values:DutyC .
397"#;
398        let quins = parse(doc);
399        assert_eq!(
400            quins.len(),
401            3,
402            "`,` repeats subject+predicate → three triples"
403        );
404        let state = h(b"https://ns.webcivics.net/values/State");
405        let bears = h(b"https://ns.webcivics.net/values/bears");
406        assert!(quins
407            .iter()
408            .all(|q| q.subject == state && q.predicate == bears));
409    }
410
411    /// Regression: non-ASCII literals (Arabic, CJK, em-dash, curly quotes) must survive intact.
412    /// A correct object hash proves the lexical string was preserved byte-exact — a byte-by-byte
413    /// `byte as char` reconstruction would hash differently (and could panic mid-codepoint).
414    #[test]
415    fn non_ascii_literals_roundtrip_intact() {
416        let doc = "@prefix v: <https://ns.webcivics.net/values/> .\n\
417                   v:x v:label \"صحة\" ; v:note \"健康 — wellbeing's “root”\" .";
418        let quins = parse(doc);
419        let label = h(b"https://ns.webcivics.net/values/label");
420        let note = h(b"https://ns.webcivics.net/values/note");
421        assert!(
422            quins
423                .iter()
424                .any(|q| q.predicate == label && q.object == h("صحة".as_bytes())),
425            "Arabic literal must hash to its exact UTF-8 bytes"
426        );
427        assert!(
428            quins
429                .iter()
430                .any(|q| q.predicate == note
431                    && q.object == h("健康 — wellbeing's “root”".as_bytes())),
432            "CJK + em-dash + curly-quote literal must round-trip intact"
433        );
434    }
435}
qualia_core_db/sparql_library/parsers/turtle_doc.rs

qualia_core_db/sparql_library/parsers/
turtle_doc.rs
