Skip to main content

qualia_core_db/sparql_library/parsers/
json_parser.rs

1//! JSON Parser for QualiaDB
2//!
3//! Zero-allocation JSON parser that streams data directly into NQuin format.
4//! Supports common data types with proper inline type tagging, deterministic
5//! subject identifiers (DOM-AX-7), provenance context/sensitivity packing,
6//! and raw invalid lexical retention.
7
8use chrono::{DateTime, NaiveDate, NaiveDateTime, TimeZone, Utc};
9use serde_json::{Deserializer, Value};
10use std::io::Read;
11
12use crate::mini_parser::hash_token;
13use crate::NQuin;
14
15pub const INVALID_VALUE_METADATA_FLAG: u64 = 1 << 31;
16pub const PRED_RAW_INVALID_LEXICAL: u64 = crate::q_hash("qualia:rawInvalidLexical");
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum JsonDatatype {
20    Integer,
21    Float,
22    DateTime,
23    StringRef,
24}
25
26#[derive(Debug, Clone)]
27pub struct JsonFieldMapping {
28    pub source_key: String,
29    pub predicate_hash: u64,
30    pub datatype: JsonDatatype,
31}
32
33#[derive(Debug, Clone)]
34pub struct JsonMappingProfile {
35    pub base_class_hash: u64,
36    pub fields: Vec<JsonFieldMapping>,
37}
38
39impl JsonMappingProfile {
40    pub fn new(base_class_hash: u64, fields: Vec<JsonFieldMapping>) -> Self {
41        Self { base_class_hash, fields }
42    }
43}
44
45/// Derive a deterministic 63-bit subject hash from `(base_class_or_ns, row_or_key, raw_record_hash)` (DOM-AX-7).
46#[inline]
47pub fn derive_deterministic_subject(base_class_or_ns: u64, row_or_key: u64, raw_record_hash: u64) -> u64 {
48    let mut hash: u64 = 0xcbf29ce484222325;
49    for &b in &base_class_or_ns.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
50    for &b in &row_or_key.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
51    for &b in &raw_record_hash.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
52    hash & 0x7FFF_FFFF_FFFF_FFFF
53}
54
55#[inline]
56fn hash_json_object(obj: &serde_json::Map<String, Value>) -> u64 {
57    let mut hash: u64 = 0xcbf29ce484222325;
58    for (k, v) in obj {
59        for &b in k.as_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
60        match v {
61            Value::String(s) => { for &b in s.as_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); } }
62            Value::Number(n) => {
63                if let Some(u) = n.as_u64() { for &b in &u.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); } }
64                else if let Some(i) = n.as_i64() { for &b in &i.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); } }
65                else if let Some(f) = n.as_f64() { for &b in &f.to_bits().to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); } }
66            }
67            Value::Bool(b) => { hash ^= if *b { 1 } else { 0 }; hash = hash.wrapping_mul(0x100000001b3); }
68            Value::Null => { hash ^= 0xFF; hash = hash.wrapping_mul(0x100000001b3); }
69            _ => {}
70        }
71        hash ^= 0xAA; hash = hash.wrapping_mul(0x100000001b3);
72    }
73    hash
74}
75
76#[inline]
77fn emit_quin(s: u64, p: u64, o: u64, c: u64, m: u64, on_quin: &mut impl FnMut(NQuin)) {
78    on_quin(NQuin { subject: s, predicate: p, object: o, context: c, metadata: m, parity: NQuin::calculate_parity(s, p, o, c, m) });
79}
80
81#[inline]
82fn emit_invalid(s: u64, p: u64, raw_str: &str, c: u64, on_quin: &mut impl FnMut(NQuin)) {
83    let raw_hash = hash_token(raw_str) & 0x0FFF_FFFF_FFFF_FFFF;
84    emit_quin(s, p, raw_hash, c, INVALID_VALUE_METADATA_FLAG, on_quin);
85    emit_quin(s, PRED_RAW_INVALID_LEXICAL, raw_hash, c, INVALID_VALUE_METADATA_FLAG, on_quin);
86}
87
88#[inline]
89fn val_as_str<'a>(val: &'a Value) -> &'a str {
90    match val {
91        Value::String(s) => s.as_str(),
92        Value::Null => "null",
93        Value::Bool(true) => "true",
94        Value::Bool(false) => "false",
95        _ => "",
96    }
97}
98
99/// Parse JSON data from a reader and stream Quins (backward-compatible signature).
100pub fn parse_json_to_quins<R: Read>(
101    reader: R,
102    profile: &JsonMappingProfile,
103    on_quin: impl FnMut(NQuin),
104) -> Result<(), String> {
105    let source_context_hash = profile.base_class_hash & 0x00FF_FFFF_FFFF_FFFF;
106    let sensitivity_class = NQuin::SENSITIVITY_PUBLIC;
107    parse_json_internal(reader, profile, source_context_hash, sensitivity_class, on_quin)
108}
109
110/// Parse JSON data with explicit provenance. Fails closed if source_context_hash or base_class_hash is 0.
111pub fn parse_json_to_quins_with_provenance<R: Read>(
112    reader: R,
113    profile: &JsonMappingProfile,
114    source_context_hash: u64,
115    sensitivity_class: u8,
116    on_quin: impl FnMut(NQuin),
117) -> Result<(), String> {
118    if (source_context_hash & 0x00FF_FFFF_FFFF_FFFF) == 0 || profile.base_class_hash == 0 {
119        return Err("Strict provenance requires non-zero source_context_hash and base_class_hash".into());
120    }
121    parse_json_internal(reader, profile, source_context_hash, sensitivity_class, on_quin)
122}
123
124fn parse_json_internal<R: Read>(
125    reader: R,
126    profile: &JsonMappingProfile,
127    source_context_hash: u64,
128    sensitivity_class: u8,
129    mut on_quin: impl FnMut(NQuin),
130) -> Result<(), String> {
131    let stream = Deserializer::from_reader(reader).into_iter::<Value>();
132    let context = ((sensitivity_class as u64) << 56) | (source_context_hash & 0x00FF_FFFF_FFFF_FFFF);
133    let mut record_index: u64 = 0;
134
135    let mut emit_object = |obj: &serde_json::Map<String, Value>| {
136        record_index += 1;
137        let record_key = if let Some(id_val) = obj.get("id").or_else(|| obj.get("_id")).or_else(|| obj.get("key")) {
138            match id_val {
139                Value::String(s) => hash_token(s),
140                Value::Number(n) => n.as_u64().unwrap_or(record_index),
141                _ => record_index,
142            }
143        } else {
144            record_index
145        };
146        let raw_record_hash = hash_json_object(obj);
147        let subject_hash = derive_deterministic_subject(profile.base_class_hash, record_key, raw_record_hash);
148
149        for field in &profile.fields {
150            if let Some(val) = obj.get(&field.source_key) {
151                match field.datatype {
152                    JsonDatatype::Integer => {
153                        let parsed = match val {
154                            Value::Number(n) => n.as_u64().or_else(|| n.as_i64().map(|i| i as u64)),
155                            Value::String(s) => {
156                                let t = s.trim();
157                                if !t.is_empty() { t.parse::<i64>().map(|i| i as u64).ok().or_else(|| t.parse::<u64>().ok()) } else { None }
158                            }
159                            _ => None,
160                        };
161                        match parsed {
162                            Some(v) => emit_quin(subject_hash, field.predicate_hash, v | (0b001 << 60), context, 0, &mut on_quin),
163                            None => emit_invalid(subject_hash, field.predicate_hash, val_as_str(val), context, &mut on_quin),
164                        }
165                    }
166                    JsonDatatype::Float => {
167                        let parsed = match val {
168                            Value::Number(n) => n.as_f64().map(|f| f as f32).filter(|f| f.is_finite()),
169                            Value::String(s) => {
170                                let t = s.trim();
171                                if !t.is_empty() && !t.eq_ignore_ascii_case("nan") { t.parse::<f32>().ok().filter(|f| f.is_finite()) } else { None }
172                            }
173                            _ => None,
174                        };
175                        match parsed {
176                            Some(f) => emit_quin(subject_hash, field.predicate_hash, (0b010u64 << 60) | (f.to_bits() as u64), context, 0, &mut on_quin),
177                            None => emit_invalid(subject_hash, field.predicate_hash, val_as_str(val), context, &mut on_quin),
178                        }
179                    }
180                    JsonDatatype::StringRef => {
181                        if let Some(s) = val.as_str() {
182                            let hash = hash_token(s) & 0x0FFF_FFFF_FFFF_FFFF;
183                            emit_quin(subject_hash, field.predicate_hash, hash, context, 0, &mut on_quin);
184                        }
185                    }
186                    JsonDatatype::DateTime => {
187                        let s_opt = val.as_str();
188                        match s_opt.and_then(parse_datetime_millis) {
189                            Some(ms) => emit_quin(subject_hash, field.predicate_hash, (0b011u64 << 60) | ms, context, 0, &mut on_quin),
190                            None => emit_invalid(subject_hash, field.predicate_hash, s_opt.unwrap_or(""), context, &mut on_quin),
191                        }
192                    }
193                }
194            }
195        }
196    };
197
198    for value in stream {
199        match value {
200            Ok(Value::Object(obj)) => emit_object(&obj),
201            Ok(Value::Array(entries)) => {
202                for entry in entries {
203                    if let Value::Object(obj) = entry { emit_object(&obj); }
204                }
205            }
206            _ => {}
207        }
208    }
209    Ok(())
210}
211
212fn parse_datetime_millis(s: &str) -> Option<u64> {
213    if let Ok(dt) = DateTime::parse_from_rfc3339(s) {
214        return Some(dt.timestamp_millis() as u64);
215    }
216    for fmt in &["%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"] {
217        if let Ok(nd) = NaiveDateTime::parse_from_str(s, fmt) {
218            return Some(Utc.from_utc_datetime(&nd).timestamp_millis() as u64);
219        }
220    }
221    if let Ok(d) = NaiveDate::parse_from_str(s, "%Y-%m-%d") {
222        let nd = d.and_hms_opt(0, 0, 0)?;
223        return Some(Utc.from_utc_datetime(&nd).timestamp_millis() as u64);
224    }
225    None
226}
227
228#[cfg(test)]
229mod tests {
230    use super::*;
231    use crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc;
232
233    #[test]
234    fn parses_each_object_in_a_top_level_array() {
235        let profile = JsonMappingProfile::new(0, vec![
236            JsonFieldMapping { source_key: "name".into(), predicate_hash: 42, datatype: JsonDatatype::StringRef },
237        ]);
238        let mut quins = Vec::new();
239        parse_json_to_quins(br#"[{"name":"Alice"},{"name":"Bob"}]"#.as_slice(), &profile, |q| quins.push(q)).unwrap();
240        assert_eq!(quins.len(), 2);
241        assert!(quins.iter().all(|quin| quin.predicate == 42));
242    }
243
244    #[test]
245    fn deterministic_json_subjects_across_ingest() {
246        let json_data = br#"[{"id":"rec_001","name":"SensorA","val":100},{"id":"rec_002","name":"SensorB","val":200}]"#;
247        let profile = JsonMappingProfile::new(4004, vec![
248            JsonFieldMapping { source_key: "name".into(), predicate_hash: 10, datatype: JsonDatatype::StringRef },
249            JsonFieldMapping { source_key: "val".into(), predicate_hash: 20, datatype: JsonDatatype::Integer },
250        ]);
251        let mut quins1 = Vec::new();
252        parse_json_to_quins(json_data.as_slice(), &profile, |q| quins1.push(q)).unwrap();
253        let mut quins2 = Vec::new();
254        parse_json_to_quins(json_data.as_slice(), &profile, |q| quins2.push(q)).unwrap();
255
256        assert_eq!(quins1, quins2);
257        assert_eq!(quins1.len(), 4);
258        for q in &quins1 {
259            assert_ne!(q.subject, 0);
260            assert_eq!(q.subject >> 63, 0);
261            assert!(q.verify_ecc_parity());
262        }
263        assert_eq!(quins1[0].subject, quins1[1].subject);
264        assert_eq!(quins1[2].subject, quins1[3].subject);
265        assert_ne!(quins1[0].subject, quins1[2].subject);
266    }
267
268    #[test]
269    fn malformed_json_numbers_preserve_raw_lexical() {
270        let json_data = br#"[
271            {"int_val": "N/A", "flt_val": "bad_float"},
272            {"int_val": "-9999_err", "flt_val": "NaN"},
273            {"int_val": 42, "flt_val": 3.14}
274        ]"#;
275        let profile = JsonMappingProfile::new(5005, vec![
276            JsonFieldMapping { source_key: "int_val".into(), predicate_hash: 100, datatype: JsonDatatype::Integer },
277            JsonFieldMapping { source_key: "flt_val".into(), predicate_hash: 200, datatype: JsonDatatype::Float },
278        ]);
279        let mut quins = Vec::new();
280        parse_json_to_quins(json_data.as_slice(), &profile, |q| quins.push(q)).unwrap();
281
282        let na_quin = quins.iter().find(|q| q.predicate == 100).unwrap();
283        assert_eq!(na_quin.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
284        assert_eq!(na_quin.object, hash_token("N/A") & 0x0FFF_FFFF_FFFF_FFFF);
285
286        let bad_flt = quins.iter().find(|q| q.predicate == 200).unwrap();
287        assert_eq!(bad_flt.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
288        assert_eq!(bad_flt.object, hash_token("bad_float") & 0x0FFF_FFFF_FFFF_FFFF);
289
290        let err_quin = quins.iter().filter(|q| q.predicate == 100).nth(1).unwrap();
291        assert_eq!(err_quin.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
292        assert_eq!(err_quin.object, hash_token("-9999_err") & 0x0FFF_FFFF_FFFF_FFFF);
293
294        let valid_int = quins.iter().filter(|q| q.predicate == 100).nth(2).unwrap();
295        assert_eq!(valid_int.metadata & INVALID_VALUE_METADATA_FLAG, 0);
296        assert_eq!(valid_int.object, 42 | (0b001 << 60));
297
298        let valid_flt = quins.iter().filter(|q| q.predicate == 200).nth(2).unwrap();
299        assert_eq!(valid_flt.metadata & INVALID_VALUE_METADATA_FLAG, 0);
300
301        let raw_lex: Vec<_> = quins.iter().filter(|q| q.predicate == PRED_RAW_INVALID_LEXICAL).collect();
302        assert_eq!(raw_lex.len(), 4);
303        for q in &quins { assert!(q.verify_ecc_parity()); }
304    }
305
306    #[test]
307    fn json_provenance_context_and_sensitivity_packing() {
308        let json_data = br#"[{"level": 99}]"#;
309        let profile = JsonMappingProfile::new(8888, vec![
310            JsonFieldMapping { source_key: "level".into(), predicate_hash: 77, datatype: JsonDatatype::Integer },
311        ]);
312        let source_ctx = 0x00FE_DCBA_9876_5432;
313        for &sens in &[NQuin::SENSITIVITY_PUBLIC, NQuin::SENSITIVITY_RESTRICTED, NQuin::SENSITIVITY_CLASSIFIED] {
314            let mut quins = Vec::new();
315            parse_json_to_quins_with_provenance(json_data.as_slice(), &profile, source_ctx, sens, |q| quins.push(q)).unwrap();
316            assert_eq!(quins.len(), 1);
317            let q = quins[0];
318            assert_eq!(q.get_sensitivity_byte(), sens);
319            assert_eq!(q.context >> 56, sens as u64);
320            assert_eq!(q.context & 0x00FF_FFFF_FFFF_FFFF, source_ctx & 0x00FF_FFFF_FFFF_FFFF);
321            assert!(q.verify_ecc_parity());
322        }
323    }
324
325    #[test]
326    fn missing_namespace_or_context_rejected_in_strict_provenance() {
327        let json_data = br#"[{"level": 99}]"#;
328        let mut profile = JsonMappingProfile::new(0, vec![
329            JsonFieldMapping { source_key: "level".into(), predicate_hash: 77, datatype: JsonDatatype::Integer },
330        ]);
331        let res1 = parse_json_to_quins_with_provenance(json_data.as_slice(), &profile, 0x1234, NQuin::SENSITIVITY_PUBLIC, |_| {});
332        assert!(res1.is_err());
333        assert_eq!(res1.unwrap_err(), "Strict provenance requires non-zero source_context_hash and base_class_hash");
334
335        profile.base_class_hash = 1234;
336        let res2 = parse_json_to_quins_with_provenance(json_data.as_slice(), &profile, 0, NQuin::SENSITIVITY_PUBLIC, |_| {});
337        assert!(res2.is_err());
338        assert_eq!(res2.unwrap_err(), "Strict provenance requires non-zero source_context_hash and base_class_hash");
339    }
340
341    #[test]
342    fn hot_quin_packing_is_zero_heap() {
343        assert_zero_alloc("json_hot_quin_packing", || {
344            let mut sink = [NQuin::default(); 4];
345            let mut count = 0;
346            let subject = derive_deterministic_subject(0x1234, 1, 0xABCD);
347            emit_quin(subject, 42, 100 | (0b001 << 60), 0x5678, 0, &mut |q| { sink[count] = q; count += 1; });
348            emit_invalid(subject, 43, "N/A", 0x5678, &mut |q| { sink[count] = q; count += 1; });
349            assert_eq!(count, 3);
350            assert!(sink[0].verify_ecc_parity());
351            assert!(sink[1].verify_ecc_parity());
352            assert!(sink[2].verify_ecc_parity());
353        });
354    }
355}