1use chrono::{DateTime, NaiveDate, NaiveDateTime, TimeZone, Utc};
9use serde_json::{Deserializer, Value};
10use std::io::Read;
11
12use crate::mini_parser::hash_token;
13use crate::NQuin;
14
15pub const INVALID_VALUE_METADATA_FLAG: u64 = 1 << 31;
16pub const PRED_RAW_INVALID_LEXICAL: u64 = crate::q_hash("qualia:rawInvalidLexical");
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum JsonDatatype {
20 Integer,
21 Float,
22 DateTime,
23 StringRef,
24}
25
26#[derive(Debug, Clone)]
27pub struct JsonFieldMapping {
28 pub source_key: String,
29 pub predicate_hash: u64,
30 pub datatype: JsonDatatype,
31}
32
33#[derive(Debug, Clone)]
34pub struct JsonMappingProfile {
35 pub base_class_hash: u64,
36 pub fields: Vec<JsonFieldMapping>,
37}
38
39impl JsonMappingProfile {
40 pub fn new(base_class_hash: u64, fields: Vec<JsonFieldMapping>) -> Self {
41 Self { base_class_hash, fields }
42 }
43}
44
45#[inline]
47pub fn derive_deterministic_subject(base_class_or_ns: u64, row_or_key: u64, raw_record_hash: u64) -> u64 {
48 let mut hash: u64 = 0xcbf29ce484222325;
49 for &b in &base_class_or_ns.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
50 for &b in &row_or_key.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
51 for &b in &raw_record_hash.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
52 hash & 0x7FFF_FFFF_FFFF_FFFF
53}
54
55#[inline]
56fn hash_json_object(obj: &serde_json::Map<String, Value>) -> u64 {
57 let mut hash: u64 = 0xcbf29ce484222325;
58 for (k, v) in obj {
59 for &b in k.as_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
60 match v {
61 Value::String(s) => { for &b in s.as_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); } }
62 Value::Number(n) => {
63 if let Some(u) = n.as_u64() { for &b in &u.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); } }
64 else if let Some(i) = n.as_i64() { for &b in &i.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); } }
65 else if let Some(f) = n.as_f64() { for &b in &f.to_bits().to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); } }
66 }
67 Value::Bool(b) => { hash ^= if *b { 1 } else { 0 }; hash = hash.wrapping_mul(0x100000001b3); }
68 Value::Null => { hash ^= 0xFF; hash = hash.wrapping_mul(0x100000001b3); }
69 _ => {}
70 }
71 hash ^= 0xAA; hash = hash.wrapping_mul(0x100000001b3);
72 }
73 hash
74}
75
76#[inline]
77fn emit_quin(s: u64, p: u64, o: u64, c: u64, m: u64, on_quin: &mut impl FnMut(NQuin)) {
78 on_quin(NQuin { subject: s, predicate: p, object: o, context: c, metadata: m, parity: NQuin::calculate_parity(s, p, o, c, m) });
79}
80
81#[inline]
82fn emit_invalid(s: u64, p: u64, raw_str: &str, c: u64, on_quin: &mut impl FnMut(NQuin)) {
83 let raw_hash = hash_token(raw_str) & 0x0FFF_FFFF_FFFF_FFFF;
84 emit_quin(s, p, raw_hash, c, INVALID_VALUE_METADATA_FLAG, on_quin);
85 emit_quin(s, PRED_RAW_INVALID_LEXICAL, raw_hash, c, INVALID_VALUE_METADATA_FLAG, on_quin);
86}
87
88#[inline]
89fn val_as_str<'a>(val: &'a Value) -> &'a str {
90 match val {
91 Value::String(s) => s.as_str(),
92 Value::Null => "null",
93 Value::Bool(true) => "true",
94 Value::Bool(false) => "false",
95 _ => "",
96 }
97}
98
99pub fn parse_json_to_quins<R: Read>(
101 reader: R,
102 profile: &JsonMappingProfile,
103 on_quin: impl FnMut(NQuin),
104) -> Result<(), String> {
105 let source_context_hash = profile.base_class_hash & 0x00FF_FFFF_FFFF_FFFF;
106 let sensitivity_class = NQuin::SENSITIVITY_PUBLIC;
107 parse_json_internal(reader, profile, source_context_hash, sensitivity_class, on_quin)
108}
109
110pub fn parse_json_to_quins_with_provenance<R: Read>(
112 reader: R,
113 profile: &JsonMappingProfile,
114 source_context_hash: u64,
115 sensitivity_class: u8,
116 on_quin: impl FnMut(NQuin),
117) -> Result<(), String> {
118 if (source_context_hash & 0x00FF_FFFF_FFFF_FFFF) == 0 || profile.base_class_hash == 0 {
119 return Err("Strict provenance requires non-zero source_context_hash and base_class_hash".into());
120 }
121 parse_json_internal(reader, profile, source_context_hash, sensitivity_class, on_quin)
122}
123
124fn parse_json_internal<R: Read>(
125 reader: R,
126 profile: &JsonMappingProfile,
127 source_context_hash: u64,
128 sensitivity_class: u8,
129 mut on_quin: impl FnMut(NQuin),
130) -> Result<(), String> {
131 let stream = Deserializer::from_reader(reader).into_iter::<Value>();
132 let context = ((sensitivity_class as u64) << 56) | (source_context_hash & 0x00FF_FFFF_FFFF_FFFF);
133 let mut record_index: u64 = 0;
134
135 let mut emit_object = |obj: &serde_json::Map<String, Value>| {
136 record_index += 1;
137 let record_key = if let Some(id_val) = obj.get("id").or_else(|| obj.get("_id")).or_else(|| obj.get("key")) {
138 match id_val {
139 Value::String(s) => hash_token(s),
140 Value::Number(n) => n.as_u64().unwrap_or(record_index),
141 _ => record_index,
142 }
143 } else {
144 record_index
145 };
146 let raw_record_hash = hash_json_object(obj);
147 let subject_hash = derive_deterministic_subject(profile.base_class_hash, record_key, raw_record_hash);
148
149 for field in &profile.fields {
150 if let Some(val) = obj.get(&field.source_key) {
151 match field.datatype {
152 JsonDatatype::Integer => {
153 let parsed = match val {
154 Value::Number(n) => n.as_u64().or_else(|| n.as_i64().map(|i| i as u64)),
155 Value::String(s) => {
156 let t = s.trim();
157 if !t.is_empty() { t.parse::<i64>().map(|i| i as u64).ok().or_else(|| t.parse::<u64>().ok()) } else { None }
158 }
159 _ => None,
160 };
161 match parsed {
162 Some(v) => emit_quin(subject_hash, field.predicate_hash, v | (0b001 << 60), context, 0, &mut on_quin),
163 None => emit_invalid(subject_hash, field.predicate_hash, val_as_str(val), context, &mut on_quin),
164 }
165 }
166 JsonDatatype::Float => {
167 let parsed = match val {
168 Value::Number(n) => n.as_f64().map(|f| f as f32).filter(|f| f.is_finite()),
169 Value::String(s) => {
170 let t = s.trim();
171 if !t.is_empty() && !t.eq_ignore_ascii_case("nan") { t.parse::<f32>().ok().filter(|f| f.is_finite()) } else { None }
172 }
173 _ => None,
174 };
175 match parsed {
176 Some(f) => emit_quin(subject_hash, field.predicate_hash, (0b010u64 << 60) | (f.to_bits() as u64), context, 0, &mut on_quin),
177 None => emit_invalid(subject_hash, field.predicate_hash, val_as_str(val), context, &mut on_quin),
178 }
179 }
180 JsonDatatype::StringRef => {
181 if let Some(s) = val.as_str() {
182 let hash = hash_token(s) & 0x0FFF_FFFF_FFFF_FFFF;
183 emit_quin(subject_hash, field.predicate_hash, hash, context, 0, &mut on_quin);
184 }
185 }
186 JsonDatatype::DateTime => {
187 let s_opt = val.as_str();
188 match s_opt.and_then(parse_datetime_millis) {
189 Some(ms) => emit_quin(subject_hash, field.predicate_hash, (0b011u64 << 60) | ms, context, 0, &mut on_quin),
190 None => emit_invalid(subject_hash, field.predicate_hash, s_opt.unwrap_or(""), context, &mut on_quin),
191 }
192 }
193 }
194 }
195 }
196 };
197
198 for value in stream {
199 match value {
200 Ok(Value::Object(obj)) => emit_object(&obj),
201 Ok(Value::Array(entries)) => {
202 for entry in entries {
203 if let Value::Object(obj) = entry { emit_object(&obj); }
204 }
205 }
206 _ => {}
207 }
208 }
209 Ok(())
210}
211
212fn parse_datetime_millis(s: &str) -> Option<u64> {
213 if let Ok(dt) = DateTime::parse_from_rfc3339(s) {
214 return Some(dt.timestamp_millis() as u64);
215 }
216 for fmt in &["%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"] {
217 if let Ok(nd) = NaiveDateTime::parse_from_str(s, fmt) {
218 return Some(Utc.from_utc_datetime(&nd).timestamp_millis() as u64);
219 }
220 }
221 if let Ok(d) = NaiveDate::parse_from_str(s, "%Y-%m-%d") {
222 let nd = d.and_hms_opt(0, 0, 0)?;
223 return Some(Utc.from_utc_datetime(&nd).timestamp_millis() as u64);
224 }
225 None
226}
227
228#[cfg(test)]
229mod tests {
230 use super::*;
231 use crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc;
232
233 #[test]
234 fn parses_each_object_in_a_top_level_array() {
235 let profile = JsonMappingProfile::new(0, vec![
236 JsonFieldMapping { source_key: "name".into(), predicate_hash: 42, datatype: JsonDatatype::StringRef },
237 ]);
238 let mut quins = Vec::new();
239 parse_json_to_quins(br#"[{"name":"Alice"},{"name":"Bob"}]"#.as_slice(), &profile, |q| quins.push(q)).unwrap();
240 assert_eq!(quins.len(), 2);
241 assert!(quins.iter().all(|quin| quin.predicate == 42));
242 }
243
244 #[test]
245 fn deterministic_json_subjects_across_ingest() {
246 let json_data = br#"[{"id":"rec_001","name":"SensorA","val":100},{"id":"rec_002","name":"SensorB","val":200}]"#;
247 let profile = JsonMappingProfile::new(4004, vec![
248 JsonFieldMapping { source_key: "name".into(), predicate_hash: 10, datatype: JsonDatatype::StringRef },
249 JsonFieldMapping { source_key: "val".into(), predicate_hash: 20, datatype: JsonDatatype::Integer },
250 ]);
251 let mut quins1 = Vec::new();
252 parse_json_to_quins(json_data.as_slice(), &profile, |q| quins1.push(q)).unwrap();
253 let mut quins2 = Vec::new();
254 parse_json_to_quins(json_data.as_slice(), &profile, |q| quins2.push(q)).unwrap();
255
256 assert_eq!(quins1, quins2);
257 assert_eq!(quins1.len(), 4);
258 for q in &quins1 {
259 assert_ne!(q.subject, 0);
260 assert_eq!(q.subject >> 63, 0);
261 assert!(q.verify_ecc_parity());
262 }
263 assert_eq!(quins1[0].subject, quins1[1].subject);
264 assert_eq!(quins1[2].subject, quins1[3].subject);
265 assert_ne!(quins1[0].subject, quins1[2].subject);
266 }
267
268 #[test]
269 fn malformed_json_numbers_preserve_raw_lexical() {
270 let json_data = br#"[
271 {"int_val": "N/A", "flt_val": "bad_float"},
272 {"int_val": "-9999_err", "flt_val": "NaN"},
273 {"int_val": 42, "flt_val": 3.14}
274 ]"#;
275 let profile = JsonMappingProfile::new(5005, vec![
276 JsonFieldMapping { source_key: "int_val".into(), predicate_hash: 100, datatype: JsonDatatype::Integer },
277 JsonFieldMapping { source_key: "flt_val".into(), predicate_hash: 200, datatype: JsonDatatype::Float },
278 ]);
279 let mut quins = Vec::new();
280 parse_json_to_quins(json_data.as_slice(), &profile, |q| quins.push(q)).unwrap();
281
282 let na_quin = quins.iter().find(|q| q.predicate == 100).unwrap();
283 assert_eq!(na_quin.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
284 assert_eq!(na_quin.object, hash_token("N/A") & 0x0FFF_FFFF_FFFF_FFFF);
285
286 let bad_flt = quins.iter().find(|q| q.predicate == 200).unwrap();
287 assert_eq!(bad_flt.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
288 assert_eq!(bad_flt.object, hash_token("bad_float") & 0x0FFF_FFFF_FFFF_FFFF);
289
290 let err_quin = quins.iter().filter(|q| q.predicate == 100).nth(1).unwrap();
291 assert_eq!(err_quin.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
292 assert_eq!(err_quin.object, hash_token("-9999_err") & 0x0FFF_FFFF_FFFF_FFFF);
293
294 let valid_int = quins.iter().filter(|q| q.predicate == 100).nth(2).unwrap();
295 assert_eq!(valid_int.metadata & INVALID_VALUE_METADATA_FLAG, 0);
296 assert_eq!(valid_int.object, 42 | (0b001 << 60));
297
298 let valid_flt = quins.iter().filter(|q| q.predicate == 200).nth(2).unwrap();
299 assert_eq!(valid_flt.metadata & INVALID_VALUE_METADATA_FLAG, 0);
300
301 let raw_lex: Vec<_> = quins.iter().filter(|q| q.predicate == PRED_RAW_INVALID_LEXICAL).collect();
302 assert_eq!(raw_lex.len(), 4);
303 for q in &quins { assert!(q.verify_ecc_parity()); }
304 }
305
306 #[test]
307 fn json_provenance_context_and_sensitivity_packing() {
308 let json_data = br#"[{"level": 99}]"#;
309 let profile = JsonMappingProfile::new(8888, vec![
310 JsonFieldMapping { source_key: "level".into(), predicate_hash: 77, datatype: JsonDatatype::Integer },
311 ]);
312 let source_ctx = 0x00FE_DCBA_9876_5432;
313 for &sens in &[NQuin::SENSITIVITY_PUBLIC, NQuin::SENSITIVITY_RESTRICTED, NQuin::SENSITIVITY_CLASSIFIED] {
314 let mut quins = Vec::new();
315 parse_json_to_quins_with_provenance(json_data.as_slice(), &profile, source_ctx, sens, |q| quins.push(q)).unwrap();
316 assert_eq!(quins.len(), 1);
317 let q = quins[0];
318 assert_eq!(q.get_sensitivity_byte(), sens);
319 assert_eq!(q.context >> 56, sens as u64);
320 assert_eq!(q.context & 0x00FF_FFFF_FFFF_FFFF, source_ctx & 0x00FF_FFFF_FFFF_FFFF);
321 assert!(q.verify_ecc_parity());
322 }
323 }
324
325 #[test]
326 fn missing_namespace_or_context_rejected_in_strict_provenance() {
327 let json_data = br#"[{"level": 99}]"#;
328 let mut profile = JsonMappingProfile::new(0, vec![
329 JsonFieldMapping { source_key: "level".into(), predicate_hash: 77, datatype: JsonDatatype::Integer },
330 ]);
331 let res1 = parse_json_to_quins_with_provenance(json_data.as_slice(), &profile, 0x1234, NQuin::SENSITIVITY_PUBLIC, |_| {});
332 assert!(res1.is_err());
333 assert_eq!(res1.unwrap_err(), "Strict provenance requires non-zero source_context_hash and base_class_hash");
334
335 profile.base_class_hash = 1234;
336 let res2 = parse_json_to_quins_with_provenance(json_data.as_slice(), &profile, 0, NQuin::SENSITIVITY_PUBLIC, |_| {});
337 assert!(res2.is_err());
338 assert_eq!(res2.unwrap_err(), "Strict provenance requires non-zero source_context_hash and base_class_hash");
339 }
340
341 #[test]
342 fn hot_quin_packing_is_zero_heap() {
343 assert_zero_alloc("json_hot_quin_packing", || {
344 let mut sink = [NQuin::default(); 4];
345 let mut count = 0;
346 let subject = derive_deterministic_subject(0x1234, 1, 0xABCD);
347 emit_quin(subject, 42, 100 | (0b001 << 60), 0x5678, 0, &mut |q| { sink[count] = q; count += 1; });
348 emit_invalid(subject, 43, "N/A", 0x5678, &mut |q| { sink[count] = q; count += 1; });
349 assert_eq!(count, 3);
350 assert!(sink[0].verify_ecc_parity());
351 assert!(sink[1].verify_ecc_parity());
352 assert!(sink[2].verify_ecc_parity());
353 });
354 }
355}