Skip to main content

qualia_core_db/sparql_library/parsers/
csv_parser.rs

1//! CSV Parser for QualiaDB
2//!
3//! Zero-allocation CSV parser that streams data directly into NQuin format.
4//! Supports common data types with proper inline type tagging, deterministic
5//! subject identifiers (DOM-AX-7), provenance context/sensitivity packing,
6//! and raw invalid lexical retention.
7
8use chrono::{DateTime, NaiveDate, NaiveDateTime, TimeZone, Utc};
9use csv::ReaderBuilder;
10use std::io::Read;
11
12use crate::mini_parser::hash_token;
13use crate::NQuin;
14
15pub const INVALID_VALUE_METADATA_FLAG: u64 = 1 << 31;
16pub const PRED_RAW_INVALID_LEXICAL: u64 = crate::q_hash("qualia:rawInvalidLexical");
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum CsvDatatype {
20    Integer,
21    Float,
22    DateTime,
23    StringRef,
24}
25
26#[derive(Debug, Clone)]
27pub struct CsvColumnMapping {
28    pub source_key: String,
29    pub column_index: Option<usize>,
30    pub predicate_hash: u64,
31    pub datatype: CsvDatatype,
32}
33
34#[derive(Debug, Clone)]
35pub struct CsvMappingProfile {
36    pub base_class_hash: u64,
37    pub fields: Vec<CsvColumnMapping>,
38}
39
40impl CsvMappingProfile {
41    pub fn new(base_class_hash: u64, fields: Vec<CsvColumnMapping>) -> Self {
42        Self { base_class_hash, fields }
43    }
44}
45
46/// Derive a deterministic 63-bit subject hash from `(base_class_or_ns, row_or_key, raw_record_hash)` (DOM-AX-7).
47#[inline]
48pub fn derive_deterministic_subject(base_class_or_ns: u64, row_or_key: u64, raw_record_hash: u64) -> u64 {
49    let mut hash: u64 = 0xcbf29ce484222325;
50    for &b in &base_class_or_ns.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
51    for &b in &row_or_key.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
52    for &b in &raw_record_hash.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
53    hash & 0x7FFF_FFFF_FFFF_FFFF
54}
55
56#[inline]
57fn hash_record(record: &csv::ByteRecord) -> u64 {
58    let mut hash: u64 = 0xcbf29ce484222325;
59    for field in record.iter() {
60        for &b in field { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
61        hash ^= 0xFF; hash = hash.wrapping_mul(0x100000001b3);
62    }
63    hash
64}
65
66#[inline]
67fn emit_quin(s: u64, p: u64, o: u64, c: u64, m: u64, on_quin: &mut impl FnMut(NQuin)) {
68    on_quin(NQuin { subject: s, predicate: p, object: o, context: c, metadata: m, parity: NQuin::calculate_parity(s, p, o, c, m) });
69}
70
71#[inline]
72fn emit_invalid(s: u64, p: u64, raw_str: &str, c: u64, on_quin: &mut impl FnMut(NQuin)) {
73    let raw_hash = hash_token(raw_str) & 0x0FFF_FFFF_FFFF_FFFF;
74    emit_quin(s, p, raw_hash, c, INVALID_VALUE_METADATA_FLAG, on_quin);
75    emit_quin(s, PRED_RAW_INVALID_LEXICAL, raw_hash, c, INVALID_VALUE_METADATA_FLAG, on_quin);
76}
77
78/// Parse CSV data from a reader and stream Quins (backward-compatible signature).
79pub fn parse_csv_to_quins<R: Read>(
80    reader: R,
81    profile: &mut CsvMappingProfile,
82    on_quin: impl FnMut(NQuin),
83) -> Result<(), String> {
84    let source_context_hash = profile.base_class_hash & 0x00FF_FFFF_FFFF_FFFF;
85    let sensitivity_class = NQuin::SENSITIVITY_PUBLIC;
86    parse_csv_internal(reader, profile, source_context_hash, sensitivity_class, on_quin)
87}
88
89/// Parse CSV data with explicit provenance. Fails closed if source_context_hash or base_class_hash is 0.
90pub fn parse_csv_to_quins_with_provenance<R: Read>(
91    reader: R,
92    profile: &mut CsvMappingProfile,
93    source_context_hash: u64,
94    sensitivity_class: u8,
95    on_quin: impl FnMut(NQuin),
96) -> Result<(), String> {
97    if (source_context_hash & 0x00FF_FFFF_FFFF_FFFF) == 0 || profile.base_class_hash == 0 {
98        return Err("Strict provenance requires non-zero source_context_hash and base_class_hash".into());
99    }
100    parse_csv_internal(reader, profile, source_context_hash, sensitivity_class, on_quin)
101}
102
103fn parse_csv_internal<R: Read>(
104    reader: R,
105    profile: &mut CsvMappingProfile,
106    source_context_hash: u64,
107    sensitivity_class: u8,
108    mut on_quin: impl FnMut(NQuin),
109) -> Result<(), String> {
110    let mut rdr = ReaderBuilder::new().has_headers(true).from_reader(reader);
111    let headers = rdr.byte_headers().map_err(|e| format!("Failed to read headers: {}", e))?.clone();
112    for field in profile.fields.iter_mut() {
113        field.column_index = headers.iter().position(|h| h == field.source_key.as_bytes());
114    }
115    let mut record = csv::ByteRecord::new();
116    let mut row_index: u64 = 0;
117    let context = ((sensitivity_class as u64) << 56) | (source_context_hash & 0x00FF_FFFF_FFFF_FFFF);
118
119    while rdr.read_byte_record(&mut record).map_err(|e| format!("CSV read error: {}", e))? {
120        row_index += 1;
121        let raw_record_hash = hash_record(&record);
122        let subject_hash = derive_deterministic_subject(profile.base_class_hash, row_index, raw_record_hash);
123
124        for field in &profile.fields {
125            if let Some(idx) = field.column_index {
126                if let Some(raw_bytes) = record.get(idx) {
127                    match field.datatype {
128                        CsvDatatype::Integer => {
129                            let s = std::str::from_utf8(raw_bytes).unwrap_or("").trim();
130                            let parsed = if !s.is_empty() {
131                                s.parse::<i64>().map(|i| i as u64).ok().or_else(|| s.parse::<u64>().ok())
132                            } else {
133                                None
134                            };
135                            match parsed {
136                                Some(v) => emit_quin(subject_hash, field.predicate_hash, v | (0b001 << 60), context, 0, &mut on_quin),
137                                None => emit_invalid(subject_hash, field.predicate_hash, s, context, &mut on_quin),
138                            }
139                        }
140                        CsvDatatype::Float => {
141                            let s = std::str::from_utf8(raw_bytes).unwrap_or("").trim();
142                            let parsed = if !s.is_empty() && !s.eq_ignore_ascii_case("nan") {
143                                s.parse::<f32>().ok().filter(|f| f.is_finite())
144                            } else {
145                                None
146                            };
147                            match parsed {
148                                Some(f) => emit_quin(subject_hash, field.predicate_hash, (0b010u64 << 60) | (f.to_bits() as u64), context, 0, &mut on_quin),
149                                None => emit_invalid(subject_hash, field.predicate_hash, s, context, &mut on_quin),
150                            }
151                        }
152                        CsvDatatype::StringRef => {
153                            let s = std::str::from_utf8(raw_bytes).unwrap_or("");
154                            let hash = hash_token(s) & 0x0FFF_FFFF_FFFF_FFFF;
155                            emit_quin(subject_hash, field.predicate_hash, hash, context, 0, &mut on_quin);
156                        }
157                        CsvDatatype::DateTime => {
158                            let s = std::str::from_utf8(raw_bytes).unwrap_or("").trim();
159                            match parse_datetime_millis(s) {
160                                Some(ms) => emit_quin(subject_hash, field.predicate_hash, (0b011u64 << 60) | ms, context, 0, &mut on_quin),
161                                None => emit_invalid(subject_hash, field.predicate_hash, s, context, &mut on_quin),
162                            }
163                        }
164                    }
165                }
166            }
167        }
168    }
169    Ok(())
170}
171
172fn parse_datetime_millis(s: &str) -> Option<u64> {
173    if let Ok(dt) = DateTime::parse_from_rfc3339(s) {
174        return Some(dt.timestamp_millis() as u64);
175    }
176    for fmt in &["%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"] {
177        if let Ok(nd) = NaiveDateTime::parse_from_str(s, fmt) {
178            return Some(Utc.from_utc_datetime(&nd).timestamp_millis() as u64);
179        }
180    }
181    if let Ok(d) = NaiveDate::parse_from_str(s, "%Y-%m-%d") {
182        let nd = d.and_hms_opt(0, 0, 0)?;
183        return Some(Utc.from_utc_datetime(&nd).timestamp_millis() as u64);
184    }
185    None
186}
187
188#[cfg(test)]
189mod tests {
190    use super::*;
191    use crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc;
192
193    #[test]
194    fn deterministic_csv_subjects_across_ingest() {
195        let csv_data = "name,count\nAlice,10\nBob,20\n";
196        let mut profile = CsvMappingProfile::new(1001, vec![
197            CsvColumnMapping { source_key: "name".into(), column_index: None, predicate_hash: 10, datatype: CsvDatatype::StringRef },
198            CsvColumnMapping { source_key: "count".into(), column_index: None, predicate_hash: 20, datatype: CsvDatatype::Integer },
199        ]);
200        let mut quins1 = Vec::new();
201        parse_csv_to_quins(csv_data.as_bytes(), &mut profile, |q| quins1.push(q)).unwrap();
202        let mut profile2 = profile.clone();
203        let mut quins2 = Vec::new();
204        parse_csv_to_quins(csv_data.as_bytes(), &mut profile2, |q| quins2.push(q)).unwrap();
205
206        assert_eq!(quins1, quins2);
207        assert_eq!(quins1.len(), 4);
208        for q in &quins1 {
209            assert_ne!(q.subject, 0);
210            assert_eq!(q.subject >> 63, 0);
211            assert!(q.verify_ecc_parity());
212        }
213        assert_eq!(quins1[0].subject, quins1[1].subject);
214        assert_eq!(quins1[2].subject, quins1[3].subject);
215        assert_ne!(quins1[0].subject, quins1[2].subject);
216    }
217
218    #[test]
219    fn malformed_csv_numbers_preserve_raw_lexical() {
220        let csv_data = "int_col,float_col\nN/A,bad_float\n-9999_err,NaN\n100,42.5\n";
221        let mut profile = CsvMappingProfile::new(2002, vec![
222            CsvColumnMapping { source_key: "int_col".into(), column_index: None, predicate_hash: 100, datatype: CsvDatatype::Integer },
223            CsvColumnMapping { source_key: "float_col".into(), column_index: None, predicate_hash: 200, datatype: CsvDatatype::Float },
224        ]);
225        let mut quins = Vec::new();
226        parse_csv_to_quins(csv_data.as_bytes(), &mut profile, |q| quins.push(q)).unwrap();
227
228        let na_quin = quins.iter().find(|q| q.predicate == 100).unwrap();
229        assert_eq!(na_quin.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
230        assert_eq!(na_quin.object, hash_token("N/A") & 0x0FFF_FFFF_FFFF_FFFF);
231
232        let bad_float = quins.iter().find(|q| q.predicate == 200).unwrap();
233        assert_eq!(bad_float.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
234        assert_eq!(bad_float.object, hash_token("bad_float") & 0x0FFF_FFFF_FFFF_FFFF);
235
236        let err_quin = quins.iter().filter(|q| q.predicate == 100).nth(1).unwrap();
237        assert_eq!(err_quin.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
238        assert_eq!(err_quin.object, hash_token("-9999_err") & 0x0FFF_FFFF_FFFF_FFFF);
239
240        let valid_int = quins.iter().filter(|q| q.predicate == 100).nth(2).unwrap();
241        assert_eq!(valid_int.metadata & INVALID_VALUE_METADATA_FLAG, 0);
242        assert_eq!(valid_int.object, 100 | (0b001 << 60));
243
244        let valid_float = quins.iter().filter(|q| q.predicate == 200).nth(2).unwrap();
245        assert_eq!(valid_float.metadata & INVALID_VALUE_METADATA_FLAG, 0);
246
247        let raw_lex: Vec<_> = quins.iter().filter(|q| q.predicate == PRED_RAW_INVALID_LEXICAL).collect();
248        assert_eq!(raw_lex.len(), 4);
249        for q in &quins { assert!(q.verify_ecc_parity()); }
250    }
251
252    #[test]
253    fn csv_provenance_context_and_sensitivity_packing() {
254        let csv_data = "score\n50\n";
255        let mut profile = CsvMappingProfile::new(9999, vec![
256            CsvColumnMapping { source_key: "score".into(), column_index: None, predicate_hash: 55, datatype: CsvDatatype::Integer },
257        ]);
258        let source_ctx = 0x00A1_B2C3_D4E5_F600;
259        for &sens in &[NQuin::SENSITIVITY_PUBLIC, NQuin::SENSITIVITY_RESTRICTED, NQuin::SENSITIVITY_CLASSIFIED] {
260            let mut quins = Vec::new();
261            parse_csv_to_quins_with_provenance(csv_data.as_bytes(), &mut profile, source_ctx, sens, |q| quins.push(q)).unwrap();
262            assert_eq!(quins.len(), 1);
263            let q = quins[0];
264            assert_eq!(q.get_sensitivity_byte(), sens);
265            assert_eq!(q.context >> 56, sens as u64);
266            assert_eq!(q.context & 0x00FF_FFFF_FFFF_FFFF, source_ctx & 0x00FF_FFFF_FFFF_FFFF);
267            assert!(q.verify_ecc_parity());
268        }
269    }
270
271    #[test]
272    fn missing_namespace_or_context_rejected_in_strict_provenance() {
273        let csv_data = "score\n50\n";
274        let mut profile = CsvMappingProfile::new(0, vec![
275            CsvColumnMapping { source_key: "score".into(), column_index: None, predicate_hash: 55, datatype: CsvDatatype::Integer },
276        ]);
277        let res1 = parse_csv_to_quins_with_provenance(csv_data.as_bytes(), &mut profile, 0x1234, NQuin::SENSITIVITY_PUBLIC, |_| {});
278        assert!(res1.is_err());
279        assert_eq!(res1.unwrap_err(), "Strict provenance requires non-zero source_context_hash and base_class_hash");
280
281        profile.base_class_hash = 1234;
282        let res2 = parse_csv_to_quins_with_provenance(csv_data.as_bytes(), &mut profile, 0, NQuin::SENSITIVITY_PUBLIC, |_| {});
283        assert!(res2.is_err());
284        assert_eq!(res2.unwrap_err(), "Strict provenance requires non-zero source_context_hash and base_class_hash");
285    }
286
287    #[test]
288    fn hot_quin_packing_is_zero_heap() {
289        assert_zero_alloc("csv_hot_quin_packing", || {
290            let mut sink = [NQuin::default(); 4];
291            let mut count = 0;
292            let subject = derive_deterministic_subject(0x1234, 1, 0xABCD);
293            emit_quin(subject, 42, 100 | (0b001 << 60), 0x5678, 0, &mut |q| { sink[count] = q; count += 1; });
294            emit_invalid(subject, 43, "N/A", 0x5678, &mut |q| { sink[count] = q; count += 1; });
295            assert_eq!(count, 3);
296            assert!(sink[0].verify_ecc_parity());
297            assert!(sink[1].verify_ecc_parity());
298            assert!(sink[2].verify_ecc_parity());
299        });
300    }
301}