1use chrono::{DateTime, NaiveDate, NaiveDateTime, TimeZone, Utc};
9use csv::ReaderBuilder;
10use std::io::Read;
11
12use crate::mini_parser::hash_token;
13use crate::NQuin;
14
15pub const INVALID_VALUE_METADATA_FLAG: u64 = 1 << 31;
16pub const PRED_RAW_INVALID_LEXICAL: u64 = crate::q_hash("qualia:rawInvalidLexical");
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq)]
19pub enum CsvDatatype {
20 Integer,
21 Float,
22 DateTime,
23 StringRef,
24}
25
26#[derive(Debug, Clone)]
27pub struct CsvColumnMapping {
28 pub source_key: String,
29 pub column_index: Option<usize>,
30 pub predicate_hash: u64,
31 pub datatype: CsvDatatype,
32}
33
34#[derive(Debug, Clone)]
35pub struct CsvMappingProfile {
36 pub base_class_hash: u64,
37 pub fields: Vec<CsvColumnMapping>,
38}
39
40impl CsvMappingProfile {
41 pub fn new(base_class_hash: u64, fields: Vec<CsvColumnMapping>) -> Self {
42 Self { base_class_hash, fields }
43 }
44}
45
46#[inline]
48pub fn derive_deterministic_subject(base_class_or_ns: u64, row_or_key: u64, raw_record_hash: u64) -> u64 {
49 let mut hash: u64 = 0xcbf29ce484222325;
50 for &b in &base_class_or_ns.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
51 for &b in &row_or_key.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
52 for &b in &raw_record_hash.to_le_bytes() { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
53 hash & 0x7FFF_FFFF_FFFF_FFFF
54}
55
56#[inline]
57fn hash_record(record: &csv::ByteRecord) -> u64 {
58 let mut hash: u64 = 0xcbf29ce484222325;
59 for field in record.iter() {
60 for &b in field { hash ^= b as u64; hash = hash.wrapping_mul(0x100000001b3); }
61 hash ^= 0xFF; hash = hash.wrapping_mul(0x100000001b3);
62 }
63 hash
64}
65
66#[inline]
67fn emit_quin(s: u64, p: u64, o: u64, c: u64, m: u64, on_quin: &mut impl FnMut(NQuin)) {
68 on_quin(NQuin { subject: s, predicate: p, object: o, context: c, metadata: m, parity: NQuin::calculate_parity(s, p, o, c, m) });
69}
70
71#[inline]
72fn emit_invalid(s: u64, p: u64, raw_str: &str, c: u64, on_quin: &mut impl FnMut(NQuin)) {
73 let raw_hash = hash_token(raw_str) & 0x0FFF_FFFF_FFFF_FFFF;
74 emit_quin(s, p, raw_hash, c, INVALID_VALUE_METADATA_FLAG, on_quin);
75 emit_quin(s, PRED_RAW_INVALID_LEXICAL, raw_hash, c, INVALID_VALUE_METADATA_FLAG, on_quin);
76}
77
78pub fn parse_csv_to_quins<R: Read>(
80 reader: R,
81 profile: &mut CsvMappingProfile,
82 on_quin: impl FnMut(NQuin),
83) -> Result<(), String> {
84 let source_context_hash = profile.base_class_hash & 0x00FF_FFFF_FFFF_FFFF;
85 let sensitivity_class = NQuin::SENSITIVITY_PUBLIC;
86 parse_csv_internal(reader, profile, source_context_hash, sensitivity_class, on_quin)
87}
88
89pub fn parse_csv_to_quins_with_provenance<R: Read>(
91 reader: R,
92 profile: &mut CsvMappingProfile,
93 source_context_hash: u64,
94 sensitivity_class: u8,
95 on_quin: impl FnMut(NQuin),
96) -> Result<(), String> {
97 if (source_context_hash & 0x00FF_FFFF_FFFF_FFFF) == 0 || profile.base_class_hash == 0 {
98 return Err("Strict provenance requires non-zero source_context_hash and base_class_hash".into());
99 }
100 parse_csv_internal(reader, profile, source_context_hash, sensitivity_class, on_quin)
101}
102
103fn parse_csv_internal<R: Read>(
104 reader: R,
105 profile: &mut CsvMappingProfile,
106 source_context_hash: u64,
107 sensitivity_class: u8,
108 mut on_quin: impl FnMut(NQuin),
109) -> Result<(), String> {
110 let mut rdr = ReaderBuilder::new().has_headers(true).from_reader(reader);
111 let headers = rdr.byte_headers().map_err(|e| format!("Failed to read headers: {}", e))?.clone();
112 for field in profile.fields.iter_mut() {
113 field.column_index = headers.iter().position(|h| h == field.source_key.as_bytes());
114 }
115 let mut record = csv::ByteRecord::new();
116 let mut row_index: u64 = 0;
117 let context = ((sensitivity_class as u64) << 56) | (source_context_hash & 0x00FF_FFFF_FFFF_FFFF);
118
119 while rdr.read_byte_record(&mut record).map_err(|e| format!("CSV read error: {}", e))? {
120 row_index += 1;
121 let raw_record_hash = hash_record(&record);
122 let subject_hash = derive_deterministic_subject(profile.base_class_hash, row_index, raw_record_hash);
123
124 for field in &profile.fields {
125 if let Some(idx) = field.column_index {
126 if let Some(raw_bytes) = record.get(idx) {
127 match field.datatype {
128 CsvDatatype::Integer => {
129 let s = std::str::from_utf8(raw_bytes).unwrap_or("").trim();
130 let parsed = if !s.is_empty() {
131 s.parse::<i64>().map(|i| i as u64).ok().or_else(|| s.parse::<u64>().ok())
132 } else {
133 None
134 };
135 match parsed {
136 Some(v) => emit_quin(subject_hash, field.predicate_hash, v | (0b001 << 60), context, 0, &mut on_quin),
137 None => emit_invalid(subject_hash, field.predicate_hash, s, context, &mut on_quin),
138 }
139 }
140 CsvDatatype::Float => {
141 let s = std::str::from_utf8(raw_bytes).unwrap_or("").trim();
142 let parsed = if !s.is_empty() && !s.eq_ignore_ascii_case("nan") {
143 s.parse::<f32>().ok().filter(|f| f.is_finite())
144 } else {
145 None
146 };
147 match parsed {
148 Some(f) => emit_quin(subject_hash, field.predicate_hash, (0b010u64 << 60) | (f.to_bits() as u64), context, 0, &mut on_quin),
149 None => emit_invalid(subject_hash, field.predicate_hash, s, context, &mut on_quin),
150 }
151 }
152 CsvDatatype::StringRef => {
153 let s = std::str::from_utf8(raw_bytes).unwrap_or("");
154 let hash = hash_token(s) & 0x0FFF_FFFF_FFFF_FFFF;
155 emit_quin(subject_hash, field.predicate_hash, hash, context, 0, &mut on_quin);
156 }
157 CsvDatatype::DateTime => {
158 let s = std::str::from_utf8(raw_bytes).unwrap_or("").trim();
159 match parse_datetime_millis(s) {
160 Some(ms) => emit_quin(subject_hash, field.predicate_hash, (0b011u64 << 60) | ms, context, 0, &mut on_quin),
161 None => emit_invalid(subject_hash, field.predicate_hash, s, context, &mut on_quin),
162 }
163 }
164 }
165 }
166 }
167 }
168 }
169 Ok(())
170}
171
172fn parse_datetime_millis(s: &str) -> Option<u64> {
173 if let Ok(dt) = DateTime::parse_from_rfc3339(s) {
174 return Some(dt.timestamp_millis() as u64);
175 }
176 for fmt in &["%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ"] {
177 if let Ok(nd) = NaiveDateTime::parse_from_str(s, fmt) {
178 return Some(Utc.from_utc_datetime(&nd).timestamp_millis() as u64);
179 }
180 }
181 if let Ok(d) = NaiveDate::parse_from_str(s, "%Y-%m-%d") {
182 let nd = d.and_hms_opt(0, 0, 0)?;
183 return Some(Utc.from_utc_datetime(&nd).timestamp_millis() as u64);
184 }
185 None
186}
187
188#[cfg(test)]
189mod tests {
190 use super::*;
191 use crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc;
192
193 #[test]
194 fn deterministic_csv_subjects_across_ingest() {
195 let csv_data = "name,count\nAlice,10\nBob,20\n";
196 let mut profile = CsvMappingProfile::new(1001, vec![
197 CsvColumnMapping { source_key: "name".into(), column_index: None, predicate_hash: 10, datatype: CsvDatatype::StringRef },
198 CsvColumnMapping { source_key: "count".into(), column_index: None, predicate_hash: 20, datatype: CsvDatatype::Integer },
199 ]);
200 let mut quins1 = Vec::new();
201 parse_csv_to_quins(csv_data.as_bytes(), &mut profile, |q| quins1.push(q)).unwrap();
202 let mut profile2 = profile.clone();
203 let mut quins2 = Vec::new();
204 parse_csv_to_quins(csv_data.as_bytes(), &mut profile2, |q| quins2.push(q)).unwrap();
205
206 assert_eq!(quins1, quins2);
207 assert_eq!(quins1.len(), 4);
208 for q in &quins1 {
209 assert_ne!(q.subject, 0);
210 assert_eq!(q.subject >> 63, 0);
211 assert!(q.verify_ecc_parity());
212 }
213 assert_eq!(quins1[0].subject, quins1[1].subject);
214 assert_eq!(quins1[2].subject, quins1[3].subject);
215 assert_ne!(quins1[0].subject, quins1[2].subject);
216 }
217
218 #[test]
219 fn malformed_csv_numbers_preserve_raw_lexical() {
220 let csv_data = "int_col,float_col\nN/A,bad_float\n-9999_err,NaN\n100,42.5\n";
221 let mut profile = CsvMappingProfile::new(2002, vec![
222 CsvColumnMapping { source_key: "int_col".into(), column_index: None, predicate_hash: 100, datatype: CsvDatatype::Integer },
223 CsvColumnMapping { source_key: "float_col".into(), column_index: None, predicate_hash: 200, datatype: CsvDatatype::Float },
224 ]);
225 let mut quins = Vec::new();
226 parse_csv_to_quins(csv_data.as_bytes(), &mut profile, |q| quins.push(q)).unwrap();
227
228 let na_quin = quins.iter().find(|q| q.predicate == 100).unwrap();
229 assert_eq!(na_quin.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
230 assert_eq!(na_quin.object, hash_token("N/A") & 0x0FFF_FFFF_FFFF_FFFF);
231
232 let bad_float = quins.iter().find(|q| q.predicate == 200).unwrap();
233 assert_eq!(bad_float.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
234 assert_eq!(bad_float.object, hash_token("bad_float") & 0x0FFF_FFFF_FFFF_FFFF);
235
236 let err_quin = quins.iter().filter(|q| q.predicate == 100).nth(1).unwrap();
237 assert_eq!(err_quin.metadata & INVALID_VALUE_METADATA_FLAG, INVALID_VALUE_METADATA_FLAG);
238 assert_eq!(err_quin.object, hash_token("-9999_err") & 0x0FFF_FFFF_FFFF_FFFF);
239
240 let valid_int = quins.iter().filter(|q| q.predicate == 100).nth(2).unwrap();
241 assert_eq!(valid_int.metadata & INVALID_VALUE_METADATA_FLAG, 0);
242 assert_eq!(valid_int.object, 100 | (0b001 << 60));
243
244 let valid_float = quins.iter().filter(|q| q.predicate == 200).nth(2).unwrap();
245 assert_eq!(valid_float.metadata & INVALID_VALUE_METADATA_FLAG, 0);
246
247 let raw_lex: Vec<_> = quins.iter().filter(|q| q.predicate == PRED_RAW_INVALID_LEXICAL).collect();
248 assert_eq!(raw_lex.len(), 4);
249 for q in &quins { assert!(q.verify_ecc_parity()); }
250 }
251
252 #[test]
253 fn csv_provenance_context_and_sensitivity_packing() {
254 let csv_data = "score\n50\n";
255 let mut profile = CsvMappingProfile::new(9999, vec![
256 CsvColumnMapping { source_key: "score".into(), column_index: None, predicate_hash: 55, datatype: CsvDatatype::Integer },
257 ]);
258 let source_ctx = 0x00A1_B2C3_D4E5_F600;
259 for &sens in &[NQuin::SENSITIVITY_PUBLIC, NQuin::SENSITIVITY_RESTRICTED, NQuin::SENSITIVITY_CLASSIFIED] {
260 let mut quins = Vec::new();
261 parse_csv_to_quins_with_provenance(csv_data.as_bytes(), &mut profile, source_ctx, sens, |q| quins.push(q)).unwrap();
262 assert_eq!(quins.len(), 1);
263 let q = quins[0];
264 assert_eq!(q.get_sensitivity_byte(), sens);
265 assert_eq!(q.context >> 56, sens as u64);
266 assert_eq!(q.context & 0x00FF_FFFF_FFFF_FFFF, source_ctx & 0x00FF_FFFF_FFFF_FFFF);
267 assert!(q.verify_ecc_parity());
268 }
269 }
270
271 #[test]
272 fn missing_namespace_or_context_rejected_in_strict_provenance() {
273 let csv_data = "score\n50\n";
274 let mut profile = CsvMappingProfile::new(0, vec![
275 CsvColumnMapping { source_key: "score".into(), column_index: None, predicate_hash: 55, datatype: CsvDatatype::Integer },
276 ]);
277 let res1 = parse_csv_to_quins_with_provenance(csv_data.as_bytes(), &mut profile, 0x1234, NQuin::SENSITIVITY_PUBLIC, |_| {});
278 assert!(res1.is_err());
279 assert_eq!(res1.unwrap_err(), "Strict provenance requires non-zero source_context_hash and base_class_hash");
280
281 profile.base_class_hash = 1234;
282 let res2 = parse_csv_to_quins_with_provenance(csv_data.as_bytes(), &mut profile, 0, NQuin::SENSITIVITY_PUBLIC, |_| {});
283 assert!(res2.is_err());
284 assert_eq!(res2.unwrap_err(), "Strict provenance requires non-zero source_context_hash and base_class_hash");
285 }
286
287 #[test]
288 fn hot_quin_packing_is_zero_heap() {
289 assert_zero_alloc("csv_hot_quin_packing", || {
290 let mut sink = [NQuin::default(); 4];
291 let mut count = 0;
292 let subject = derive_deterministic_subject(0x1234, 1, 0xABCD);
293 emit_quin(subject, 42, 100 | (0b001 << 60), 0x5678, 0, &mut |q| { sink[count] = q; count += 1; });
294 emit_invalid(subject, 43, "N/A", 0x5678, &mut |q| { sink[count] = q; count += 1; });
295 assert_eq!(count, 3);
296 assert!(sink[0].verify_ecc_parity());
297 assert!(sink[1].verify_ecc_parity());
298 assert!(sink[2].verify_ecc_parity());
299 });
300 }
301}