qualia_cli/ingest/
csv_mapper.rs1use qualia_core_db::sparql_library::parsers::csv_parser::{
2 parse_csv_to_quins_with_provenance, CsvColumnMapping, CsvDatatype, CsvMappingProfile,
3};
4use std::fs::File;
5
6pub fn stream_csv_to_quins_with_provenance(
8 csv_path: &str,
9 output_path: &str,
10 profile: &mut super::mapper::MappingProfile,
11 source_context_hash: u64,
12 sensitivity_class: u8,
13) {
14 let mut writer = super::writer::SuperBlockWriter::new(std::path::Path::new(output_path))
15 .expect("Failed to create SuperBlockWriter");
16 let file = File::open(csv_path).expect("Failed to open CSV");
17
18 let mut core_profile = CsvMappingProfile {
20 base_class_hash: profile.base_class_hash,
21 fields: profile
22 .fields
23 .iter()
24 .map(|f| CsvColumnMapping {
25 source_key: f.source_key.clone(),
26 column_index: f.column_index,
27 predicate_hash: f.predicate_hash,
28 datatype: match f.datatype {
29 super::mapper::TargetDatatype::Integer => CsvDatatype::Integer,
30 super::mapper::TargetDatatype::Float => CsvDatatype::Float,
31 super::mapper::TargetDatatype::DateTime => CsvDatatype::DateTime,
32 super::mapper::TargetDatatype::StringRef => CsvDatatype::StringRef,
33 },
34 })
35 .collect(),
36 };
37
38 parse_csv_to_quins_with_provenance(
40 file,
41 &mut core_profile,
42 source_context_hash,
43 sensitivity_class,
44 |quin| {
45 writer.push(quin).expect("Failed to write to SuperBlock");
46 },
47 )
48 .expect("Failed to parse CSV");
49
50 for (i, field) in profile.fields.iter_mut().enumerate() {
52 if let Some(core_field) = core_profile.fields.get(i) {
53 field.column_index = core_field.column_index;
54 }
55 }
56}
57
58pub fn stream_csv_to_quins(
60 csv_path: &str,
61 output_path: &str,
62 profile: &mut super::mapper::MappingProfile,
63) {
64 let source_context_hash = profile.base_class_hash & 0x00FF_FFFF_FFFF_FFFF;
65 let sensitivity_class = qualia_core_db::NQuin::SENSITIVITY_PUBLIC;
66 stream_csv_to_quins_with_provenance(
67 csv_path,
68 output_path,
69 profile,
70 source_context_hash,
71 sensitivity_class,
72 );
73}