1use crate::{q_hash, NQuin};
9
10pub const QB_DATASET: &str = "http://purl.org/linked-data/cube#dataSet";
12pub const QB_OBSERVATION: &str = "http://purl.org/linked-data/cube#Observation";
13pub const QB_MEASURE_TYPE: &str = "http://purl.org/linked-data/cube#measureType";
14pub const QB_TIME_PERIOD: &str = "http://purl.org/linked-data/sdmx/2009/dimension#timePeriod";
15pub const QB_REF_AREA: &str = "http://purl.org/linked-data/sdmx/2009/dimension#refArea";
16pub const QB_OBS_VALUE: &str = "http://purl.org/linked-data/sdmx/2009/measure#obsValue";
17pub const QB_CONF_STATUS: &str = "http://purl.org/linked-data/sdmx/2009/attribute#confStatus";
18
19#[repr(u8)]
21#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
22pub enum ConfidentialityMark {
23 #[default]
25 Public = 0,
26 SuppressedForPrivacy = 1,
28 Perturbed = 2,
30 ModelledEstimate = 3,
32}
33
34impl ConfidentialityMark {
35 #[inline]
38 pub fn sensitivity_class(&self) -> u8 {
39 match self {
40 Self::Public | Self::Perturbed | Self::ModelledEstimate => 0,
41 Self::SuppressedForPrivacy => 1, }
43 }
44
45 #[inline]
47 pub fn epistemic_certainty(&self) -> f32 {
48 match self {
49 Self::Public => 1.0,
50 Self::SuppressedForPrivacy => 0.0,
51 Self::Perturbed => 0.85,
52 Self::ModelledEstimate => 0.70,
53 }
54 }
55}
56
57#[derive(Debug, Clone, Copy, PartialEq)]
59pub struct SdmxObservation {
60 pub dataset_id: u64,
61 pub observation_id: u64,
62 pub time_period: u32,
63 pub geo_region: u64,
64 pub measure_uri: u64,
65 pub measure_value: f64,
66 pub confidentiality: ConfidentialityMark,
67}
68
69pub fn compile_observation_to_quins(obs: &SdmxObservation, out: &mut [NQuin]) -> usize {
82 if out.len() < 6 {
83 return 0;
84 }
85
86 let p_type = q_hash("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
87 let o_obs_type = q_hash(QB_OBSERVATION);
88 let p_dataset = q_hash(QB_DATASET);
89 let p_time = q_hash(QB_TIME_PERIOD);
90 let p_area = q_hash(QB_REF_AREA);
91 let p_val = q_hash(QB_OBS_VALUE);
92 let p_conf = q_hash(QB_CONF_STATUS);
93
94 let context = (obs.dataset_id & 0x00FF_FFFF_FFFF_FFFF)
96 | ((obs.confidentiality.sensitivity_class() as u64) << 56);
97
98 let certainty_u16 =
100 ((obs.confidentiality.epistemic_certainty() * 65535.0).clamp(0.0, 65535.0)) as u64;
101 let metadata = certainty_u16;
102
103 let make_q = |p: u64, o: u64| {
104 let mut q = NQuin {
105 subject: obs.observation_id,
106 predicate: p,
107 object: o,
108 context,
109 metadata,
110 parity: 0,
111 };
112 q.parity = q.subject ^ q.predicate ^ q.object ^ q.context;
113 q
114 };
115
116 out[0] = make_q(p_type, o_obs_type);
118 out[1] = make_q(p_dataset, obs.dataset_id);
120 out[2] = make_q(p_time, obs.time_period as u64);
122 out[3] = make_q(p_area, obs.geo_region);
124 let val_payload = if obs.confidentiality == ConfidentialityMark::SuppressedForPrivacy {
126 0u64
127 } else {
128 (obs.measure_value * 1000.0) as u64
130 };
131 out[4] = make_q(p_val, val_payload);
132 out[5] = make_q(p_conf, obs.confidentiality as u64);
134
135 6
136}
137
138pub fn parse_conf_status(code: &str) -> ConfidentialityMark {
140 match code.trim().to_uppercase().as_str() {
141 "C" | "N" | "CONFIDENTIAL" | "SUPPRESSED" => ConfidentialityMark::SuppressedForPrivacy,
142 "P" | "PERTURBED" | "PROVISIONAL" => ConfidentialityMark::Perturbed,
143 "E" | "ESTIMATED" | "MODELLED" => ConfidentialityMark::ModelledEstimate,
144 _ => ConfidentialityMark::Public,
145 }
146}
147
148pub fn parse_sdmx_csv_into(csv: &str, out: &mut [SdmxObservation]) -> usize {
155 let mut written = 0usize;
156 let mut col_dataflow = 0;
157 let mut col_area = 1;
158 let mut col_time = 2;
159 let mut col_measure = 3;
160 let mut col_val = 4;
161 let mut col_conf = 5;
162
163 for line in csv.lines() {
164 let line = line.trim();
165 if line.is_empty() || line.starts_with('#') {
166 continue;
167 }
168
169 let mut fields = [""; 16];
170 let mut num_fields = 0usize;
171 for part in line.split(',') {
172 if num_fields < 16 {
173 fields[num_fields] = part.trim().trim_matches('"');
174 num_fields += 1;
175 }
176 }
177 if num_fields == 0 {
178 continue;
179 }
180
181 let upper = fields[0].to_uppercase();
183 if upper.contains("DATAFLOW") || upper.contains("DATASET") || upper.contains("KEY") {
184 for (i, &part) in fields[..num_fields].iter().enumerate() {
185 let col_name = part.to_uppercase();
186 match col_name.as_str() {
187 "DATAFLOW" | "DATASET" => col_dataflow = i,
188 "REF_AREA" | "REGION" | "GEO" => col_area = i,
189 "TIME_PERIOD" | "TIME" | "YEAR" => col_time = i,
190 "MEASURE" | "MEASURE_TYPE" | "INDICATOR" => col_measure = i,
191 "OBS_VALUE" | "VALUE" => col_val = i,
192 "CONF_STATUS" | "OBS_STATUS" | "STATUS" => col_conf = i,
193 _ => {}
194 }
195 }
196 continue;
197 }
198
199 if written >= out.len() {
200 break;
201 }
202
203 if num_fields <= col_val {
204 continue;
205 }
206
207 let dataset_str = if col_dataflow < num_fields {
208 fields[col_dataflow]
209 } else {
210 "default_dataset"
211 };
212 let area_str = if col_area < num_fields {
213 fields[col_area]
214 } else {
215 "global"
216 };
217 let time_str = if col_time < num_fields {
218 fields[col_time]
219 } else {
220 "2026"
221 };
222 let measure_str = if col_measure < num_fields {
223 fields[col_measure]
224 } else {
225 "value"
226 };
227 let val_str = if col_val < num_fields {
228 fields[col_val]
229 } else {
230 "0"
231 };
232 let conf_str = if col_conf < num_fields {
233 fields[col_conf]
234 } else {
235 "F"
236 };
237
238 let val: f64 = match val_str.parse() {
239 Ok(v) => v,
240 Err(_) => continue,
241 };
242
243 let time_period: u32 = time_str
244 .split('-')
245 .next()
246 .unwrap_or("2026")
247 .parse()
248 .unwrap_or(2026);
249 let dataset_id = q_hash(dataset_str);
250 let geo_region = q_hash(area_str);
251 let measure_uri = q_hash(measure_str);
252 let confidentiality = parse_conf_status(conf_str);
253
254 let observation_id = dataset_id
256 ^ (time_period as u64)
257 ^ (geo_region.rotate_left(16))
258 ^ (measure_uri.rotate_left(32));
259
260 out[written] = SdmxObservation {
261 dataset_id,
262 observation_id,
263 time_period,
264 geo_region,
265 measure_uri,
266 measure_value: val,
267 confidentiality,
268 };
269 written += 1;
270 }
271
272 written
273}
274
275pub fn ingest_sdmx_csv_to_quins(csv: &str, out_quins: &mut [NQuin]) -> usize {
279 let mut quins_written = 0usize;
280 let mut obs_buf = [SdmxObservation {
281 dataset_id: 0,
282 observation_id: 0,
283 time_period: 0,
284 geo_region: 0,
285 measure_uri: 0,
286 measure_value: 0.0,
287 confidentiality: ConfidentialityMark::Public,
288 }; 1];
289
290 for line in csv.lines() {
291 let line = line.trim();
292 if line.is_empty() || line.starts_with('#') {
293 continue;
294 }
295 if quins_written + 6 > out_quins.len() {
296 break;
297 }
298 let parsed = parse_sdmx_csv_into(line, &mut obs_buf);
299 if parsed > 0 {
300 let n = compile_observation_to_quins(&obs_buf[0], &mut out_quins[quins_written..]);
301 quins_written += n;
302 }
303 }
304
305 quins_written
306}
307
308#[cfg(test)]
309mod tests {
310 use super::*;
311
312 #[test]
313 fn test_compile_public_observation() {
314 let obs = SdmxObservation {
315 dataset_id: q_hash("urn:dataset:abs_biodiversity_expenditure_2026"),
316 observation_id: q_hash("urn:obs:biodiv_exp_2026_vic"),
317 time_period: 2026,
318 geo_region: q_hash("urn:geo:asgs:victoria"),
319 measure_uri: q_hash("qudt:AustralianDollar"),
320 measure_value: 450000.0,
321 confidentiality: ConfidentialityMark::Public,
322 };
323
324 let mut quins = [NQuin::default(); 6];
325 let n = compile_observation_to_quins(&obs, &mut quins);
326 assert_eq!(n, 6);
327 assert_eq!(
328 quins[0].predicate,
329 q_hash("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")
330 );
331 assert_eq!(quins[0].object, q_hash(QB_OBSERVATION));
332 assert_eq!(quins[0].context >> 56, 0);
334 assert_eq!(quins[0].metadata & 0xFFFF, 65535);
336 assert_eq!(quins[4].object, 450000000);
338 }
339
340 #[test]
341 fn test_compile_suppressed_privacy_observation() {
342 let obs = SdmxObservation {
343 dataset_id: q_hash("urn:dataset:species_location_confidential"),
344 observation_id: q_hash("urn:obs:rare_parrot_count"),
345 time_period: 2026,
346 geo_region: q_hash("urn:geo:sensitive_grid_cell"),
347 measure_uri: q_hash("qudt:IndividualCount"),
348 measure_value: 3.0,
349 confidentiality: ConfidentialityMark::SuppressedForPrivacy,
350 };
351
352 let mut quins = [NQuin::default(); 6];
353 let n = compile_observation_to_quins(&obs, &mut quins);
354 assert_eq!(n, 6);
355 assert_eq!(quins[0].context >> 56, 1);
357 assert_eq!(quins[0].metadata & 0xFFFF, 0);
359 assert_eq!(quins[4].object, 0);
361 }
362
363 #[test]
364 fn test_parse_sdmx_csv_into() {
365 let csv = r#"DATAFLOW,REF_AREA,TIME_PERIOD,MEASURE,OBS_VALUE,CONF_STATUS
366ABS_LAND_ACCOUNT_2026,AUS,2026,HECTARES,145200.5,F
367ABS_LAND_ACCOUNT_2026,VIC,2026,HECTARES,32000.0,C
368ABS_LAND_ACCOUNT_2026,NSW,2026,HECTARES,45000.0,E
369"#;
370 let mut obs = [SdmxObservation {
371 dataset_id: 0,
372 observation_id: 0,
373 time_period: 0,
374 geo_region: 0,
375 measure_uri: 0,
376 measure_value: 0.0,
377 confidentiality: ConfidentialityMark::Public,
378 }; 4];
379
380 let count = parse_sdmx_csv_into(csv, &mut obs);
381 assert_eq!(count, 3);
382
383 assert_eq!(obs[0].dataset_id, q_hash("ABS_LAND_ACCOUNT_2026"));
384 assert_eq!(obs[0].geo_region, q_hash("AUS"));
385 assert_eq!(obs[0].time_period, 2026);
386 assert_eq!(obs[0].measure_value, 145200.5);
387 assert_eq!(obs[0].confidentiality, ConfidentialityMark::Public);
388
389 assert_eq!(obs[1].geo_region, q_hash("VIC"));
390 assert_eq!(
391 obs[1].confidentiality,
392 ConfidentialityMark::SuppressedForPrivacy
393 );
394
395 assert_eq!(obs[2].geo_region, q_hash("NSW"));
396 assert_eq!(
397 obs[2].confidentiality,
398 ConfidentialityMark::ModelledEstimate
399 );
400 }
401
402 #[test]
403 fn test_ingest_sdmx_csv_to_quins() {
404 let csv = r#"DATAFLOW,REF_AREA,TIME_PERIOD,MEASURE,OBS_VALUE,CONF_STATUS
405ABS_LAND_ACCOUNT_2026,AUS,2026,HECTARES,145200.5,F
406ABS_LAND_ACCOUNT_2026,VIC,2026,HECTARES,32000.0,C
407"#;
408 let mut quins = [NQuin::default(); 16];
409 let n = ingest_sdmx_csv_to_quins(csv, &mut quins);
410 assert_eq!(n, 12);
412 assert_eq!(
414 quins[0].predicate,
415 q_hash("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")
416 );
417 assert_eq!(quins[0].object, q_hash(QB_OBSERVATION));
418 assert_eq!(quins[6].context >> 56, 1);
420 assert_eq!(quins[10].object, 0); }
422}