Skip to main content

qualia_core_db/query/
sdmx_cube.rs

1//! SDMX and RDF Data Cube (`qb:`) observation engine with confidentiality-mark semantics (§GAP-09 / G38 / G42).
2//!
3//! Provides structured statistical observation mapping (`qb:DataSet`, `qb:Observation`,
4//! `qb:DataStructureDefinition`, `qb:dimension`, `qb:measure`, `qb:attribute`) and first-class
5//! confidentiality marks (`Public`, `SuppressedForPrivacy`, `Perturbed`, `ModelledEstimate`)
6//! bound to `NQuin` sensitivity contexts and epistemic certainty weights. Zero-heap hot path.
7
8use crate::{q_hash, NQuin};
9
10/// RDF Data Cube (`qb:`) predicate URIs.
11pub const QB_DATASET: &str = "http://purl.org/linked-data/cube#dataSet";
12pub const QB_OBSERVATION: &str = "http://purl.org/linked-data/cube#Observation";
13pub const QB_MEASURE_TYPE: &str = "http://purl.org/linked-data/cube#measureType";
14pub const QB_TIME_PERIOD: &str = "http://purl.org/linked-data/sdmx/2009/dimension#timePeriod";
15pub const QB_REF_AREA: &str = "http://purl.org/linked-data/sdmx/2009/dimension#refArea";
16pub const QB_OBS_VALUE: &str = "http://purl.org/linked-data/sdmx/2009/measure#obsValue";
17pub const QB_CONF_STATUS: &str = "http://purl.org/linked-data/sdmx/2009/attribute#confStatus";
18
19/// Confidentiality-mark semantics for statistical cells (`G42`).
20#[repr(u8)]
21#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
22pub enum ConfidentialityMark {
23    /// Freely publishable cell.
24    #[default]
25    Public = 0,
26    /// Cell suppressed to prevent primary or secondary identification.
27    SuppressedForPrivacy = 1,
28    /// Cell value perturbed (e.g. additive noise or random rounding).
29    Perturbed = 2,
30    /// Modelled or imputed estimate rather than direct census enumeration.
31    ModelledEstimate = 3,
32}
33
34impl ConfidentialityMark {
35    /// Maps confidentiality mark to an NQuin sensitivity class (bits [56..63] of context):
36    /// 0 = Public, 1 = Restricted, 2 = Classified.
37    #[inline]
38    pub fn sensitivity_class(&self) -> u8 {
39        match self {
40            Self::Public | Self::Perturbed | Self::ModelledEstimate => 0,
41            Self::SuppressedForPrivacy => 1, // Restricted
42        }
43    }
44
45    /// Epistemic certainty weight in `[0.0, 1.0]` corresponding to the mark.
46    #[inline]
47    pub fn epistemic_certainty(&self) -> f32 {
48        match self {
49            Self::Public => 1.0,
50            Self::SuppressedForPrivacy => 0.0,
51            Self::Perturbed => 0.85,
52            Self::ModelledEstimate => 0.70,
53        }
54    }
55}
56
57/// A structured statistical observation.
58#[derive(Debug, Clone, Copy, PartialEq)]
59pub struct SdmxObservation {
60    pub dataset_id: u64,
61    pub observation_id: u64,
62    pub time_period: u32,
63    pub geo_region: u64,
64    pub measure_uri: u64,
65    pub measure_value: f64,
66    pub confidentiality: ConfidentialityMark,
67}
68
69/// Compile an SDMX statistical observation into compliant `qb:` NQuins.
70/// Writes up to 6 Quins into `out`:
71/// 1. `rdf:type` -> `qb:Observation`
72/// 2. `qb:dataSet` -> `dataset_id`
73/// 3. `sdmx-dimension:timePeriod` -> `time_period`
74/// 4. `sdmx-dimension:refArea` -> `geo_region`
75/// 5. `sdmx-measure:obsValue` -> packed measure (zeroed if suppressed)
76/// 6. `sdmx-attribute:confStatus` -> confidentiality mark
77///
78/// Context field embeds sensitivity class in MSB bits `[56..63]`.
79/// Metadata embeds epistemic certainty weight in bits `[0..15]`.
80/// Zero-heap caller-buffered.
81pub fn compile_observation_to_quins(obs: &SdmxObservation, out: &mut [NQuin]) -> usize {
82    if out.len() < 6 {
83        return 0;
84    }
85
86    let p_type = q_hash("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
87    let o_obs_type = q_hash(QB_OBSERVATION);
88    let p_dataset = q_hash(QB_DATASET);
89    let p_time = q_hash(QB_TIME_PERIOD);
90    let p_area = q_hash(QB_REF_AREA);
91    let p_val = q_hash(QB_OBS_VALUE);
92    let p_conf = q_hash(QB_CONF_STATUS);
93
94    // Embed sensitivity in top 8 bits of context
95    let context = (obs.dataset_id & 0x00FF_FFFF_FFFF_FFFF)
96        | ((obs.confidentiality.sensitivity_class() as u64) << 56);
97
98    // Embed certainty in bits [0..15] of metadata
99    let certainty_u16 =
100        ((obs.confidentiality.epistemic_certainty() * 65535.0).clamp(0.0, 65535.0)) as u64;
101    let metadata = certainty_u16;
102
103    let make_q = |p: u64, o: u64| {
104        let mut q = NQuin {
105            subject: obs.observation_id,
106            predicate: p,
107            object: o,
108            context,
109            metadata,
110            parity: 0,
111        };
112        q.parity = q.subject ^ q.predicate ^ q.object ^ q.context;
113        q
114    };
115
116    // 1. rdf:type qb:Observation
117    out[0] = make_q(p_type, o_obs_type);
118    // 2. qb:dataSet
119    out[1] = make_q(p_dataset, obs.dataset_id);
120    // 3. timePeriod
121    out[2] = make_q(p_time, obs.time_period as u64);
122    // 4. refArea
123    out[3] = make_q(p_area, obs.geo_region);
124    // 5. obsValue (suppressed cells are zeroed)
125    let val_payload = if obs.confidentiality == ConfidentialityMark::SuppressedForPrivacy {
126        0u64
127    } else {
128        // Pack scaled integer (e.g. value * 1000)
129        (obs.measure_value * 1000.0) as u64
130    };
131    out[4] = make_q(p_val, val_payload);
132    // 6. confStatus
133    out[5] = make_q(p_conf, obs.confidentiality as u64);
134
135    6
136}
137
138/// Parse standard SDMX confidentiality status codes (`confStatus` / `OBS_STATUS`).
139pub fn parse_conf_status(code: &str) -> ConfidentialityMark {
140    match code.trim().to_uppercase().as_str() {
141        "C" | "N" | "CONFIDENTIAL" | "SUPPRESSED" => ConfidentialityMark::SuppressedForPrivacy,
142        "P" | "PERTURBED" | "PROVISIONAL" => ConfidentialityMark::Perturbed,
143        "E" | "ESTIMATED" | "MODELLED" => ConfidentialityMark::ModelledEstimate,
144        _ => ConfidentialityMark::Public,
145    }
146}
147
148/// Parse SDMX-CSV formatted lines into a caller-supplied `SdmxObservation` slice.
149///
150/// Accepts standard SDMX-CSV header shapes:
151/// `DATAFLOW,REF_AREA,TIME_PERIOD,MEASURE,OBS_VALUE,CONF_STATUS`
152/// or flexible order when a header row is present.
153/// Zero heap, hot path.
154pub fn parse_sdmx_csv_into(csv: &str, out: &mut [SdmxObservation]) -> usize {
155    let mut written = 0usize;
156    let mut col_dataflow = 0;
157    let mut col_area = 1;
158    let mut col_time = 2;
159    let mut col_measure = 3;
160    let mut col_val = 4;
161    let mut col_conf = 5;
162
163    for line in csv.lines() {
164        let line = line.trim();
165        if line.is_empty() || line.starts_with('#') {
166            continue;
167        }
168
169        let mut fields = [""; 16];
170        let mut num_fields = 0usize;
171        for part in line.split(',') {
172            if num_fields < 16 {
173                fields[num_fields] = part.trim().trim_matches('"');
174                num_fields += 1;
175            }
176        }
177        if num_fields == 0 {
178            continue;
179        }
180
181        // Header detection
182        let upper = fields[0].to_uppercase();
183        if upper.contains("DATAFLOW") || upper.contains("DATASET") || upper.contains("KEY") {
184            for (i, &part) in fields[..num_fields].iter().enumerate() {
185                let col_name = part.to_uppercase();
186                match col_name.as_str() {
187                    "DATAFLOW" | "DATASET" => col_dataflow = i,
188                    "REF_AREA" | "REGION" | "GEO" => col_area = i,
189                    "TIME_PERIOD" | "TIME" | "YEAR" => col_time = i,
190                    "MEASURE" | "MEASURE_TYPE" | "INDICATOR" => col_measure = i,
191                    "OBS_VALUE" | "VALUE" => col_val = i,
192                    "CONF_STATUS" | "OBS_STATUS" | "STATUS" => col_conf = i,
193                    _ => {}
194                }
195            }
196            continue;
197        }
198
199        if written >= out.len() {
200            break;
201        }
202
203        if num_fields <= col_val {
204            continue;
205        }
206
207        let dataset_str = if col_dataflow < num_fields {
208            fields[col_dataflow]
209        } else {
210            "default_dataset"
211        };
212        let area_str = if col_area < num_fields {
213            fields[col_area]
214        } else {
215            "global"
216        };
217        let time_str = if col_time < num_fields {
218            fields[col_time]
219        } else {
220            "2026"
221        };
222        let measure_str = if col_measure < num_fields {
223            fields[col_measure]
224        } else {
225            "value"
226        };
227        let val_str = if col_val < num_fields {
228            fields[col_val]
229        } else {
230            "0"
231        };
232        let conf_str = if col_conf < num_fields {
233            fields[col_conf]
234        } else {
235            "F"
236        };
237
238        let val: f64 = match val_str.parse() {
239            Ok(v) => v,
240            Err(_) => continue,
241        };
242
243        let time_period: u32 = time_str
244            .split('-')
245            .next()
246            .unwrap_or("2026")
247            .parse()
248            .unwrap_or(2026);
249        let dataset_id = q_hash(dataset_str);
250        let geo_region = q_hash(area_str);
251        let measure_uri = q_hash(measure_str);
252        let confidentiality = parse_conf_status(conf_str);
253
254        // Deterministic observation ID: FNV hash fold
255        let observation_id = dataset_id
256            ^ (time_period as u64)
257            ^ (geo_region.rotate_left(16))
258            ^ (measure_uri.rotate_left(32));
259
260        out[written] = SdmxObservation {
261            dataset_id,
262            observation_id,
263            time_period,
264            geo_region,
265            measure_uri,
266            measure_value: val,
267            confidentiality,
268        };
269        written += 1;
270    }
271
272    written
273}
274
275/// Ingest an SDMX-CSV stream directly into caller-supplied `NQuin` buffer.
276/// Each observation compiles into up to 6 compliant `qb:` NQuins.
277/// Completely zero heap, hot path.
278pub fn ingest_sdmx_csv_to_quins(csv: &str, out_quins: &mut [NQuin]) -> usize {
279    let mut quins_written = 0usize;
280    let mut obs_buf = [SdmxObservation {
281        dataset_id: 0,
282        observation_id: 0,
283        time_period: 0,
284        geo_region: 0,
285        measure_uri: 0,
286        measure_value: 0.0,
287        confidentiality: ConfidentialityMark::Public,
288    }; 1];
289
290    for line in csv.lines() {
291        let line = line.trim();
292        if line.is_empty() || line.starts_with('#') {
293            continue;
294        }
295        if quins_written + 6 > out_quins.len() {
296            break;
297        }
298        let parsed = parse_sdmx_csv_into(line, &mut obs_buf);
299        if parsed > 0 {
300            let n = compile_observation_to_quins(&obs_buf[0], &mut out_quins[quins_written..]);
301            quins_written += n;
302        }
303    }
304
305    quins_written
306}
307
308#[cfg(test)]
309mod tests {
310    use super::*;
311
312    #[test]
313    fn test_compile_public_observation() {
314        let obs = SdmxObservation {
315            dataset_id: q_hash("urn:dataset:abs_biodiversity_expenditure_2026"),
316            observation_id: q_hash("urn:obs:biodiv_exp_2026_vic"),
317            time_period: 2026,
318            geo_region: q_hash("urn:geo:asgs:victoria"),
319            measure_uri: q_hash("qudt:AustralianDollar"),
320            measure_value: 450000.0,
321            confidentiality: ConfidentialityMark::Public,
322        };
323
324        let mut quins = [NQuin::default(); 6];
325        let n = compile_observation_to_quins(&obs, &mut quins);
326        assert_eq!(n, 6);
327        assert_eq!(
328            quins[0].predicate,
329            q_hash("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")
330        );
331        assert_eq!(quins[0].object, q_hash(QB_OBSERVATION));
332        // Sensitivity class is 0 (public)
333        assert_eq!(quins[0].context >> 56, 0);
334        // Epistemic certainty is ~65535 (1.0)
335        assert_eq!(quins[0].metadata & 0xFFFF, 65535);
336        // Value is present
337        assert_eq!(quins[4].object, 450000000);
338    }
339
340    #[test]
341    fn test_compile_suppressed_privacy_observation() {
342        let obs = SdmxObservation {
343            dataset_id: q_hash("urn:dataset:species_location_confidential"),
344            observation_id: q_hash("urn:obs:rare_parrot_count"),
345            time_period: 2026,
346            geo_region: q_hash("urn:geo:sensitive_grid_cell"),
347            measure_uri: q_hash("qudt:IndividualCount"),
348            measure_value: 3.0,
349            confidentiality: ConfidentialityMark::SuppressedForPrivacy,
350        };
351
352        let mut quins = [NQuin::default(); 6];
353        let n = compile_observation_to_quins(&obs, &mut quins);
354        assert_eq!(n, 6);
355        // Sensitivity class is 1 (restricted)
356        assert_eq!(quins[0].context >> 56, 1);
357        // Certainty weight is 0
358        assert_eq!(quins[0].metadata & 0xFFFF, 0);
359        // Value payload is zeroed out to prevent leakage
360        assert_eq!(quins[4].object, 0);
361    }
362
363    #[test]
364    fn test_parse_sdmx_csv_into() {
365        let csv = r#"DATAFLOW,REF_AREA,TIME_PERIOD,MEASURE,OBS_VALUE,CONF_STATUS
366ABS_LAND_ACCOUNT_2026,AUS,2026,HECTARES,145200.5,F
367ABS_LAND_ACCOUNT_2026,VIC,2026,HECTARES,32000.0,C
368ABS_LAND_ACCOUNT_2026,NSW,2026,HECTARES,45000.0,E
369"#;
370        let mut obs = [SdmxObservation {
371            dataset_id: 0,
372            observation_id: 0,
373            time_period: 0,
374            geo_region: 0,
375            measure_uri: 0,
376            measure_value: 0.0,
377            confidentiality: ConfidentialityMark::Public,
378        }; 4];
379
380        let count = parse_sdmx_csv_into(csv, &mut obs);
381        assert_eq!(count, 3);
382
383        assert_eq!(obs[0].dataset_id, q_hash("ABS_LAND_ACCOUNT_2026"));
384        assert_eq!(obs[0].geo_region, q_hash("AUS"));
385        assert_eq!(obs[0].time_period, 2026);
386        assert_eq!(obs[0].measure_value, 145200.5);
387        assert_eq!(obs[0].confidentiality, ConfidentialityMark::Public);
388
389        assert_eq!(obs[1].geo_region, q_hash("VIC"));
390        assert_eq!(
391            obs[1].confidentiality,
392            ConfidentialityMark::SuppressedForPrivacy
393        );
394
395        assert_eq!(obs[2].geo_region, q_hash("NSW"));
396        assert_eq!(
397            obs[2].confidentiality,
398            ConfidentialityMark::ModelledEstimate
399        );
400    }
401
402    #[test]
403    fn test_ingest_sdmx_csv_to_quins() {
404        let csv = r#"DATAFLOW,REF_AREA,TIME_PERIOD,MEASURE,OBS_VALUE,CONF_STATUS
405ABS_LAND_ACCOUNT_2026,AUS,2026,HECTARES,145200.5,F
406ABS_LAND_ACCOUNT_2026,VIC,2026,HECTARES,32000.0,C
407"#;
408        let mut quins = [NQuin::default(); 16];
409        let n = ingest_sdmx_csv_to_quins(csv, &mut quins);
410        // 2 observations * 6 quins = 12 quins
411        assert_eq!(n, 12);
412        // Check first observation quins
413        assert_eq!(
414            quins[0].predicate,
415            q_hash("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")
416        );
417        assert_eq!(quins[0].object, q_hash(QB_OBSERVATION));
418        // Check second observation (suppressed) has restricted sensitivity
419        assert_eq!(quins[6].context >> 56, 1);
420        assert_eq!(quins[10].object, 0); // suppressed value
421    }
422}