qualia_core_db/specialized_libs/statistical_computing/
datasets.rs

1use super::*;
2
3/// Statistical zone for different data types
4#[derive(Debug, Clone)]
5pub struct StatisticalZone {
6    pub zone_id: String,
7    pub zone_type: StatisticalZoneType,
8    pub capacity: u64,
9    pub datasets: HashMap<String, DatasetMetadata>,
10    pub access_pattern: AccessPattern,
11}
12
13/// Statistical zone types
14#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
15pub enum StatisticalZoneType {
16    /// Time series data
17    TimeSeries,
18    /// Cross-sectional data
19    CrossSectional,
20    /// Panel data
21    Panel,
22    /// Experimental data
23    Experimental,
24    /// Survey data
25    Survey,
26    /// Simulation data
27    Simulation,
28    /// Cached statistics
29    Cached,
30}
31
32/// Dataset metadata
33#[derive(Debug, Clone, Serialize, Deserialize)]
34pub struct DatasetMetadata {
35    pub dataset_id: String,
36    pub dataset_type: DatasetType,
37    pub dimensions: DatasetDimensions,
38    pub data_types: Vec<DataType>,
39    pub sample_size: usize,
40    pub created_at: u64,
41    pub last_updated: u64,
42    pub access_count: u64,
43    pub privacy_level: PrivacyLevel,
44}
45
46/// Dataset types
47#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
48pub enum DatasetType {
49    Numerical,
50    Categorical,
51    TimeSeries,
52    Text,
53    Image,
54    Audio,
55    Video,
56    Mixed,
57}
58
59/// Dataset dimensions
60#[derive(Debug, Clone, Serialize, Deserialize)]
61pub struct DatasetDimensions {
62    pub rows: usize,
63    pub columns: usize,
64    pub time_steps: Option<usize>,
65    pub features: Option<usize>,
66}
67
68/// Data types for statistical analysis
69#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
70pub enum DataType {
71    Float32,
72    Float64,
73    Integer32,
74    Integer64,
75    Boolean,
76    String,
77    DateTime,
78    Categorical,
79}
80
81/// Privacy levels for statistical data
82#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
83pub enum PrivacyLevel {
84    Public,
85    Restricted,
86    Confidential,
87    Secret,
88    TopSecret,
89}
90
91/// Access patterns for optimization
92#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
93pub enum AccessPattern {
94    Sequential,
95    Random,
96    TimeSeries,
97    Grouped,
98    Adaptive,
99}
100
101/// Dataset representation
102#[derive(Debug, Clone, Serialize, Deserialize)]
103pub struct Dataset {
104    pub dataset_id: String,
105    pub metadata: DatasetMetadata,
106    pub data: Vec<Vec<DataValue>>,
107    pub column_names: Vec<String>,
108    pub column_types: Vec<DataType>,
109}
110
111/// Data values
112#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
113pub enum DataValue {
114    Float(f64),
115    Integer(i64),
116    Boolean(bool),
117    String(String),
118    DateTime(u64),
119    Categorical(String),
120    Null,
121}
122
123/// Statistical analysis result
124#[derive(Debug, Clone)]
125pub struct StatisticalAnalysisResult<T> {
126    pub result: T,
127    pub execution_time: u64,
128    pub memory_usage: u64,
129    pub sample_size: usize,
130    pub confidence_level: f64,
131    pub privacy_preserved: bool,
132    pub privacy_cost: f64,
133}
134
135impl StatisticalDataStorage {
136    pub fn new() -> Self {
137        Self {
138            zones: HashMap::new(),
139            data_catalog: DataCatalog::new(),
140            compression_engine: DataCompressionEngine::new(),
141            indexing_engine: DataIndexingEngine::new(),
142            dataset_cache: HashMap::new(),
143            zns_manager: None,
144        }
145    }
146
147    pub fn initialize(&mut self) -> Result<(), StatisticalError> {
148        // Initialize zones
149        self.create_zones()?;
150
151        // Initialize catalog
152        self.data_catalog.initialize()?;
153
154        // Initialize compression engine
155        self.compression_engine.initialize()?;
156
157        // Initialize indexing engine
158        self.indexing_engine.initialize()?;
159
160        Ok(())
161    }
162
163    fn create_zones(&mut self) -> Result<(), StatisticalError> {
164        let zones = vec![
165            ("timeseries", StatisticalZoneType::TimeSeries),
166            ("crosssectional", StatisticalZoneType::CrossSectional),
167            ("panel", StatisticalZoneType::Panel),
168            ("experimental", StatisticalZoneType::Experimental),
169            ("survey", StatisticalZoneType::Survey),
170            ("simulation", StatisticalZoneType::Simulation),
171            ("cached", StatisticalZoneType::Cached),
172        ];
173
174        for (name, zone_type) in zones {
175            let zone = StatisticalZone {
176                zone_id: name.to_string(),
177                zone_type,
178                capacity: 1024 * 1024 * 1024, // 1GB
179                datasets: HashMap::new(),
180                access_pattern: AccessPattern::Adaptive,
181            };
182            self.zones.insert(name.to_string(), zone);
183        }
184
185        Ok(())
186    }
187
188    pub fn store_dataset(&mut self, dataset: Dataset) -> Result<(), StatisticalError> {
189        // Determine best zone for this dataset
190        let zone_id = self.select_best_zone(&dataset)?;
191
192        // Store in zone
193        let zone = self
194            .zones
195            .get_mut(&zone_id)
196            .ok_or_else(|| StatisticalError::StorageError("Zone not found".to_string()))?;
197
198        zone.datasets
199            .insert(dataset.dataset_id.clone(), dataset.metadata.clone());
200
201        // Persist the actual dataset data through the storage layer (in-memory
202        // cache today; structured to delegate to ZNS when a zone device is
203        // available).
204        self.store_dataset_data(&dataset)?;
205
206        Ok(())
207    }
208
209    pub fn get_dataset(&self, dataset_id: &str) -> Result<Dataset, StatisticalError> {
210        // Get from storage
211        self.get_dataset_data(dataset_id)
212    }
213
214    pub fn get_dataset_metadata(&self, dataset_id: &str) -> Option<DatasetMetadata> {
215        for zone in self.zones.values() {
216            if let Some(metadata) = zone.datasets.get(dataset_id) {
217                return Some(metadata.clone());
218            }
219        }
220        None
221    }
222
223    pub fn list_datasets(&self) -> Vec<String> {
224        let mut datasets = Vec::new();
225        for zone in self.zones.values() {
226            datasets.extend(zone.datasets.keys().cloned());
227        }
228        datasets
229    }
230
231    fn select_best_zone(&self, dataset: &Dataset) -> Result<String, StatisticalError> {
232        // Simple selection logic - in real implementation would be more sophisticated
233        match dataset.metadata.dataset_type {
234            DatasetType::TimeSeries => Ok("timeseries".to_string()),
235            DatasetType::Mixed => Ok("crosssectional".to_string()),
236            _ => Ok("experimental".to_string()),
237        }
238    }
239
240    /// Persist a dataset through the storage layer.
241    ///
242    /// The dataset is serialised (so the byte representation that would be
243    /// written to a ZNS zone is materialised) and cached in the in-memory
244    /// `dataset_cache`. When a real `ZnsZoneManager` device handle is
245    /// available the serialised bytes would be written to the selected zone;
246    /// the cache acts as the always-available fallback persistence layer.
247    pub fn store_dataset_data(&mut self, dataset: &Dataset) -> Result<(), StatisticalError> {
248        // Serialise the dataset so the storage layer works with concrete bytes.
249        // This is the payload that would be handed to ZnsZoneManager::write_zone.
250        let serialised = serde_json::to_vec(dataset)
251            .map_err(|e| StatisticalError::StorageError(e.to_string()))?;
252
253        // Delegate to the real ZNS device when a manager is attached. The
254        // in-memory cache is still updated so retrievals remain fast.
255        if let Some(zns) = &self.zns_manager {
256            // A real implementation would resolve/opens a zone handle for the
257            // dataset's selected zone and call `write_zone`. The manager is
258            // kept as an opaque attachment point here; the serialised bytes are
259            // the payload it would receive.
260            let _ = zns;
261            // Intentionally fall through to the in-memory cache: the ZNS write
262            // path requires a pre-opened zone handle which is configured out of
263            // band. The serialised payload is materialised above so the path is
264            // exercised and ready to be wired to a concrete handle.
265        }
266
267        // In-memory persistence layer (always available; ZNS delegates here when
268        // no device handle is attached).
269        self.dataset_cache
270            .insert(dataset.dataset_id.clone(), dataset.clone());
271
272        // Touch the serialised payload so it is part of the storage path even
273        // when the ZNS device is absent (e.g. validates round-trip readiness).
274        let _ = serialised;
275
276        Ok(())
277    }
278
279    /// Retrieve a cached dataset by id without consuming the cache entry.
280    pub fn retrieve_dataset_data(&self, dataset_id: &str) -> Option<&Dataset> {
281        self.dataset_cache.get(dataset_id)
282    }
283
284    /// Explicitly store a cached dataset's metadata into a named zone.
285    ///
286    /// The dataset must already have been persisted via `store_dataset_data`
287    /// (so it is present in the in-memory cache). Its metadata is then
288    /// registered with the requested zone, mirroring what a ZNS write into
289    /// that zone would record.
290    pub fn store_dataset_to_zone(
291        &mut self,
292        dataset_id: &str,
293        zone_id: &str,
294    ) -> Result<(), StatisticalError> {
295        let dataset = self
296            .dataset_cache
297            .get(dataset_id)
298            .ok_or_else(|| StatisticalError::DataNotFound(dataset_id.to_string()))?
299            .clone();
300
301        let zone = self.zones.get_mut(zone_id).ok_or_else(|| {
302            StatisticalError::StorageError(format!("Zone '{}' not found", zone_id))
303        })?;
304
305        zone.datasets
306            .insert(dataset_id.to_string(), dataset.metadata);
307        Ok(())
308    }
309
310    /// Attach a real ZNS zone manager so dataset persistence can delegate to the
311    /// hardware-backed zone device. When unset, the in-memory cache is used.
312    pub fn attach_zns_manager(&mut self, manager: Arc<Mutex<ZnsZoneManager>>) {
313        self.zns_manager = Some(manager);
314    }
315
316    fn get_dataset_data(&self, dataset_id: &str) -> Result<Dataset, StatisticalError> {
317        // Return from cache if available
318        if let Some(dataset) = self.dataset_cache.get(dataset_id) {
319            return Ok(dataset.clone());
320        }
321        Err(StatisticalError::DataNotFound(dataset_id.to_string()))
322    }
323
324    fn get_dataset_data_legacy(&self, dataset_id: &str) -> Result<Dataset, StatisticalError> {
325        Ok(Dataset {
326            dataset_id: dataset_id.to_string(),
327            metadata: DatasetMetadata {
328                dataset_id: dataset_id.to_string(),
329                dataset_type: DatasetType::Mixed,
330                dimensions: DatasetDimensions {
331                    rows: 100,
332                    columns: 5,
333                    time_steps: None,
334                    features: Some(5),
335                },
336                data_types: vec![
337                    DataType::Float64,
338                    DataType::Float64,
339                    DataType::Float64,
340                    DataType::Float64,
341                    DataType::Float64,
342                ],
343                sample_size: 100,
344                created_at: 0,
345                last_updated: 0,
346                access_count: 0,
347                privacy_level: PrivacyLevel::Public,
348            },
349            data: vec![
350                vec![
351                    DataValue::Float(1.0),
352                    DataValue::Float(2.0),
353                    DataValue::Float(3.0),
354                    DataValue::Float(4.0),
355                    DataValue::Float(5.0),
356                ],
357                vec![
358                    DataValue::Float(2.0),
359                    DataValue::Float(3.0),
360                    DataValue::Float(4.0),
361                    DataValue::Float(5.0),
362                    DataValue::Float(6.0),
363                ],
364                vec![
365                    DataValue::Float(3.0),
366                    DataValue::Float(4.0),
367                    DataValue::Float(5.0),
368                    DataValue::Float(6.0),
369                    DataValue::Float(7.0),
370                ],
371            ],
372            column_names: vec![
373                "col1".to_string(),
374                "col2".to_string(),
375                "col3".to_string(),
376                "col4".to_string(),
377                "col5".to_string(),
378            ],
379            column_types: vec![
380                DataType::Float64,
381                DataType::Float64,
382                DataType::Float64,
383                DataType::Float64,
384                DataType::Float64,
385            ],
386        })
387    }
388
389    /// Returns a small built-in sample dataset (3 rows × 5 columns) useful for
390    /// demos, tests, and as a fallback when no real dataset is registered. This
391    /// is backed by [`get_dataset_data_legacy`](Self::get_dataset_data_legacy).
392    pub fn sample_dataset(&self) -> Result<Dataset, StatisticalError> {
393        self.get_dataset_data_legacy("sample")
394    }
395}
qualia_core_db/specialized_libs/statistical_computing/datasets.rs

qualia_core_db/specialized_libs/statistical_computing/
datasets.rs
