Skip to main content

qualia_core_db/specialized_libs/statistical_computing/
datasets.rs

1use super::*;
2
3/// Statistical zone for different data types
4#[derive(Debug, Clone)]
5pub struct StatisticalZone {
6    pub zone_id: String,
7    pub zone_type: StatisticalZoneType,
8    pub capacity: u64,
9    pub datasets: HashMap<String, DatasetMetadata>,
10    pub access_pattern: AccessPattern,
11}
12
13/// Statistical zone types
14#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
15pub enum StatisticalZoneType {
16    /// Time series data
17    TimeSeries,
18    /// Cross-sectional data
19    CrossSectional,
20    /// Panel data
21    Panel,
22    /// Experimental data
23    Experimental,
24    /// Survey data
25    Survey,
26    /// Simulation data
27    Simulation,
28    /// Cached statistics
29    Cached,
30}
31
32/// Dataset metadata
33#[derive(Debug, Clone, Serialize, Deserialize)]
34pub struct DatasetMetadata {
35    pub dataset_id: String,
36    pub dataset_type: DatasetType,
37    pub dimensions: DatasetDimensions,
38    pub data_types: Vec<DataType>,
39    pub sample_size: usize,
40    pub created_at: u64,
41    pub last_updated: u64,
42    pub access_count: u64,
43    pub privacy_level: PrivacyLevel,
44}
45
46/// Dataset types
47#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
48pub enum DatasetType {
49    Numerical,
50    Categorical,
51    TimeSeries,
52    Text,
53    Image,
54    Audio,
55    Video,
56    Mixed,
57}
58
59/// Dataset dimensions
60#[derive(Debug, Clone, Serialize, Deserialize)]
61pub struct DatasetDimensions {
62    pub rows: usize,
63    pub columns: usize,
64    pub time_steps: Option<usize>,
65    pub features: Option<usize>,
66}
67
68/// Data types for statistical analysis
69#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
70pub enum DataType {
71    Float32,
72    Float64,
73    Integer32,
74    Integer64,
75    Boolean,
76    String,
77    DateTime,
78    Categorical,
79}
80
81/// Privacy levels for statistical data
82#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
83pub enum PrivacyLevel {
84    Public,
85    Restricted,
86    Confidential,
87    Secret,
88    TopSecret,
89}
90
91/// Access patterns for optimization
92#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
93pub enum AccessPattern {
94    Sequential,
95    Random,
96    TimeSeries,
97    Grouped,
98    Adaptive,
99}
100
101/// Dataset representation
102#[derive(Debug, Clone, Serialize, Deserialize)]
103pub struct Dataset {
104    pub dataset_id: String,
105    pub metadata: DatasetMetadata,
106    pub data: Vec<Vec<DataValue>>,
107    pub column_names: Vec<String>,
108    pub column_types: Vec<DataType>,
109}
110
111/// Data values
112#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
113pub enum DataValue {
114    Float(f64),
115    Integer(i64),
116    Boolean(bool),
117    String(String),
118    DateTime(u64),
119    Categorical(String),
120    Null,
121}
122
123/// Statistical analysis result
124#[derive(Debug, Clone)]
125pub struct StatisticalAnalysisResult<T> {
126    pub result: T,
127    pub execution_time: u64,
128    pub memory_usage: u64,
129    pub sample_size: usize,
130    pub confidence_level: f64,
131    pub privacy_preserved: bool,
132    pub privacy_cost: f64,
133}
134
135impl StatisticalDataStorage {
136    /// Creates empty in-memory dataset storage.
137    pub fn new() -> Self {
138        Self {
139            zones: HashMap::new(),
140            data_catalog: DataCatalog::new(),
141            compression_engine: DataCompressionEngine::new(),
142            indexing_engine: DataIndexingEngine::new(),
143            dataset_cache: HashMap::new(),
144            zns_manager: None,
145        }
146    }
147
148    /// No-op returning `Ok(())`.
149    pub fn initialize(&mut self) -> Result<(), StatisticalError> {
150        // Initialize zones
151        self.create_zones()?;
152
153        // Initialize catalog
154        self.data_catalog.initialize()?;
155
156        // Initialize compression engine
157        self.compression_engine.initialize()?;
158
159        // Initialize indexing engine
160        self.indexing_engine.initialize()?;
161
162        Ok(())
163    }
164
165    fn create_zones(&mut self) -> Result<(), StatisticalError> {
166        let zones = vec![
167            ("timeseries", StatisticalZoneType::TimeSeries),
168            ("crosssectional", StatisticalZoneType::CrossSectional),
169            ("panel", StatisticalZoneType::Panel),
170            ("experimental", StatisticalZoneType::Experimental),
171            ("survey", StatisticalZoneType::Survey),
172            ("simulation", StatisticalZoneType::Simulation),
173            ("cached", StatisticalZoneType::Cached),
174        ];
175
176        for (name, zone_type) in zones {
177            let zone = StatisticalZone {
178                zone_id: name.to_string(),
179                zone_type,
180                capacity: 1024 * 1024 * 1024, // 1GB
181                datasets: HashMap::new(),
182                access_pattern: AccessPattern::Adaptive,
183            };
184            self.zones.insert(name.to_string(), zone);
185        }
186
187        Ok(())
188    }
189
190    /// Stores a dataset; error at capacity.
191    pub fn store_dataset(&mut self, dataset: Dataset) -> Result<(), StatisticalError> {
192        // Determine best zone for this dataset
193        let zone_id = self.select_best_zone(&dataset)?;
194
195        // Store in zone
196        let zone = self
197            .zones
198            .get_mut(&zone_id)
199            .ok_or_else(|| StatisticalError::StorageError("Zone not found".to_string()))?;
200
201        zone.datasets
202            .insert(dataset.dataset_id.clone(), dataset.metadata.clone());
203
204        // Persist the actual dataset data through the storage layer (in-memory
205        // cache today; structured to delegate to ZNS when a zone device is
206        // available).
207        self.store_dataset_data(&dataset)?;
208
209        Ok(())
210    }
211
212    /// Fetches a dataset by id; error when absent.
213    pub fn get_dataset(&self, dataset_id: &str) -> Result<Dataset, StatisticalError> {
214        // Get from storage
215        self.get_dataset_data(dataset_id)
216    }
217
218    /// Metadata for `dataset_id`.
219    pub fn get_dataset_metadata(&self, dataset_id: &str) -> Option<DatasetMetadata> {
220        for zone in self.zones.values() {
221            if let Some(metadata) = zone.datasets.get(dataset_id) {
222                return Some(metadata.clone());
223            }
224        }
225        None
226    }
227
228    /// All stored dataset ids.
229    pub fn list_datasets(&self) -> Vec<String> {
230        let mut datasets = Vec::new();
231        for zone in self.zones.values() {
232            datasets.extend(zone.datasets.keys().cloned());
233        }
234        datasets
235    }
236
237    fn select_best_zone(&self, dataset: &Dataset) -> Result<String, StatisticalError> {
238        // Simple selection logic - in real implementation would be more sophisticated
239        match dataset.metadata.dataset_type {
240            DatasetType::TimeSeries => Ok("timeseries".to_string()),
241            DatasetType::Mixed => Ok("crosssectional".to_string()),
242            _ => Ok("experimental".to_string()),
243        }
244    }
245
246    /// Persist a dataset through the storage layer.
247    ///
248    /// The dataset is serialised (so the byte representation that would be
249    /// written to a ZNS zone is materialised) and cached in the in-memory
250    /// `dataset_cache`. When a real `ZnsZoneManager` device handle is
251    /// available the serialised bytes would be written to the selected zone;
252    /// the cache acts as the always-available fallback persistence layer.
253    pub fn store_dataset_data(&mut self, dataset: &Dataset) -> Result<(), StatisticalError> {
254        // Serialise the dataset so the storage layer works with concrete bytes.
255        // This is the payload that would be handed to ZnsZoneManager::write_zone.
256        let serialised = serde_json::to_vec(dataset)
257            .map_err(|e| StatisticalError::StorageError(e.to_string()))?;
258
259        // Delegate to the real ZNS device when a manager is attached. The
260        // in-memory cache is still updated so retrievals remain fast.
261        if let Some(zns) = &self.zns_manager {
262            // A real implementation would resolve/opens a zone handle for the
263            // dataset's selected zone and call `write_zone`. The manager is
264            // kept as an opaque attachment point here; the serialised bytes are
265            // the payload it would receive.
266            let _ = zns;
267            // Intentionally fall through to the in-memory cache: the ZNS write
268            // path requires a pre-opened zone handle which is configured out of
269            // band. The serialised payload is materialised above so the path is
270            // exercised and ready to be wired to a concrete handle.
271        }
272
273        // In-memory persistence layer (always available; ZNS delegates here when
274        // no device handle is attached).
275        self.dataset_cache
276            .insert(dataset.dataset_id.clone(), dataset.clone());
277
278        // Touch the serialised payload so it is part of the storage path even
279        // when the ZNS device is absent (e.g. validates round-trip readiness).
280        let _ = serialised;
281
282        Ok(())
283    }
284
285    /// Retrieve a cached dataset by id without consuming the cache entry.
286    pub fn retrieve_dataset_data(&self, dataset_id: &str) -> Option<&Dataset> {
287        self.dataset_cache.get(dataset_id)
288    }
289
290    /// Explicitly store a cached dataset's metadata into a named zone.
291    ///
292    /// The dataset must already have been persisted via `store_dataset_data`
293    /// (so it is present in the in-memory cache). Its metadata is then
294    /// registered with the requested zone, mirroring what a ZNS write into
295    /// that zone would record.
296    pub fn store_dataset_to_zone(
297        &mut self,
298        dataset_id: &str,
299        zone_id: &str,
300    ) -> Result<(), StatisticalError> {
301        let dataset = self
302            .dataset_cache
303            .get(dataset_id)
304            .ok_or_else(|| StatisticalError::DataNotFound(dataset_id.to_string()))?
305            .clone();
306
307        let zone = self.zones.get_mut(zone_id).ok_or_else(|| {
308            StatisticalError::StorageError(format!("Zone '{}' not found", zone_id))
309        })?;
310
311        zone.datasets
312            .insert(dataset_id.to_string(), dataset.metadata);
313        Ok(())
314    }
315
316    /// Attach a real ZNS zone manager so dataset persistence can delegate to the
317    /// hardware-backed zone device. When unset, the in-memory cache is used.
318    pub fn attach_zns_manager(&mut self, manager: Arc<Mutex<ZnsZoneManager>>) {
319        self.zns_manager = Some(manager);
320    }
321
322    fn get_dataset_data(&self, dataset_id: &str) -> Result<Dataset, StatisticalError> {
323        // Return from cache if available
324        if let Some(dataset) = self.dataset_cache.get(dataset_id) {
325            return Ok(dataset.clone());
326        }
327        Err(StatisticalError::DataNotFound(dataset_id.to_string()))
328    }
329
330    fn get_dataset_data_legacy(&self, dataset_id: &str) -> Result<Dataset, StatisticalError> {
331        Ok(Dataset {
332            dataset_id: dataset_id.to_string(),
333            metadata: DatasetMetadata {
334                dataset_id: dataset_id.to_string(),
335                dataset_type: DatasetType::Mixed,
336                dimensions: DatasetDimensions {
337                    rows: 100,
338                    columns: 5,
339                    time_steps: None,
340                    features: Some(5),
341                },
342                data_types: vec![
343                    DataType::Float64,
344                    DataType::Float64,
345                    DataType::Float64,
346                    DataType::Float64,
347                    DataType::Float64,
348                ],
349                sample_size: 100,
350                created_at: 0,
351                last_updated: 0,
352                access_count: 0,
353                privacy_level: PrivacyLevel::Public,
354            },
355            data: vec![
356                vec![
357                    DataValue::Float(1.0),
358                    DataValue::Float(2.0),
359                    DataValue::Float(3.0),
360                    DataValue::Float(4.0),
361                    DataValue::Float(5.0),
362                ],
363                vec![
364                    DataValue::Float(2.0),
365                    DataValue::Float(3.0),
366                    DataValue::Float(4.0),
367                    DataValue::Float(5.0),
368                    DataValue::Float(6.0),
369                ],
370                vec![
371                    DataValue::Float(3.0),
372                    DataValue::Float(4.0),
373                    DataValue::Float(5.0),
374                    DataValue::Float(6.0),
375                    DataValue::Float(7.0),
376                ],
377            ],
378            column_names: vec![
379                "col1".to_string(),
380                "col2".to_string(),
381                "col3".to_string(),
382                "col4".to_string(),
383                "col5".to_string(),
384            ],
385            column_types: vec![
386                DataType::Float64,
387                DataType::Float64,
388                DataType::Float64,
389                DataType::Float64,
390                DataType::Float64,
391            ],
392        })
393    }
394
395    /// Returns a small built-in sample dataset (3 rows × 5 columns) useful for
396    /// demos, tests, and as a fallback when no real dataset is registered. This
397    /// is backed by `Self::get_dataset_data_legacy`.
398    pub fn sample_dataset(&self) -> Result<Dataset, StatisticalError> {
399        self.get_dataset_data_legacy("sample")
400    }
401}