Skip to main content

qualia_core_db/specialized_libs/statistical_computing/
library.rs

1use super::*;
2
3impl StatisticalComputingLibrary {
4    /// Create new statistical computing library
5    pub fn new() -> Self {
6        Self {
7            data_storage: StatisticalDataStorage::new(),
8            computation_engine: StatisticalComputationEngine::new(),
9            privacy_engine: StatisticalPrivacyEngine::new(),
10            analysis_engine: StatisticalAnalysisEngine::new(),
11            performance_monitor: StatisticalPerformanceMonitor::new(),
12        }
13    }
14
15    /// Initialize the library
16    pub fn initialize(&mut self) -> Result<(), StatisticalError> {
17        // Initialize storage
18        self.data_storage.initialize()?;
19
20        // Initialize computation engine
21        self.computation_engine.initialize()?;
22
23        // Initialize privacy engine
24        self.privacy_engine.initialize()?;
25
26        // Initialize analysis engine
27        self.analysis_engine.initialize()?;
28
29        Ok(())
30    }
31
32    /// Sample a standard normal Gaussian random variable using high-entropy PRNG.
33    pub fn sample_standard_normal(&mut self) -> f64 {
34        let t = std::time::SystemTime::now()
35            .duration_since(std::time::UNIX_EPOCH)
36            .unwrap_or_default()
37            .as_nanos() as u64;
38        let mut seed = t.wrapping_mul(0x9E37_79B9_7F4A_7C15);
39        seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
40        let u1 = (((seed >> 11) as f64) / ((1u64 << 53) as f64)).max(1e-15);
41        seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
42        let u2 = ((seed >> 11) as f64) / ((1u64 << 53) as f64);
43        (-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos()
44    }
45
46    /// Create a new dataset
47    pub fn create_dataset(
48        &mut self,
49        dataset_id: String,
50        data: Vec<Vec<DataValue>>,
51        column_names: Vec<String>,
52        column_types: Vec<DataType>,
53        privacy_level: PrivacyLevel,
54    ) -> Result<Dataset, StatisticalError> {
55        // Validate input
56        if data.is_empty() {
57            return Err(StatisticalError::InvalidData(
58                "Dataset cannot be empty".to_string(),
59            ));
60        }
61        if column_names.len() != column_types.len() {
62            return Err(StatisticalError::InvalidData(
63                "Column names and types must match".to_string(),
64            ));
65        }
66        if data.iter().any(|row| row.len() != column_names.len()) {
67            return Err(StatisticalError::InvalidData(
68                "All rows must have same number of columns".to_string(),
69            ));
70        }
71
72        // Create metadata
73        let metadata = DatasetMetadata {
74            dataset_id: dataset_id.clone(),
75            dataset_type: DatasetType::Mixed,
76            dimensions: DatasetDimensions {
77                rows: data.len(),
78                columns: column_names.len(),
79                time_steps: None,
80                features: Some(column_names.len()),
81            },
82            data_types: column_types.clone(),
83            sample_size: data.len(),
84            created_at: std::time::SystemTime::now()
85                .duration_since(std::time::UNIX_EPOCH)
86                .unwrap()
87                .as_secs(),
88            last_updated: 0,
89            access_count: 0,
90            privacy_level,
91        };
92
93        // Create dataset
94        let dataset = Dataset {
95            dataset_id: dataset_id.clone(),
96            metadata,
97            data,
98            column_names,
99            column_types,
100        };
101
102        // Store dataset
103        self.data_storage.store_dataset(dataset.clone())?;
104
105        Ok(dataset)
106    }
107
108    /// Compute mean of a column
109    pub fn mean(
110        &mut self,
111        dataset_id: &str,
112        column: &str,
113        privacy_preserved: bool,
114    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
115        let start_time = std::time::Instant::now();
116
117        // Get dataset
118        let dataset = self.data_storage.get_dataset(dataset_id)?;
119
120        // Find column index
121        let column_index = dataset
122            .column_names
123            .iter()
124            .position(|name| name == column)
125            .ok_or_else(|| StatisticalError::InvalidColumn(column.to_string()))?;
126
127        // Validate column type
128        if !matches!(
129            dataset.column_types[column_index],
130            DataType::Float32 | DataType::Float64
131        ) {
132            return Err(StatisticalError::InvalidOperation(
133                "Mean can only be computed on numeric columns".to_string(),
134            ));
135        }
136
137        // Extract column data
138        let mut values = Vec::new();
139        for row in &dataset.data {
140            match &row[column_index] {
141                DataValue::Float(value) => values.push(*value),
142                DataValue::Integer(value) => values.push(*value as f64),
143                DataValue::Null => continue,
144                _ => {
145                    return Err(StatisticalError::InvalidOperation(
146                        "Non-numeric data in column".to_string(),
147                    ))
148                }
149            }
150        }
151
152        if values.is_empty() {
153            return Err(StatisticalError::InvalidData(
154                "No valid data in column".to_string(),
155            ));
156        }
157
158        // Compute mean via the engine's canonical statistics solver (Modality-First:
159        // no inline re-implementation). `values` is the caller-owned slice.
160        let mean = crate::solvers::statistics::mean(&values)
161            .ok_or_else(|| StatisticalError::InvalidData("No valid data in column".to_string()))?;
162
163        let execution_time = start_time.elapsed().as_millis() as u64;
164
165        // Apply privacy if requested. Sensitivity is calibrated via the
166        // differential-privacy sensitivity analyzer (mean sensitivity = 1/n)
167        // instead of the previous hardcoded 1.0, so noise scales with the
168        // actual query sensitivity.
169        let (final_mean, privacy_cost) = if privacy_preserved {
170            let sensitivity = {
171                let analyzer = &mut self
172                    .privacy_engine
173                    .differential_privacy
174                    .sensitivity_analyzer;
175                analyzer.get_sensitivity("mean", &values).unwrap_or(1.0)
176            };
177            let (noisy_mean, cost) = self.privacy_engine.add_laplace_noise(mean, sensitivity)?;
178            (noisy_mean, cost)
179        } else {
180            (mean, 0.0)
181        };
182
183        // Update performance metrics
184        self.performance_monitor
185            .record_operation("mean", execution_time, 0, privacy_cost);
186
187        Ok(StatisticalAnalysisResult {
188            result: final_mean,
189            execution_time,
190            memory_usage: 0,
191            sample_size: values.len(),
192            confidence_level: 0.95,
193            privacy_preserved,
194            privacy_cost,
195        })
196    }
197
198    /// Compute median of a column
199    pub fn median(
200        &mut self,
201        dataset_id: &str,
202        column: &str,
203        privacy_preserved: bool,
204    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
205        let start_time = std::time::Instant::now();
206
207        // Get dataset
208        let dataset = self.data_storage.get_dataset(dataset_id)?;
209
210        // Find column index
211        let column_index = dataset
212            .column_names
213            .iter()
214            .position(|name| name == column)
215            .ok_or_else(|| StatisticalError::InvalidColumn(column.to_string()))?;
216
217        // Validate column type
218        if !matches!(
219            dataset.column_types[column_index],
220            DataType::Float32 | DataType::Float64
221        ) {
222            return Err(StatisticalError::InvalidOperation(
223                "Median can only be computed on numeric columns".to_string(),
224            ));
225        }
226
227        // Extract column data
228        let mut values = Vec::new();
229        for row in &dataset.data {
230            match &row[column_index] {
231                DataValue::Float(value) => values.push(*value),
232                DataValue::Integer(value) => values.push(*value as f64),
233                DataValue::Null => continue,
234                _ => {
235                    return Err(StatisticalError::InvalidOperation(
236                        "Non-numeric data in column".to_string(),
237                    ))
238                }
239            }
240        }
241
242        if values.is_empty() {
243            return Err(StatisticalError::InvalidData(
244                "No valid data in column".to_string(),
245            ));
246        }
247
248        // Compute median via the engine's canonical statistics solver (sorts the
249        // caller-owned buffer in place; no inline re-implementation).
250        let median = crate::solvers::statistics::median_in_place(&mut values)
251            .ok_or_else(|| StatisticalError::InvalidData("No valid data in column".to_string()))?;
252
253        let execution_time = start_time.elapsed().as_millis() as u64;
254
255        // Apply privacy if requested
256        let (final_median, privacy_cost) = if privacy_preserved {
257            let (noisy_median, cost) = self.privacy_engine.add_laplace_noise(median, 1.0)?;
258            (noisy_median, cost)
259        } else {
260            (median, 0.0)
261        };
262
263        // Update performance metrics
264        self.performance_monitor
265            .record_operation("median", execution_time, 0, privacy_cost);
266
267        Ok(StatisticalAnalysisResult {
268            result: final_median,
269            execution_time,
270            memory_usage: 0,
271            sample_size: values.len(),
272            confidence_level: 0.95,
273            privacy_preserved,
274            privacy_cost,
275        })
276    }
277
278    /// Compute variance of a column
279    pub fn variance(
280        &mut self,
281        dataset_id: &str,
282        column: &str,
283        sample: bool,
284        privacy_preserved: bool,
285    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
286        let start_time = std::time::Instant::now();
287
288        // Get dataset
289        let dataset = self.data_storage.get_dataset(dataset_id)?;
290
291        // Find column index
292        let column_index = dataset
293            .column_names
294            .iter()
295            .position(|name| name == column)
296            .ok_or_else(|| StatisticalError::InvalidColumn(column.to_string()))?;
297
298        // Validate column type
299        if !matches!(
300            dataset.column_types[column_index],
301            DataType::Float32 | DataType::Float64
302        ) {
303            return Err(StatisticalError::InvalidOperation(
304                "Variance can only be computed on numeric columns".to_string(),
305            ));
306        }
307
308        // Extract column data
309        let mut values = Vec::new();
310        for row in &dataset.data {
311            match &row[column_index] {
312                DataValue::Float(value) => values.push(*value),
313                DataValue::Integer(value) => values.push(*value as f64),
314                DataValue::Null => continue,
315                _ => {
316                    return Err(StatisticalError::InvalidOperation(
317                        "Non-numeric data in column".to_string(),
318                    ))
319                }
320            }
321        }
322
323        if values.is_empty() {
324            return Err(StatisticalError::InvalidData(
325                "No valid data in column".to_string(),
326            ));
327        }
328
329        // Compute variance via the engine's canonical statistics solver
330        // (Modality-First: no inline re-implementation).
331        let variance = crate::solvers::statistics::variance(&values, sample)
332            .ok_or_else(|| StatisticalError::InvalidData("No valid data in column".to_string()))?;
333
334        let execution_time = start_time.elapsed().as_millis() as u64;
335
336        // Apply privacy if requested
337        let (final_variance, privacy_cost) = if privacy_preserved {
338            let (noisy_variance, cost) = self.privacy_engine.add_laplace_noise(variance, 1.0)?;
339            (noisy_variance, cost)
340        } else {
341            (variance, 0.0)
342        };
343
344        // Update performance metrics
345        self.performance_monitor
346            .record_operation("variance", execution_time, 0, privacy_cost);
347
348        Ok(StatisticalAnalysisResult {
349            result: final_variance,
350            execution_time,
351            memory_usage: 0,
352            sample_size: values.len(),
353            confidence_level: 0.95,
354            privacy_preserved,
355            privacy_cost,
356        })
357    }
358
359    /// Compute correlation between two columns
360    pub fn correlation(
361        &mut self,
362        dataset_id: &str,
363        column1: &str,
364        column2: &str,
365        method: CorrelationMethod,
366        privacy_preserved: bool,
367    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
368        let start_time = std::time::Instant::now();
369
370        // Get dataset
371        let dataset = self.data_storage.get_dataset(dataset_id)?;
372
373        // Find column indices
374        let column1_index = dataset
375            .column_names
376            .iter()
377            .position(|name| name == column1)
378            .ok_or_else(|| StatisticalError::InvalidColumn(column1.to_string()))?;
379
380        let column2_index = dataset
381            .column_names
382            .iter()
383            .position(|name| name == column2)
384            .ok_or_else(|| StatisticalError::InvalidColumn(column2.to_string()))?;
385
386        // Validate column types
387        if !matches!(
388            dataset.column_types[column1_index],
389            DataType::Float32 | DataType::Float64
390        ) {
391            return Err(StatisticalError::InvalidOperation(
392                "Correlation can only be computed on numeric columns".to_string(),
393            ));
394        }
395        if !matches!(
396            dataset.column_types[column2_index],
397            DataType::Float32 | DataType::Float64
398        ) {
399            return Err(StatisticalError::InvalidOperation(
400                "Correlation can only be computed on numeric columns".to_string(),
401            ));
402        }
403
404        // Extract column data
405        let mut x_values = Vec::new();
406        let mut y_values = Vec::new();
407
408        for row in &dataset.data {
409            let x_val = match &row[column1_index] {
410                DataValue::Float(value) => *value,
411                DataValue::Integer(value) => *value as f64,
412                DataValue::Null => continue,
413                _ => {
414                    return Err(StatisticalError::InvalidOperation(
415                        "Non-numeric data in column".to_string(),
416                    ))
417                }
418            };
419
420            let y_val = match &row[column2_index] {
421                DataValue::Float(value) => *value,
422                DataValue::Integer(value) => *value as f64,
423                DataValue::Null => continue,
424                _ => {
425                    return Err(StatisticalError::InvalidOperation(
426                        "Non-numeric data in column".to_string(),
427                    ))
428                }
429            };
430
431            x_values.push(x_val);
432            y_values.push(y_val);
433        }
434
435        if x_values.len() < 2 {
436            return Err(StatisticalError::InvalidData(
437                "Insufficient data for correlation".to_string(),
438            ));
439        }
440
441        // Compute correlation based on method
442        let correlation = match method {
443            CorrelationMethod::Pearson => self.pearson_correlation(&x_values, &y_values)?,
444            CorrelationMethod::Spearman => self.spearman_correlation(&x_values, &y_values)?,
445            CorrelationMethod::Kendall => self.kendall_correlation(&x_values, &y_values)?,
446            _ => {
447                return Err(StatisticalError::InvalidOperation(
448                    "Correlation method not supported".to_string(),
449                ))
450            }
451        };
452
453        let execution_time = start_time.elapsed().as_millis() as u64;
454
455        // Apply privacy if requested
456        let (final_correlation, privacy_cost) = if privacy_preserved {
457            let (noisy_correlation, cost) =
458                self.privacy_engine.add_laplace_noise(correlation, 0.1)?;
459            (noisy_correlation.clamp(-1.0, 1.0), cost)
460        } else {
461            (correlation, 0.0)
462        };
463
464        // Update performance metrics
465        self.performance_monitor
466            .record_operation("correlation", execution_time, 0, privacy_cost);
467
468        Ok(StatisticalAnalysisResult {
469            result: final_correlation,
470            execution_time,
471            memory_usage: 0,
472            sample_size: x_values.len(),
473            confidence_level: 0.95,
474            privacy_preserved,
475            privacy_cost,
476        })
477    }
478
479    /// Perform t-test
480    pub fn t_test(
481        &mut self,
482        dataset_id: &str,
483        column: &str,
484        hypothesis_type: HypothesisType,
485        privacy_preserved: bool,
486    ) -> Result<StatisticalAnalysisResult<TTestResult>, StatisticalError> {
487        let start_time = std::time::Instant::now();
488
489        // Get dataset
490        let dataset = self.data_storage.get_dataset(dataset_id)?;
491
492        // Find column index
493        let column_index = dataset
494            .column_names
495            .iter()
496            .position(|name| name == column)
497            .ok_or_else(|| StatisticalError::InvalidColumn(column.to_string()))?;
498
499        // Validate column type
500        if !matches!(
501            dataset.column_types[column_index],
502            DataType::Float32 | DataType::Float64
503        ) {
504            return Err(StatisticalError::InvalidOperation(
505                "T-test can only be computed on numeric columns".to_string(),
506            ));
507        }
508
509        // Extract column data
510        let mut values = Vec::new();
511        for row in &dataset.data {
512            match &row[column_index] {
513                DataValue::Float(value) => values.push(*value),
514                DataValue::Integer(value) => values.push(*value as f64),
515                DataValue::Null => continue,
516                _ => {
517                    return Err(StatisticalError::InvalidOperation(
518                        "Non-numeric data in column".to_string(),
519                    ))
520                }
521            }
522        }
523
524        if values.len() < 2 {
525            return Err(StatisticalError::InvalidData(
526                "Insufficient data for t-test".to_string(),
527            ));
528        }
529
530        // Compute t-test based on hypothesis type
531        let t_test_result = match hypothesis_type {
532            HypothesisType::OneSample => self.one_sample_t_test(&values, 0.0)?,
533            HypothesisType::TwoSample => {
534                return Err(StatisticalError::InvalidOperation(
535                    "Two-sample t-test requires two datasets".to_string(),
536                ))
537            }
538            HypothesisType::Paired => {
539                return Err(StatisticalError::InvalidOperation(
540                    "Paired t-test requires paired data".to_string(),
541                ))
542            }
543            HypothesisType::Independent => {
544                return Err(StatisticalError::InvalidOperation(
545                    "Independent t-test requires two samples".to_string(),
546                ))
547            }
548        };
549
550        let execution_time = start_time.elapsed().as_millis() as u64;
551
552        // Apply privacy if requested
553        let (final_result, privacy_cost) = if privacy_preserved {
554            let (noisy_t_statistic, cost) = self
555                .privacy_engine
556                .add_laplace_noise(t_test_result.t_statistic, 1.0)?;
557            let noisy_result = TTestResult {
558                t_statistic: noisy_t_statistic,
559                p_value: t_test_result.p_value,
560                degrees_of_freedom: t_test_result.degrees_of_freedom,
561                confidence_interval: t_test_result.confidence_interval,
562            };
563            (noisy_result, cost)
564        } else {
565            (t_test_result, 0.0)
566        };
567
568        // Update performance metrics
569        self.performance_monitor
570            .record_operation("t_test", execution_time, 0, privacy_cost);
571
572        Ok(StatisticalAnalysisResult {
573            result: final_result,
574            execution_time,
575            memory_usage: 0,
576            sample_size: values.len(),
577            confidence_level: 0.95,
578            privacy_preserved,
579            privacy_cost,
580        })
581    }
582
583    /// Generate histogram
584    pub fn histogram(
585        &mut self,
586        dataset_id: &str,
587        column: &str,
588        bins: usize,
589        privacy_preserved: bool,
590    ) -> Result<StatisticalAnalysisResult<HistogramResult>, StatisticalError> {
591        let start_time = std::time::Instant::now();
592
593        // Get dataset
594        let dataset = self.data_storage.get_dataset(dataset_id)?;
595
596        // Find column index
597        let column_index = dataset
598            .column_names
599            .iter()
600            .position(|name| name == column)
601            .ok_or_else(|| StatisticalError::InvalidColumn(column.to_string()))?;
602
603        // Validate column type
604        if !matches!(
605            dataset.column_types[column_index],
606            DataType::Float32 | DataType::Float64
607        ) {
608            return Err(StatisticalError::InvalidOperation(
609                "Histogram can only be computed on numeric columns".to_string(),
610            ));
611        }
612
613        // Extract column data
614        let mut values = Vec::new();
615        for row in &dataset.data {
616            match &row[column_index] {
617                DataValue::Float(value) => values.push(*value),
618                DataValue::Integer(value) => values.push(*value as f64),
619                DataValue::Null => continue,
620                _ => {
621                    return Err(StatisticalError::InvalidOperation(
622                        "Non-numeric data in column".to_string(),
623                    ))
624                }
625            }
626        }
627
628        if values.is_empty() {
629            return Err(StatisticalError::InvalidData(
630                "No valid data in column".to_string(),
631            ));
632        }
633
634        // Compute histogram
635        let histogram_result = self.compute_histogram(&values, bins)?;
636
637        let execution_time = start_time.elapsed().as_millis() as u64;
638
639        // Apply privacy if requested
640        let (final_result, privacy_cost) = if privacy_preserved {
641            let (noisy_counts, cost) = self
642                .privacy_engine
643                .add_histogram_noise(&histogram_result.counts)?;
644            let noisy_result = HistogramResult {
645                bins: histogram_result.bins,
646                counts: noisy_counts,
647                min_value: histogram_result.min_value,
648                max_value: histogram_result.max_value,
649                bin_width: histogram_result.bin_width,
650            };
651            (noisy_result, cost)
652        } else {
653            (histogram_result, 0.0)
654        };
655
656        // Update performance metrics
657        self.performance_monitor
658            .record_operation("histogram", execution_time, 0, privacy_cost);
659
660        Ok(StatisticalAnalysisResult {
661            result: final_result,
662            execution_time,
663            memory_usage: 0,
664            sample_size: values.len(),
665            confidence_level: 0.95,
666            privacy_preserved,
667            privacy_cost,
668        })
669    }
670
671    // ========================================================================
672    // Wired capability methods.
673    //
674    // Each declared `StatisticalOperation` is a genuine, MCP-reachable
675    // computation. Every method below marshals the caller's dataset into a
676    // slice and delegates to the canonical numeric kernels in
677    // `crate::solvers` (Modality-First Composition — no inline re-derivation);
678    // the descriptive/correlation/regression/hypothesis kernels live in
679    // `solvers::statistics`, the learning models in `solvers::learning`, and
680    // polynomial least-squares in `solvers::interpolation`.
681    // ========================================================================
682
683    /// Extract one numeric column as an owned `Vec<f64>` (Integer widened,
684    /// Null rows skipped). Errors on unknown column, non-numeric cell, or empty.
685    fn numeric_column(&self, dataset_id: &str, column: &str) -> Result<Vec<f64>, StatisticalError> {
686        let dataset = self.data_storage.get_dataset(dataset_id)?;
687        let idx = dataset
688            .column_names
689            .iter()
690            .position(|n| n == column)
691            .ok_or_else(|| StatisticalError::InvalidColumn(column.to_string()))?;
692        let mut out = Vec::with_capacity(dataset.data.len());
693        for row in &dataset.data {
694            match &row[idx] {
695                DataValue::Float(v) => out.push(*v),
696                DataValue::Integer(v) => out.push(*v as f64),
697                DataValue::Null => continue,
698                _ => {
699                    return Err(StatisticalError::InvalidOperation(
700                        "Non-numeric data in column".to_string(),
701                    ))
702                }
703            }
704        }
705        if out.is_empty() {
706            return Err(StatisticalError::InvalidData(
707                "No valid data in column".to_string(),
708            ));
709        }
710        Ok(out)
711    }
712
713    /// Extract several numeric columns row-aligned into a row-major `n × p`
714    /// matrix, skipping any row where one of the requested cells is Null (so
715    /// every returned row is complete). Returns `(data, n, p)`.
716    fn numeric_matrix(
717        &self,
718        dataset_id: &str,
719        columns: &[&str],
720    ) -> Result<(Vec<f64>, usize, usize), StatisticalError> {
721        let dataset = self.data_storage.get_dataset(dataset_id)?;
722        let p = columns.len();
723        if p == 0 {
724            return Err(StatisticalError::InvalidOperation(
725                "no columns given".to_string(),
726            ));
727        }
728        let mut idxs = Vec::with_capacity(p);
729        for c in columns {
730            idxs.push(
731                dataset
732                    .column_names
733                    .iter()
734                    .position(|n| n == c)
735                    .ok_or_else(|| StatisticalError::InvalidColumn((*c).to_string()))?,
736            );
737        }
738        let mut data = Vec::with_capacity(dataset.data.len() * p);
739        let mut n = 0;
740        'rows: for row in &dataset.data {
741            let mut tmp = [0.0f64; 0].to_vec();
742            tmp.reserve(p);
743            for &ix in &idxs {
744                match &row[ix] {
745                    DataValue::Float(v) => tmp.push(*v),
746                    DataValue::Integer(v) => tmp.push(*v as f64),
747                    DataValue::Null => continue 'rows,
748                    _ => {
749                        return Err(StatisticalError::InvalidOperation(
750                            "Non-numeric data in column".to_string(),
751                        ))
752                    }
753                }
754            }
755            data.extend_from_slice(&tmp);
756            n += 1;
757        }
758        if n == 0 {
759            return Err(StatisticalError::InvalidData(
760                "No complete rows across the requested columns".to_string(),
761            ));
762        }
763        Ok((data, n, p))
764    }
765
766    /// Split feature columns + a trailing label column into `(x, y, n, p)`,
767    /// row-aligned (rows with a Null in any of them are dropped together).
768    fn features_and_label(
769        &self,
770        dataset_id: &str,
771        feature_columns: &[&str],
772        label_column: &str,
773    ) -> Result<(Vec<f64>, Vec<f64>, usize, usize), StatisticalError> {
774        let mut cols: Vec<&str> = feature_columns.to_vec();
775        cols.push(label_column);
776        let (mat, n, p_all) = self.numeric_matrix(dataset_id, &cols)?;
777        let p = p_all - 1;
778        let mut x = Vec::with_capacity(n * p);
779        let mut y = Vec::with_capacity(n);
780        for r in 0..n {
781            x.extend_from_slice(&mat[r * p_all..r * p_all + p]);
782            y.push(mat[r * p_all + p]);
783        }
784        Ok((x, y, n, p))
785    }
786
787    fn scalar_result(&self, value: f64, n: usize) -> StatisticalAnalysisResult<f64> {
788        StatisticalAnalysisResult {
789            result: value,
790            execution_time: 0,
791            memory_usage: 0,
792            sample_size: n,
793            confidence_level: 0.95,
794            privacy_preserved: false,
795            privacy_cost: 0.0,
796        }
797    }
798
799    /// Standard deviation (`sample = true` → Bessel-corrected).
800    pub fn standard_deviation(
801        &self,
802        dataset_id: &str,
803        column: &str,
804        sample: bool,
805    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
806        let v = self.numeric_column(dataset_id, column)?;
807        let sd = crate::solvers::statistics::std_dev(&v, sample)
808            .ok_or_else(|| StatisticalError::InvalidData("empty column".to_string()))?;
809        Ok(self.scalar_result(sd, v.len()))
810    }
811
812    /// Sample skewness (Fisher-Pearson).
813    pub fn skewness(
814        &self,
815        dataset_id: &str,
816        column: &str,
817    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
818        let v = self.numeric_column(dataset_id, column)?;
819        let s = crate::solvers::statistics::skewness(&v)
820            .ok_or_else(|| StatisticalError::InvalidData("skewness undefined".to_string()))?;
821        Ok(self.scalar_result(s, v.len()))
822    }
823
824    /// Excess kurtosis.
825    pub fn kurtosis(
826        &self,
827        dataset_id: &str,
828        column: &str,
829    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
830        let v = self.numeric_column(dataset_id, column)?;
831        let k = crate::solvers::statistics::kurtosis(&v)
832            .ok_or_else(|| StatisticalError::InvalidData("kurtosis undefined".to_string()))?;
833        Ok(self.scalar_result(k, v.len()))
834    }
835
836    /// Modal value + its frequency.
837    pub fn mode(&self, dataset_id: &str, column: &str) -> Result<ModeResult, StatisticalError> {
838        let mut v = self.numeric_column(dataset_id, column)?;
839        let n = v.len();
840        let (value, count) = crate::solvers::statistics::mode_in_place(&mut v)
841            .ok_or_else(|| StatisticalError::InvalidData("empty column".to_string()))?;
842        Ok(ModeResult {
843            value,
844            count,
845            sample_size: n,
846        })
847    }
848
849    /// The `q`-quantile (`q ∈ [0,1]`) via linear interpolation between order
850    /// statistics. `percentile` is the same with `p ∈ [0,100]`.
851    pub fn quantile(
852        &self,
853        dataset_id: &str,
854        column: &str,
855        q: f64,
856    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
857        let mut v = self.numeric_column(dataset_id, column)?;
858        let n = v.len();
859        let val = crate::solvers::statistics::quantile_in_place(&mut v, q).ok_or_else(|| {
860            StatisticalError::InvalidOperation("quantile requires q in [0,1]".to_string())
861        })?;
862        Ok(self.scalar_result(val, n))
863    }
864
865    /// Covariance between two columns (`sample = true` → divide by n-1).
866    pub fn covariance(
867        &self,
868        dataset_id: &str,
869        column_x: &str,
870        column_y: &str,
871        sample: bool,
872    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
873        let (mat, n, _p) = self.numeric_matrix(dataset_id, &[column_x, column_y])?;
874        let x: Vec<f64> = (0..n).map(|r| mat[r * 2]).collect();
875        let y: Vec<f64> = (0..n).map(|r| mat[r * 2 + 1]).collect();
876        let cov = crate::solvers::statistics::covariance(&x, &y, sample)
877            .ok_or_else(|| StatisticalError::InvalidData("covariance undefined".to_string()))?;
878        Ok(self.scalar_result(cov, n))
879    }
880
881    /// Ordinary-least-squares simple linear regression `y ~ x` with full
882    /// inferential statistics (slope/intercept, R², standard errors, t, p).
883    pub fn linear_regression(
884        &self,
885        dataset_id: &str,
886        column_x: &str,
887        column_y: &str,
888    ) -> Result<crate::solvers::statistics::LinearRegression, StatisticalError> {
889        let (mat, n, _p) = self.numeric_matrix(dataset_id, &[column_x, column_y])?;
890        let x: Vec<f64> = (0..n).map(|r| mat[r * 2]).collect();
891        let y: Vec<f64> = (0..n).map(|r| mat[r * 2 + 1]).collect();
892        crate::solvers::statistics::simple_linear_regression(&x, &y).ok_or_else(|| {
893            StatisticalError::InvalidData(
894                "regression undefined (n<3 or zero variance in x)".to_string(),
895            )
896        })
897    }
898
899    /// Polynomial regression `y ~ poly(x, degree)` by least squares (normal
900    /// equations). Returns ascending-power coefficients and in-sample R².
901    pub fn polynomial_regression(
902        &self,
903        dataset_id: &str,
904        column_x: &str,
905        column_y: &str,
906        degree: usize,
907    ) -> Result<PolynomialFit, StatisticalError> {
908        let (mat, n, _p) = self.numeric_matrix(dataset_id, &[column_x, column_y])?;
909        let x: Vec<f64> = (0..n).map(|r| mat[r * 2]).collect();
910        let y: Vec<f64> = (0..n).map(|r| mat[r * 2 + 1]).collect();
911        let coeffs = crate::solvers::interpolation::least_squares::poly_fit(&x, &y, degree)
912            .map_err(|e| StatisticalError::InvalidOperation(format!("poly_fit: {:?}", e)))?;
913        // In-sample R² from the fitted coefficients.
914        let y_mean = crate::solvers::statistics::mean(&y).unwrap_or(0.0);
915        let mut ss_res = 0.0;
916        let mut ss_tot = 0.0;
917        for i in 0..n {
918            let yhat = crate::solvers::interpolation::least_squares::poly_eval(&coeffs, x[i]);
919            ss_res += (y[i] - yhat) * (y[i] - yhat);
920            ss_tot += (y[i] - y_mean) * (y[i] - y_mean);
921        }
922        let r_squared = if ss_tot > 0.0 {
923            1.0 - ss_res / ss_tot
924        } else {
925            0.0
926        };
927        Ok(PolynomialFit {
928            degree,
929            coefficients: coeffs,
930            r_squared,
931            n,
932        })
933    }
934
935    /// Binary logistic regression by IRLS (`label` in {0,1}), with Wald
936    /// standard errors, z-statistics and p-values on the coefficients.
937    pub fn logistic_regression(
938        &self,
939        dataset_id: &str,
940        feature_columns: &[&str],
941        label_column: &str,
942        fit_intercept: bool,
943    ) -> Result<crate::solvers::learning::glm::GlmModel, StatisticalError> {
944        let (x, y, n, p) = self.features_and_label(dataset_id, feature_columns, label_column)?;
945        crate::solvers::learning::glm::fit_logistic(&x, &y, n, p, fit_intercept)
946            .map_err(|e| StatisticalError::InvalidOperation(format!("logistic: {:?}", e)))
947    }
948
949    /// One-way ANOVA across the named columns (each column is a group). Groups
950    /// may have unequal lengths; Null cells are dropped per column.
951    pub fn anova(
952        &self,
953        dataset_id: &str,
954        group_columns: &[&str],
955    ) -> Result<crate::solvers::statistics::AnovaResult, StatisticalError> {
956        if group_columns.len() < 2 {
957            return Err(StatisticalError::InvalidOperation(
958                "ANOVA needs at least two groups".to_string(),
959            ));
960        }
961        let groups: Vec<Vec<f64>> = group_columns
962            .iter()
963            .map(|c| self.numeric_column(dataset_id, c))
964            .collect::<Result<_, _>>()?;
965        let refs: Vec<&[f64]> = groups.iter().map(|g| g.as_slice()).collect();
966        crate::solvers::statistics::one_way_anova(&refs).ok_or_else(|| {
967            StatisticalError::InvalidData("ANOVA undefined for these groups".to_string())
968        })
969    }
970
971    /// Chi-square goodness-of-fit test: observed counts vs. expected counts.
972    /// If `expected_column` is `None`, a uniform expectation is used.
973    pub fn chi_square_gof(
974        &self,
975        dataset_id: &str,
976        observed_column: &str,
977        expected_column: Option<&str>,
978    ) -> Result<crate::solvers::statistics::ChiSquareResult, StatisticalError> {
979        let observed = self.numeric_column(dataset_id, observed_column)?;
980        let expected = match expected_column {
981            Some(c) => self.numeric_column(dataset_id, c)?,
982            None => {
983                let total: f64 = observed.iter().sum();
984                let u = total / observed.len() as f64;
985                vec![u; observed.len()]
986            }
987        };
988        crate::solvers::statistics::chi_square_gof(&observed, &expected).ok_or_else(|| {
989            StatisticalError::InvalidOperation("chi-square GoF undefined".to_string())
990        })
991    }
992
993    /// Chi-square test of independence over a contingency table whose columns
994    /// are the named columns (each row of the table is one dataset row).
995    pub fn chi_square_independence(
996        &self,
997        dataset_id: &str,
998        columns: &[&str],
999    ) -> Result<crate::solvers::statistics::ChiSquareResult, StatisticalError> {
1000        let (mat, n, p) = self.numeric_matrix(dataset_id, columns)?;
1001        let rows: Vec<&[f64]> = (0..n).map(|r| &mat[r * p..(r + 1) * p]).collect();
1002        crate::solvers::statistics::chi_square_independence(&rows).ok_or_else(|| {
1003            StatisticalError::InvalidOperation("chi-square independence undefined".to_string())
1004        })
1005    }
1006
1007    /// Autocorrelation of a column at the given lag (biased estimator).
1008    pub fn autocorrelation(
1009        &self,
1010        dataset_id: &str,
1011        column: &str,
1012        lag: usize,
1013    ) -> Result<StatisticalAnalysisResult<f64>, StatisticalError> {
1014        let v = self.numeric_column(dataset_id, column)?;
1015        let r = crate::solvers::statistics::autocorrelation(&v, lag).ok_or_else(|| {
1016            StatisticalError::InvalidOperation(
1017                "autocorrelation undefined (lag>=n or constant series)".to_string(),
1018            )
1019        })?;
1020        Ok(self.scalar_result(r, v.len()))
1021    }
1022
1023    /// Simple moving average of a column with the given window.
1024    pub fn moving_average(
1025        &self,
1026        dataset_id: &str,
1027        column: &str,
1028        window: usize,
1029    ) -> Result<Vec<f64>, StatisticalError> {
1030        let v = self.numeric_column(dataset_id, column)?;
1031        if window == 0 || window > v.len() {
1032            return Err(StatisticalError::InvalidOperation(
1033                "window must be in 1..=n".to_string(),
1034            ));
1035        }
1036        let mut out = vec![0.0; v.len() - window + 1];
1037        crate::solvers::statistics::moving_average_into(&v, window, &mut out).ok_or_else(|| {
1038            StatisticalError::InvalidOperation("moving average failed".to_string())
1039        })?;
1040        Ok(out)
1041    }
1042
1043    /// Single exponential smoothing of a column with factor `alpha ∈ (0,1]`.
1044    pub fn exponential_smoothing(
1045        &self,
1046        dataset_id: &str,
1047        column: &str,
1048        alpha: f64,
1049    ) -> Result<Vec<f64>, StatisticalError> {
1050        let v = self.numeric_column(dataset_id, column)?;
1051        let mut out = vec![0.0; v.len()];
1052        crate::solvers::statistics::exponential_smoothing_into(&v, alpha, &mut out).ok_or_else(
1053            || StatisticalError::InvalidOperation("alpha must be in (0,1]".to_string()),
1054        )?;
1055        Ok(out)
1056    }
1057
1058    /// K-means clustering over the named feature columns. Returns the fitted
1059    /// model (centroids, per-point labels, inertia, convergence).
1060    pub fn kmeans(
1061        &self,
1062        dataset_id: &str,
1063        feature_columns: &[&str],
1064        k: usize,
1065        max_iter: usize,
1066        seed: u64,
1067    ) -> Result<crate::solvers::learning::clustering::kmeans::KMeansModel, StatisticalError> {
1068        let (x, n, p) = self.numeric_matrix(dataset_id, feature_columns)?;
1069        crate::solvers::learning::clustering::kmeans::fit(&x, n, p, k, max_iter, seed)
1070            .map_err(|e| StatisticalError::InvalidOperation(format!("kmeans: {:?}", e)))
1071    }
1072
1073    /// Soft-margin linear SVM over the named feature columns with a boolean
1074    /// `label` column (non-zero = positive class). Returns a fit summary with
1075    /// support-vector count and in-sample accuracy.
1076    pub fn linear_svm(
1077        &self,
1078        dataset_id: &str,
1079        feature_columns: &[&str],
1080        label_column: &str,
1081        c: f64,
1082    ) -> Result<SvmFitResult, StatisticalError> {
1083        let (x, y, n, p) = self.features_and_label(dataset_id, feature_columns, label_column)?;
1084        let labels: Vec<bool> = y.iter().map(|&v| v != 0.0).collect();
1085        let model = crate::solvers::learning::classification::svm::fit(
1086            &x,
1087            &labels,
1088            n,
1089            p,
1090            c,
1091            crate::solvers::learning::classification::svm::Kernel::Linear,
1092            100,
1093            1e-3,
1094        )
1095        .map_err(|e| StatisticalError::InvalidOperation(format!("svm: {:?}", e)))?;
1096        let mut correct = 0usize;
1097        for i in 0..n {
1098            if model.predict_row(&x[i * p..(i + 1) * p]) == labels[i] {
1099                correct += 1;
1100            }
1101        }
1102        Ok(SvmFitResult {
1103            n_support_vectors: model.n_support_vectors(),
1104            train_accuracy: correct as f64 / n as f64,
1105            n,
1106            n_features: p,
1107        })
1108    }
1109
1110    /// Random-forest fit over the named feature columns and a target column.
1111    /// `classifier = true` fits a classification forest (integer labels) and
1112    /// reports in-sample accuracy; otherwise a regression forest reporting R².
1113    pub fn random_forest(
1114        &self,
1115        dataset_id: &str,
1116        feature_columns: &[&str],
1117        target_column: &str,
1118        n_trees: usize,
1119        classifier: bool,
1120        seed: u64,
1121    ) -> Result<RandomForestFitResult, StatisticalError> {
1122        use crate::solvers::learning::trees::decision_tree::TreeParams;
1123        use crate::solvers::learning::trees::random_forest::RandomForest;
1124        let (x, y, n, p) = self.features_and_label(dataset_id, feature_columns, target_column)?;
1125        let params = TreeParams::default();
1126        let (metric, model_predict): (f64, Vec<f64>) = if classifier {
1127            let labels: Vec<usize> = y.iter().map(|&v| v.round().max(0.0) as usize).collect();
1128            let rf = RandomForest::fit_classifier(&x, &labels, n, p, n_trees, params, seed)
1129                .map_err(|e| StatisticalError::InvalidOperation(format!("forest: {:?}", e)))?;
1130            let mut correct = 0usize;
1131            for i in 0..n {
1132                if rf.predict_class(&x[i * p..(i + 1) * p]) == labels[i] {
1133                    correct += 1;
1134                }
1135            }
1136            (correct as f64 / n as f64, vec![])
1137        } else {
1138            let rf = RandomForest::fit_regressor(&x, &y, n, p, n_trees, params, seed)
1139                .map_err(|e| StatisticalError::InvalidOperation(format!("forest: {:?}", e)))?;
1140            let preds: Vec<f64> = (0..n)
1141                .map(|i| rf.predict_row(&x[i * p..(i + 1) * p]))
1142                .collect();
1143            let y_mean = crate::solvers::statistics::mean(&y).unwrap_or(0.0);
1144            let mut ss_res = 0.0;
1145            let mut ss_tot = 0.0;
1146            for i in 0..n {
1147                ss_res += (y[i] - preds[i]) * (y[i] - preds[i]);
1148                ss_tot += (y[i] - y_mean) * (y[i] - y_mean);
1149            }
1150            let r2 = if ss_tot > 0.0 {
1151                1.0 - ss_res / ss_tot
1152            } else {
1153                0.0
1154            };
1155            (r2, preds)
1156        };
1157        let _ = model_predict;
1158        Ok(RandomForestFitResult {
1159            n_trees,
1160            classifier,
1161            train_metric: metric,
1162            n,
1163            n_features: p,
1164        })
1165    }
1166
1167    /// Get performance statistics
1168    pub fn get_performance_stats(&self) -> SystemMetrics {
1169        self.performance_monitor.get_system_metrics()
1170    }
1171
1172    /// List all datasets
1173    pub fn list_datasets(&self) -> Vec<String> {
1174        self.data_storage.list_datasets()
1175    }
1176
1177    /// Get dataset information
1178    pub fn get_dataset_info(&self, dataset_id: &str) -> Option<DatasetMetadata> {
1179        self.data_storage.get_dataset_metadata(dataset_id)
1180    }
1181
1182    // Internal methods
1183
1184    /// Compute Pearson correlation
1185    fn pearson_correlation(&self, x: &[f64], y: &[f64]) -> Result<f64, StatisticalError> {
1186        // Modality-First: the math lives in the engine's statistics solver.
1187        crate::solvers::statistics::pearson(x, y).ok_or_else(|| {
1188            StatisticalError::InvalidData("Invalid data for correlation".to_string())
1189        })
1190    }
1191
1192    /// Compute Spearman correlation
1193    fn spearman_correlation(&self, x: &[f64], y: &[f64]) -> Result<f64, StatisticalError> {
1194        // Convert to ranks
1195        let x_ranked = self.rank_values(x);
1196        let y_ranked = self.rank_values(y);
1197
1198        // Compute Pearson correlation on ranks
1199        self.pearson_correlation(&x_ranked, &y_ranked)
1200    }
1201
1202    /// Compute Kendall correlation
1203    fn kendall_correlation(&self, x: &[f64], y: &[f64]) -> Result<f64, StatisticalError> {
1204        // Modality-First: the math lives in the engine's statistics solver.
1205        crate::solvers::statistics::kendall(x, y).ok_or_else(|| {
1206            StatisticalError::InvalidData("Invalid data for correlation".to_string())
1207        })
1208    }
1209
1210    /// Rank values
1211    fn rank_values(&self, values: &[f64]) -> Vec<f64> {
1212        // Modality-First: ranking lives in the engine. The wrapper owns the
1213        // scratch/output buffers (heap is fine at this composition boundary).
1214        let n = values.len();
1215        let mut idx = vec![0usize; n];
1216        let mut ranks = vec![0.0; n];
1217        let _ = crate::solvers::statistics::rank_into(values, &mut idx, &mut ranks);
1218        ranks
1219    }
1220
1221    /// One sample t-test
1222    fn one_sample_t_test(&self, values: &[f64], mu: f64) -> Result<TTestResult, StatisticalError> {
1223        let n = values.len();
1224        if n < 2 {
1225            return Err(StatisticalError::InvalidData(
1226                "Insufficient data for t-test".to_string(),
1227            ));
1228        }
1229
1230        let t = crate::solvers::statistics::one_sample_t(values, mu).ok_or_else(|| {
1231            StatisticalError::InvalidData("Insufficient data for t-test".to_string())
1232        })?;
1233        Ok(TTestResult {
1234            t_statistic: t.t_statistic,
1235            p_value: t.p_value,
1236            degrees_of_freedom: t.degrees_of_freedom,
1237            confidence_interval: t.confidence_interval,
1238        })
1239    }
1240
1241    /// Compute histogram
1242    fn compute_histogram(
1243        &self,
1244        values: &[f64],
1245        bins: usize,
1246    ) -> Result<HistogramResult, StatisticalError> {
1247        if values.is_empty() {
1248            return Err(StatisticalError::InvalidData(
1249                "No data for histogram".to_string(),
1250            ));
1251        }
1252
1253        // Modality-First: binning lives in the engine; the wrapper owns the
1254        // counts buffer and builds the domain result.
1255        let mut counts = vec![0u32; bins];
1256        let range = crate::solvers::statistics::histogram_into(values, &mut counts)
1257            .ok_or_else(|| StatisticalError::InvalidData("No data for histogram".to_string()))?;
1258        Ok(HistogramResult {
1259            bins,
1260            counts,
1261            min_value: range.min,
1262            max_value: range.max,
1263            bin_width: range.bin_width,
1264        })
1265    }
1266}
1267
1268// Supporting implementations