qualia_core_db/solvers/statistics/regression/
outliers.rs1use crate::solvers::linear_algebra::cholesky::{cholesky_factor, cholesky_solve};
4use crate::solvers::statistics::anomaly::{
5 is_multivariate_outlier, mahalanobis_sq, z_score_outliers,
6};
7use crate::solvers::statistics::descriptive::{mean, median_sorted};
8
9use serde::Serialize;
10
11#[derive(Debug, Clone, PartialEq, Serialize)]
13pub struct UnivariateOutlierScreen {
14 pub mean: f64,
15 pub median: f64,
16 pub mean_median_gap: f64,
17 pub sigma2_indices: Vec<usize>,
18 pub sigma3_indices: Vec<usize>,
19}
20
21pub fn univariate_outlier_screen(values: &[f64]) -> Option<UnivariateOutlierScreen> {
23 if values.len() < 4 {
24 return None;
25 }
26 let mu = mean(values)?;
27 let mut sorted = values.to_vec();
28 sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(core::cmp::Ordering::Equal));
29 let med = median_sorted(&sorted)?;
30 let s2 = z_score_outliers(values, 2.0).unwrap_or_default();
31 let s3 = z_score_outliers(values, 3.0).unwrap_or_default();
32 Some(UnivariateOutlierScreen {
33 mean: mu,
34 median: med,
35 mean_median_gap: (mu - med).abs(),
36 sigma2_indices: s2,
37 sigma3_indices: s3,
38 })
39}
40
41#[derive(Debug, Clone, PartialEq, Serialize)]
42pub struct MahalanobisOutliers {
43 pub distances_sq: Vec<f64>,
44 pub flagged_indices: Vec<usize>,
45 pub alpha: f64,
46}
47
48pub fn mahalanobis_outliers(
50 x: &[f64],
51 n: usize,
52 d: usize,
53 alpha: f64,
54) -> Option<MahalanobisOutliers> {
55 if d == 0 || n <= d + 1 || x.len() != n * d || !(0.0 < alpha && alpha < 1.0) {
56 return None;
57 }
58 let mut mu = vec![0.0; d];
59 for i in 0..n {
60 for j in 0..d {
61 mu[j] += x[i * d + j];
62 }
63 }
64 for m in mu.iter_mut() {
65 *m /= n as f64;
66 }
67 let mut cov = vec![0.0; d * d];
68 for i in 0..n {
69 for a in 0..d {
70 let da = x[i * d + a] - mu[a];
71 for b in 0..d {
72 cov[a * d + b] += da * (x[i * d + b] - mu[b]);
73 }
74 }
75 }
76 let denom = (n - 1) as f64;
77 for v in cov.iter_mut() {
78 *v /= denom;
79 }
80 let mut l = vec![0.0; d * d];
82 cholesky_factor(d, &cov, &mut l).ok()?;
83 let mut inv = vec![0.0; d * d];
84 let mut ej = vec![0.0; d];
85 let mut cj = vec![0.0; d];
86 for j in 0..d {
87 ej.iter_mut().for_each(|v| *v = 0.0);
88 ej[j] = 1.0;
89 cholesky_solve(d, &l, &ej, &mut cj).ok()?;
90 for i in 0..d {
91 inv[i * d + j] = cj[i];
92 }
93 }
94
95 let mut distances_sq = Vec::with_capacity(n);
96 let mut flagged_indices = Vec::new();
97 for i in 0..n {
98 let row = &x[i * d..(i + 1) * d];
99 let d2 = mahalanobis_sq(row, &mu, &inv)?;
100 distances_sq.push(d2);
101 if is_multivariate_outlier(row, &mu, &inv, alpha)? {
102 flagged_indices.push(i);
103 }
104 }
105 Some(MahalanobisOutliers {
106 distances_sq,
107 flagged_indices,
108 alpha,
109 })
110}
111
112#[cfg(test)]
113mod tests {
114 use super::*;
115
116 #[test]
117 fn univariate_flags_spike() {
118 let v = [10.0, 11.0, 9.5, 10.2, 10.1, 9.8, 50.0];
119 let s = univariate_outlier_screen(&v).unwrap();
120 assert!(s.sigma2_indices.contains(&6));
121 }
122
123 #[test]
124 fn mahalanobis_flags_far_point() {
125 let mut x = Vec::new();
127 for i in 0..12 {
128 let t = (i as f64) * 0.02;
129 x.push(t);
130 x.push(-t);
131 }
132 x.extend_from_slice(&[4.0, 4.0]);
133 let n = 13;
134 let r = mahalanobis_outliers(&x, n, 2, 0.05).unwrap();
135 assert!(
136 r.flagged_indices.contains(&(n - 1))
137 || r.distances_sq[n - 1] == r.distances_sq.iter().cloned().fold(0.0_f64, f64::max),
138 "flags={:?} d2={:?}",
139 r.flagged_indices,
140 r.distances_sq
141 );
142 assert!(r.distances_sq[n - 1] > 5.0);
143 }
144}