qualia_core_db/solvers/learning/experiment/
ab_test.rs

1//! A/B testing (Practical Statistics ch 3) — compare two variants' conversion rates
2//! with a two-proportion z-test and a confidence interval on the lift. The honest
3//! output is "B beat A by 2.1pp [0.4pp, 3.8pp], p = 0.01" — effect size *with*
4//! uncertainty, not a bare "B wins". Reuses the Normal CDF/quantile.
5
6use crate::solvers::statistics::distributions::normal;
7
8/// Result of comparing two conversion rates.
9#[derive(Debug, Clone, Copy, PartialEq)]
10pub struct AbResult {
11    pub rate_a: f64,
12    pub rate_b: f64,
13    /// `rate_b − rate_a` (positive ⇒ B converts better).
14    pub difference: f64,
15    pub z_statistic: f64,
16    /// Two-sided p-value of "the rates are equal".
17    pub p_value: f64,
18    /// Confidence interval for the difference `rate_b − rate_a`.
19    pub ci: (f64, f64),
20    pub confidence: f64,
21}
22
23/// Two-proportion z-test. `conv_*` are conversions (successes), `n_*` the totals.
24/// `alpha` sets the CI/p-value (two-sided). `None` on invalid counts.
25pub fn ab_test(conv_a: u64, n_a: u64, conv_b: u64, n_b: u64, alpha: f64) -> Option<AbResult> {
26    if n_a == 0
27        || n_b == 0
28        || conv_a > n_a
29        || conv_b > n_b
30        || !(0.0..1.0).contains(&alpha)
31        || alpha <= 0.0
32    {
33        return None;
34    }
35    let (na, nb) = (n_a as f64, n_b as f64);
36    let pa = conv_a as f64 / na;
37    let pb = conv_b as f64 / nb;
38    let diff = pb - pa;
39
40    // Pooled proportion for the test statistic (under H0: pa == pb).
41    let pooled = (conv_a + conv_b) as f64 / (na + nb);
42    let se_pooled = (pooled * (1.0 - pooled) * (1.0 / na + 1.0 / nb)).sqrt();
43    let z = if se_pooled > 0.0 {
44        diff / se_pooled
45    } else {
46        0.0
47    };
48    let p_value = 2.0 * (1.0 - normal::standard_cdf(z.abs()));
49
50    // Unpooled SE for the confidence interval on the difference.
51    let se_diff = (pa * (1.0 - pa) / na + pb * (1.0 - pb) / nb).sqrt();
52    let zc = normal::standard_quantile(1.0 - alpha / 2.0);
53    let margin = zc * se_diff;
54
55    Some(AbResult {
56        rate_a: pa,
57        rate_b: pb,
58        difference: diff,
59        z_statistic: z,
60        p_value,
61        ci: (diff - margin, diff + margin),
62        confidence: 1.0 - alpha,
63    })
64}
65
66#[cfg(test)]
67mod tests {
68    use super::*;
69
70    #[test]
71    fn detects_a_real_lift() {
72        // B (12%) clearly beats A (10%) at large n → significant, CI excludes 0.
73        let r = ab_test(1000, 10_000, 1200, 10_000, 0.05).unwrap();
74        assert!((r.rate_a - 0.10).abs() < 1e-9 && (r.rate_b - 0.12).abs() < 1e-9);
75        assert!(r.difference > 0.0);
76        assert!(r.p_value < 0.01, "p={}", r.p_value);
77        assert!(r.ci.0 > 0.0, "CI lower {} should exclude 0", r.ci.0);
78    }
79
80    #[test]
81    fn no_real_difference_is_not_significant() {
82        let r = ab_test(500, 5000, 505, 5000, 0.05).unwrap();
83        assert!(r.p_value > 0.2, "p={}", r.p_value);
84        assert!(r.ci.0 < 0.0 && r.ci.1 > 0.0, "CI should straddle 0");
85    }
86
87    #[test]
88    fn small_sample_is_underpowered() {
89        // Same rates as the significant case but tiny n → not significant.
90        let r = ab_test(10, 100, 12, 100, 0.05).unwrap();
91        assert!(r.p_value > 0.05);
92    }
93
94    #[test]
95    fn guards() {
96        assert_eq!(ab_test(10, 0, 5, 100, 0.05), None);
97        assert_eq!(ab_test(150, 100, 5, 100, 0.05), None); // conv > n
98    }
99}
qualia_core_db/solvers/learning/experiment/ab_test.rs

qualia_core_db/solvers/learning/experiment/
ab_test.rs
