Skip to main content

qualia_core_db/inference/
inference_awq.rs

1//! W1/AWQ — activation-statistics capture for Activation-aware Weight Quantization (no external libs).
2//!
3//! AWQ's premise: a weight's salience is set by the magnitude of the *activation* it multiplies, so
4//! scaling each input channel by `s_j = max|X_j|` before quantizing preserves the channels that matter
5//! — letting aggressive (ternary) quantization survive. This module is AWQ **step 1, the forward hook**:
6//! during a calibration forward over the eval corpus it records, per FFN layer, the per-input-channel
7//! max |activation| at the FFN input (post-`ffn_norm`, the input to the gate/up projections).
8//!
9//! Lock-free + gated: off in production (one relaxed atomic load on the FFN path). The accumulator is
10//! `fetch_max` over `|x|.to_bits()` — valid because `|x| >= 0`, so u32 bit-order matches float order.
11//! Heap (the stats buffer) is calibration-only, allocated once, never on a production hot path.
12
13use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
14use std::sync::OnceLock;
15
16/// Upper bounds for the fixed stats buffer (covers up to ~8B-param transformer shapes).
17const MAX_LAYERS: usize = 80;
18const MAX_CHAN: usize = 8192;
19
20static ENABLED: AtomicBool = AtomicBool::new(false);
21static N_LAYER: AtomicU32 = AtomicU32::new(0);
22static N_CHAN: AtomicU32 = AtomicU32::new(0);
23/// Per-forward layer index (reset by [`begin_forward`]); the FFN hook increments it 0..n_layer-1.
24static LAYER_CURSOR: AtomicU32 = AtomicU32::new(0);
25
26/// `max |activation|` bits at `[layer * N_CHAN + chan]`. Allocated once at first use.
27fn stats() -> &'static Vec<AtomicU32> {
28    static STATS: OnceLock<Vec<AtomicU32>> = OnceLock::new();
29    STATS.get_or_init(|| {
30        (0..MAX_LAYERS * MAX_CHAN)
31            .map(|_| AtomicU32::new(0))
32            .collect()
33    })
34}
35
36/// Begin an AWQ calibration capture for a model with `n_layer` FFN layers of `n_chan` input channels.
37pub fn enable(n_layer: u32, n_chan: u32) -> Result<(), String> {
38    if n_layer as usize > MAX_LAYERS || n_chan as usize > MAX_CHAN {
39        return Err(format!(
40            "AWQ capture exceeds bounds: n_layer={n_layer} (max {MAX_LAYERS}), n_chan={n_chan} (max {MAX_CHAN})"
41        ));
42    }
43    N_LAYER.store(n_layer, Ordering::Relaxed);
44    N_CHAN.store(n_chan, Ordering::Relaxed);
45    reset();
46    ENABLED.store(true, Ordering::Relaxed);
47    Ok(())
48}
49
50/// Disables AWQ globally.
51pub fn disable() {
52    ENABLED.store(false, Ordering::Relaxed);
53}
54
55/// Zero the accumulators + reset the per-forward layer cursor.
56pub fn reset() {
57    LAYER_CURSOR.store(0, Ordering::Relaxed);
58    for a in stats().iter() {
59        a.store(0, Ordering::Relaxed);
60    }
61}
62
63/// Call once at the start of each token's forward so the layer cursor tracks layers 0..n_layer-1.
64/// No-op (one atomic load) when capture is off.
65#[inline]
66pub fn begin_forward() {
67    if ENABLED.load(Ordering::Relaxed) {
68        LAYER_CURSOR.store(0, Ordering::Relaxed);
69    }
70}
71
72/// Record one FFN layer's post-norm input channels. Called from the FFN forward; no-op when off.
73#[inline]
74pub fn record_ffn_input(x: &[f32]) {
75    if !ENABLED.load(Ordering::Relaxed) {
76        return;
77    }
78    let n_chan = N_CHAN.load(Ordering::Relaxed) as usize;
79    let n_layer = N_LAYER.load(Ordering::Relaxed) as usize;
80    if n_chan == 0 {
81        return;
82    }
83    let layer = LAYER_CURSOR.fetch_add(1, Ordering::Relaxed) as usize;
84    if layer >= n_layer {
85        return;
86    }
87    let s = stats();
88    let base = layer * n_chan;
89    let lim = x.len().min(n_chan);
90    for c in 0..lim {
91        // |x| >= 0 → u32 bit pattern is monotone in the float value, so fetch_max is a true max.
92        s[base + c].fetch_max(x[c].abs().to_bits(), Ordering::Relaxed);
93    }
94}
95
96/// Per-layer per-channel max |activation| (`[layer][chan]`). Snapshot after the calibration pass.
97pub fn snapshot() -> Vec<Vec<f32>> {
98    let n_layer = N_LAYER.load(Ordering::Relaxed) as usize;
99    let n_chan = N_CHAN.load(Ordering::Relaxed) as usize;
100    if n_chan == 0 {
101        return Vec::new();
102    }
103    let s = stats();
104    (0..n_layer)
105        .map(|l| {
106            (0..n_chan)
107                .map(|c| f32::from_bits(s[l * n_chan + c].load(Ordering::Relaxed)))
108                .collect()
109        })
110        .collect()
111}
112
113#[inline]
114/// `true` when enabled.
115pub fn is_enabled() -> bool {
116    ENABLED.load(Ordering::Relaxed)
117}