qualia_core_db/inference/
inference_awq.rs1use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
14use std::sync::OnceLock;
15
16const MAX_LAYERS: usize = 80;
18const MAX_CHAN: usize = 8192;
19
20static ENABLED: AtomicBool = AtomicBool::new(false);
21static N_LAYER: AtomicU32 = AtomicU32::new(0);
22static N_CHAN: AtomicU32 = AtomicU32::new(0);
23static LAYER_CURSOR: AtomicU32 = AtomicU32::new(0);
25
26fn stats() -> &'static Vec<AtomicU32> {
28 static STATS: OnceLock<Vec<AtomicU32>> = OnceLock::new();
29 STATS.get_or_init(|| {
30 (0..MAX_LAYERS * MAX_CHAN)
31 .map(|_| AtomicU32::new(0))
32 .collect()
33 })
34}
35
36pub fn enable(n_layer: u32, n_chan: u32) -> Result<(), String> {
38 if n_layer as usize > MAX_LAYERS || n_chan as usize > MAX_CHAN {
39 return Err(format!(
40 "AWQ capture exceeds bounds: n_layer={n_layer} (max {MAX_LAYERS}), n_chan={n_chan} (max {MAX_CHAN})"
41 ));
42 }
43 N_LAYER.store(n_layer, Ordering::Relaxed);
44 N_CHAN.store(n_chan, Ordering::Relaxed);
45 reset();
46 ENABLED.store(true, Ordering::Relaxed);
47 Ok(())
48}
49
50pub fn disable() {
52 ENABLED.store(false, Ordering::Relaxed);
53}
54
55pub fn reset() {
57 LAYER_CURSOR.store(0, Ordering::Relaxed);
58 for a in stats().iter() {
59 a.store(0, Ordering::Relaxed);
60 }
61}
62
63#[inline]
66pub fn begin_forward() {
67 if ENABLED.load(Ordering::Relaxed) {
68 LAYER_CURSOR.store(0, Ordering::Relaxed);
69 }
70}
71
72#[inline]
74pub fn record_ffn_input(x: &[f32]) {
75 if !ENABLED.load(Ordering::Relaxed) {
76 return;
77 }
78 let n_chan = N_CHAN.load(Ordering::Relaxed) as usize;
79 let n_layer = N_LAYER.load(Ordering::Relaxed) as usize;
80 if n_chan == 0 {
81 return;
82 }
83 let layer = LAYER_CURSOR.fetch_add(1, Ordering::Relaxed) as usize;
84 if layer >= n_layer {
85 return;
86 }
87 let s = stats();
88 let base = layer * n_chan;
89 let lim = x.len().min(n_chan);
90 for c in 0..lim {
91 s[base + c].fetch_max(x[c].abs().to_bits(), Ordering::Relaxed);
93 }
94}
95
96pub fn snapshot() -> Vec<Vec<f32>> {
98 let n_layer = N_LAYER.load(Ordering::Relaxed) as usize;
99 let n_chan = N_CHAN.load(Ordering::Relaxed) as usize;
100 if n_chan == 0 {
101 return Vec::new();
102 }
103 let s = stats();
104 (0..n_layer)
105 .map(|l| {
106 (0..n_chan)
107 .map(|c| f32::from_bits(s[l * n_chan + c].load(Ordering::Relaxed)))
108 .collect()
109 })
110 .collect()
111}
112
113#[inline]
114pub fn is_enabled() -> bool {
116 ENABLED.load(Ordering::Relaxed)
117}