qualia_core_db/inference/kv_capture.rs
1//! W5b — KV-vector capture hook for sparse-dictionary calibration.
2//!
3//! Sibling of [`crate::llm_awq`] (AWQ activation capture): a gated forward-pass hook that records the
4//! engine's real per-layer K and V vectors during a calibration forward over the eval corpus. The
5//! sparse-dictionary go/no-go ([`crate::wgsl_forge::calibration`]) needs OUR engine's actual KV
6//! geometry — GQA layout, RoPE convention, and layer shapes are engine-specific, so this cannot come
7//! from synthetic data or another runtime.
8//!
9//! Capture point: native attention runs through `gguf_bridge::…::cpu_attention_pass` (the wasm-proven
10//! CPU SDPA the native path routes through), which writes each token's post-RoPE K and pre-RoPE V into
11//! the KV cache. We tap those exact vectors there — post-RoPE K is what the int8 KV cache would
12//! quantize, so this is an apples-to-apples source for the int8-vs-dictionary comparison.
13//!
14//! Gated + bounded: off in production (one relaxed atomic load on the attention path). When on, it
15//! appends into a per-layer buffer under a mutex, capped at `max_per_layer` vectors per layer per
16//! stream (K, V) so memory stays bounded regardless of corpus length. Calibration-only — never a
17//! production hot path.
18
19#![cfg(not(target_arch = "wasm32"))]
20
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Mutex, OnceLock};
23
24static ENABLED: AtomicBool = AtomicBool::new(false);
25
26struct KvBuf {
27 /// Head-dim of the captured vectors; 0 until the first record self-sizes it.
28 head_dim: usize,
29 /// Cap on vectors per layer per stream.
30 max_per_layer: usize,
31 n_layer: usize,
32 /// Per-layer flat blobs; `k[layer]` = captured K vectors concatenated (len = count * head_dim).
33 k: Vec<Vec<f32>>,
34 v: Vec<Vec<f32>>,
35}
36
37fn buf() -> &'static Mutex<Option<KvBuf>> {
38 static B: OnceLock<Mutex<Option<KvBuf>>> = OnceLock::new();
39 B.get_or_init(|| Mutex::new(None))
40}
41
42/// Begin a KV capture for a model with up to `n_layer` layers, keeping at most `max_per_layer` K and
43/// `max_per_layer` V vectors per layer. `head_dim` self-sizes on the first recorded vector.
44pub fn enable(n_layer: usize, max_per_layer: usize) {
45 if let Ok(mut g) = buf().lock() {
46 *g = Some(KvBuf {
47 head_dim: 0,
48 max_per_layer,
49 n_layer,
50 k: (0..n_layer).map(|_| Vec::new()).collect(),
51 v: (0..n_layer).map(|_| Vec::new()).collect(),
52 });
53 }
54 ENABLED.store(true, Ordering::Relaxed);
55}
56
57/// Disables KV capture globally.
58pub fn disable() {
59 ENABLED.store(false, Ordering::Relaxed);
60}
61
62#[inline]
63/// `true` when enabled.
64pub fn is_enabled() -> bool {
65 ENABLED.load(Ordering::Relaxed)
66}
67
68/// Record every head's K (`k_not_v = true`) or V vector from a projection slice. `proj` holds `n_kv`
69/// contiguous head vectors of `head_dim` each (`[h0…][h1…]…`). No-op (one atomic load) when disabled;
70/// stops appending to a layer/stream once its `max_per_layer` cap is hit.
71#[inline]
72pub fn record(layer: usize, k_not_v: bool, proj: &[f32], n_kv: usize, head_dim: usize) {
73 if !ENABLED.load(Ordering::Relaxed) || head_dim == 0 {
74 return;
75 }
76 let Ok(mut g) = buf().lock() else {
77 return;
78 };
79 let Some(b) = g.as_mut() else {
80 return;
81 };
82 if b.head_dim == 0 {
83 b.head_dim = head_dim;
84 }
85 if b.head_dim != head_dim || layer >= b.n_layer {
86 return;
87 }
88 let cap_floats = b.max_per_layer.saturating_mul(head_dim);
89 let dst = if k_not_v {
90 &mut b.k[layer]
91 } else {
92 &mut b.v[layer]
93 };
94 for h in 0..n_kv {
95 if dst.len() >= cap_floats {
96 break;
97 }
98 let s = h * head_dim;
99 if s + head_dim > proj.len() {
100 break;
101 }
102 dst.extend_from_slice(&proj[s..s + head_dim]);
103 }
104}
105
106/// One layer's captured vectors, split into `head_dim`-length rows.
107pub struct KvCapture {
108 pub head_dim: usize,
109 /// `k[layer]` = list of captured K vectors (each `head_dim` long).
110 pub k: Vec<Vec<Vec<f32>>>,
111 pub v: Vec<Vec<Vec<f32>>>,
112}
113
114impl KvCapture {
115 /// Total K vectors captured across all layers.
116 pub fn total_k(&self) -> usize {
117 self.k.iter().map(|l| l.len()).sum()
118 }
119 /// Total V vectors captured across all layers.
120 pub fn total_v(&self) -> usize {
121 self.v.iter().map(|l| l.len()).sum()
122 }
123}
124
125/// Copy out the captured vectors (splitting the flat per-layer blobs into rows). Returns `None` if
126/// nothing was captured (capture never enabled, or the forward never hit the CPU attention path).
127pub fn snapshot() -> Option<KvCapture> {
128 let g = buf().lock().ok()?;
129 let b = g.as_ref()?;
130 if b.head_dim == 0 {
131 return None;
132 }
133 let hd = b.head_dim;
134 let rows =
135 |flat: &Vec<f32>| -> Vec<Vec<f32>> { flat.chunks_exact(hd).map(|c| c.to_vec()).collect() };
136 Some(KvCapture {
137 head_dim: hd,
138 k: b.k.iter().map(rows).collect(),
139 v: b.v.iter().map(rows).collect(),
140 })
141}
142
143/// Drop the capture buffer (free the calibration-only heap).
144pub fn clear() {
145 if let Ok(mut g) = buf().lock() {
146 *g = None;
147 }
148}