Skip to main content

qualia_core_db/inference/
kv_capture.rs

1//! W5b — KV-vector capture hook for sparse-dictionary calibration.
2//!
3//! Sibling of [`crate::llm_awq`] (AWQ activation capture): a gated forward-pass hook that records the
4//! engine's real per-layer K and V vectors during a calibration forward over the eval corpus. The
5//! sparse-dictionary go/no-go ([`crate::wgsl_forge::calibration`]) needs OUR engine's actual KV
6//! geometry — GQA layout, RoPE convention, and layer shapes are engine-specific, so this cannot come
7//! from synthetic data or another runtime.
8//!
9//! Capture point: native attention runs through `gguf_bridge::…::cpu_attention_pass` (the wasm-proven
10//! CPU SDPA the native path routes through), which writes each token's post-RoPE K and pre-RoPE V into
11//! the KV cache. We tap those exact vectors there — post-RoPE K is what the int8 KV cache would
12//! quantize, so this is an apples-to-apples source for the int8-vs-dictionary comparison.
13//!
14//! Gated + bounded: off in production (one relaxed atomic load on the attention path). When on, it
15//! appends into a per-layer buffer under a mutex, capped at `max_per_layer` vectors per layer per
16//! stream (K, V) so memory stays bounded regardless of corpus length. Calibration-only — never a
17//! production hot path.
18
19#![cfg(not(target_arch = "wasm32"))]
20
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Mutex, OnceLock};
23
24static ENABLED: AtomicBool = AtomicBool::new(false);
25
26struct KvBuf {
27    /// Head-dim of the captured vectors; 0 until the first record self-sizes it.
28    head_dim: usize,
29    /// Cap on vectors per layer per stream.
30    max_per_layer: usize,
31    n_layer: usize,
32    /// Per-layer flat blobs; `k[layer]` = captured K vectors concatenated (len = count * head_dim).
33    k: Vec<Vec<f32>>,
34    v: Vec<Vec<f32>>,
35}
36
37fn buf() -> &'static Mutex<Option<KvBuf>> {
38    static B: OnceLock<Mutex<Option<KvBuf>>> = OnceLock::new();
39    B.get_or_init(|| Mutex::new(None))
40}
41
42/// Begin a KV capture for a model with up to `n_layer` layers, keeping at most `max_per_layer` K and
43/// `max_per_layer` V vectors per layer. `head_dim` self-sizes on the first recorded vector.
44pub fn enable(n_layer: usize, max_per_layer: usize) {
45    if let Ok(mut g) = buf().lock() {
46        *g = Some(KvBuf {
47            head_dim: 0,
48            max_per_layer,
49            n_layer,
50            k: (0..n_layer).map(|_| Vec::new()).collect(),
51            v: (0..n_layer).map(|_| Vec::new()).collect(),
52        });
53    }
54    ENABLED.store(true, Ordering::Relaxed);
55}
56
57/// Disables KV capture globally.
58pub fn disable() {
59    ENABLED.store(false, Ordering::Relaxed);
60}
61
62#[inline]
63/// `true` when enabled.
64pub fn is_enabled() -> bool {
65    ENABLED.load(Ordering::Relaxed)
66}
67
68/// Record every head's K (`k_not_v = true`) or V vector from a projection slice. `proj` holds `n_kv`
69/// contiguous head vectors of `head_dim` each (`[h0…][h1…]…`). No-op (one atomic load) when disabled;
70/// stops appending to a layer/stream once its `max_per_layer` cap is hit.
71#[inline]
72pub fn record(layer: usize, k_not_v: bool, proj: &[f32], n_kv: usize, head_dim: usize) {
73    if !ENABLED.load(Ordering::Relaxed) || head_dim == 0 {
74        return;
75    }
76    let Ok(mut g) = buf().lock() else {
77        return;
78    };
79    let Some(b) = g.as_mut() else {
80        return;
81    };
82    if b.head_dim == 0 {
83        b.head_dim = head_dim;
84    }
85    if b.head_dim != head_dim || layer >= b.n_layer {
86        return;
87    }
88    let cap_floats = b.max_per_layer.saturating_mul(head_dim);
89    let dst = if k_not_v {
90        &mut b.k[layer]
91    } else {
92        &mut b.v[layer]
93    };
94    for h in 0..n_kv {
95        if dst.len() >= cap_floats {
96            break;
97        }
98        let s = h * head_dim;
99        if s + head_dim > proj.len() {
100            break;
101        }
102        dst.extend_from_slice(&proj[s..s + head_dim]);
103    }
104}
105
106/// One layer's captured vectors, split into `head_dim`-length rows.
107pub struct KvCapture {
108    pub head_dim: usize,
109    /// `k[layer]` = list of captured K vectors (each `head_dim` long).
110    pub k: Vec<Vec<Vec<f32>>>,
111    pub v: Vec<Vec<Vec<f32>>>,
112}
113
114impl KvCapture {
115    /// Total K vectors captured across all layers.
116    pub fn total_k(&self) -> usize {
117        self.k.iter().map(|l| l.len()).sum()
118    }
119    /// Total V vectors captured across all layers.
120    pub fn total_v(&self) -> usize {
121        self.v.iter().map(|l| l.len()).sum()
122    }
123}
124
125/// Copy out the captured vectors (splitting the flat per-layer blobs into rows). Returns `None` if
126/// nothing was captured (capture never enabled, or the forward never hit the CPU attention path).
127pub fn snapshot() -> Option<KvCapture> {
128    let g = buf().lock().ok()?;
129    let b = g.as_ref()?;
130    if b.head_dim == 0 {
131        return None;
132    }
133    let hd = b.head_dim;
134    let rows =
135        |flat: &Vec<f32>| -> Vec<Vec<f32>> { flat.chunks_exact(hd).map(|c| c.to_vec()).collect() };
136    Some(KvCapture {
137        head_dim: hd,
138        k: b.k.iter().map(rows).collect(),
139        v: b.v.iter().map(rows).collect(),
140    })
141}
142
143/// Drop the capture buffer (free the calibration-only heap).
144pub fn clear() {
145    if let Ok(mut g) = buf().lock() {
146        *g = None;
147    }
148}