Skip to main content

qualia_core_db/inference/
kv_dict_runtime.rs

1//! W5b Phase 4b — runtime KV-dictionary install + reconstruction, in CORE (engine-side, no forge dep).
2//!
3//! Holds the certified per-layer K/V dictionaries and, when enabled, reconstructs each K/V vector on the
4//! KV-cache **write** path (`reconstruct_kv`) so attention reads the dictionary-reconstructed vectors.
5//! This is the engine half of "forge produces, engine runs": the forge learns + certifies + packages a
6//! dictionary artifact; the engine [`load_certified`]s it (verifying the provenance gate) and installs
7//! it here. Reconstruct-on-write is quality-identical to a real compressed cache (store code, reconstruct
8//! on read) — the compressed GPU cache layout + shader reconstruction is the remaining Phase 4b work.
9//!
10//! Gated + zero-cost when off (one relaxed atomic load on the attention path).
11
12#![cfg(not(target_arch = "wasm32"))]
13
14use crate::kv_dict::KvDictionary;
15use std::sync::atomic::{AtomicBool, Ordering};
16use std::sync::{Mutex, OnceLock};
17
18static ENABLED: AtomicBool = AtomicBool::new(false);
19
20struct Rt {
21    /// Per-layer K dictionaries (`None` = layer not certified / too few vectors → passthrough).
22    k: Vec<Option<KvDictionary>>,
23    v: Vec<Option<KvDictionary>>,
24    sparsity: usize,
25    /// head_dim (atom length) of the installed dictionaries; 0 if none installed.
26    head_dim: usize,
27    /// n_atoms of the installed dictionaries (from the first non-None dict).
28    n_atoms: usize,
29}
30
31fn rt() -> &'static Mutex<Option<Rt>> {
32    static R: OnceLock<Mutex<Option<Rt>>> = OnceLock::new();
33    R.get_or_init(|| Mutex::new(None))
34}
35
36/// The serialized dictionary artifact payload (what rides inside the framed `.q42art` after the
37/// provenance header). Shared by the forge packager and the engine loader — the one source of truth
38/// for the on-disk dictionary format.
39#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
40pub struct KvDictArtifact {
41    pub sparsity: usize,
42    pub head_dim: usize,
43    pub k: Vec<Option<KvDictionary>>,
44    pub v: Vec<Option<KvDictionary>>,
45}
46
47/// Install the per-layer dictionaries and turn reconstruction ON.
48pub fn enable(k: Vec<Option<KvDictionary>>, v: Vec<Option<KvDictionary>>, sparsity: usize) {
49    let first = k.iter().chain(v.iter()).flatten().next();
50    let head_dim = first.map(|d| d.dim).unwrap_or(0);
51    let n_atoms = first.map(|d| d.n_atoms).unwrap_or(0);
52    if let Ok(mut g) = rt().lock() {
53        *g = Some(Rt {
54            k,
55            v,
56            sparsity,
57            head_dim,
58            n_atoms,
59        });
60    }
61    ENABLED.store(true, Ordering::Relaxed);
62}
63
64/// Flatten the installed atoms into the arena layout `[layer][K atoms n_atoms×head_dim][V atoms …]`,
65/// with `(flat, n_atoms, head_dim)`; layers/streams with no dictionary are zero (never selected). The
66/// engine uploads this into the tail of each layer's KV-arena slice for the GPU shader to reconstruct.
67pub fn atoms_flat() -> Option<(Vec<f32>, usize, usize)> {
68    let g = rt().lock().ok()?;
69    let rt = g.as_ref()?;
70    if rt.head_dim == 0 || rt.n_atoms == 0 {
71        return None;
72    }
73    let (na, hd) = (rt.n_atoms, rt.head_dim);
74    let per_stream = na * hd;
75    let n_layer = rt.k.len().max(rt.v.len());
76    let mut out = vec![0f32; n_layer * 2 * per_stream];
77    for l in 0..n_layer {
78        let base = l * 2 * per_stream;
79        if let Some(Some(d)) = rt.k.get(l) {
80            let n = per_stream.min(d.atoms.len());
81            out[base..base + n].copy_from_slice(&d.atoms[..n]);
82        }
83        if let Some(Some(d)) = rt.v.get(l) {
84            let n = per_stream.min(d.atoms.len());
85            out[base + per_stream..base + per_stream + n].copy_from_slice(&d.atoms[..n]);
86        }
87    }
88    Some((out, na, hd))
89}
90
91/// The installed dictionary's `(sparsity, head_dim)`, or `None` if nothing is installed. The KV-cache
92/// layout consults this to size the dict-coded slots (Phase 4b step 3).
93pub fn installed_meta() -> Option<(usize, usize, usize)> {
94    let g = rt().lock().ok()?;
95    let rt = g.as_ref()?;
96    if rt.head_dim == 0 {
97        None
98    } else {
99        Some((rt.sparsity, rt.head_dim, rt.n_atoms))
100    }
101}
102
103/// The installed sparsity `k` (0 if nothing installed).
104pub fn sparsity() -> usize {
105    rt().lock()
106        .ok()
107        .and_then(|g| g.as_ref().map(|r| r.sparsity))
108        .unwrap_or(0)
109}
110
111/// Clone a layer's K (`k_not_v = true`) or V dictionary, or `None` if that layer/stream is passthrough.
112/// The dict-cache write (encode) and read (reconstruct) paths clone once per attention call and work
113/// against the local copy, avoiding a mutex lock per KV vector on the hot loop.
114pub fn clone_layer_dict(layer: usize, k_not_v: bool) -> Option<KvDictionary> {
115    let g = rt().lock().ok()?;
116    let rt = g.as_ref()?;
117    let dicts = if k_not_v { &rt.k } else { &rt.v };
118    dicts.get(layer).cloned().flatten()
119}
120
121/// Disables the KV dictionary runtime globally.
122pub fn disable() {
123    ENABLED.store(false, Ordering::Relaxed);
124}
125
126#[inline]
127/// `true` when enabled.
128pub fn is_enabled() -> bool {
129    ENABLED.load(Ordering::Relaxed)
130}
131
132/// Free the installed dictionaries.
133pub fn clear() {
134    if let Ok(mut g) = rt().lock() {
135        *g = None;
136    }
137}
138
139/// Metadata returned by [`load_certified`] on success — the gate numbers the artifact was certified at.
140#[derive(Debug, Clone)]
141pub struct CertInfo {
142    pub sparsity: usize,
143    pub head_dim: usize,
144    /// Certified ΔPPL fraction (e.g. 0.0065 = +0.65%).
145    pub delta_ppl: f64,
146    /// Layers with an installed K (resp. V) dictionary.
147    pub k_layers: usize,
148    pub v_layers: usize,
149}
150
151/// The frame magic written by the forge packager (`package::FRAME_MAGIC`). Duplicated here so the engine
152/// can read the format without the forge feature; kept in sync with `wgsl_forge::calibration::package`.
153const FRAME_MAGIC: &[u8; 8] = b"QCAL0001";
154
155/// Minimal read-side view of the provenance header — just the fields the engine gates on. Extra fields
156/// in the CBOR map are ignored; `#[serde(default)]` tolerates any this engine build doesn't know.
157#[derive(serde::Deserialize, Default)]
158struct MiniProvenance {
159    #[serde(default)]
160    kind: String,
161    #[serde(default)]
162    delta_ppl: f64,
163    #[serde(default)]
164    passed: bool,
165}
166
167/// Decode a dictionary artifact payload (CBOR [`KvDictArtifact`]) and install it. The payload is the
168/// bytes AFTER the provenance frame header — see [`load_certified`] for the full framed path.
169pub fn install_from_cbor(payload: &[u8]) -> Result<CertInfo, String> {
170    let art: KvDictArtifact =
171        ciborium::from_reader(payload).map_err(|e| format!("KvDictArtifact CBOR: {e}"))?;
172    let info = CertInfo {
173        sparsity: art.sparsity,
174        head_dim: art.head_dim,
175        delta_ppl: f64::NAN, // filled by load_certified from provenance; NaN when installed raw
176        k_layers: art.k.iter().filter(|d| d.is_some()).count(),
177        v_layers: art.v.iter().filter(|d| d.is_some()).count(),
178    };
179    if info.k_layers == 0 && info.v_layers == 0 {
180        return Err("artifact has no dictionaries".into());
181    }
182    enable(art.k, art.v, art.sparsity);
183    Ok(info)
184}
185
186/// Load a certified KV-dictionary artifact from a framed `.q42art` file, **verify its provenance gate**
187/// (kind == KvDictionary AND passed == true), and install it. Fail-closed: a bad frame, wrong artifact
188/// kind, or an artifact that did NOT pass its ΔPPL gate is refused — the engine only runs certified
189/// artifacts. Returns the certified gate numbers on success.
190pub fn load_certified(path: &std::path::Path) -> Result<CertInfo, String> {
191    let bytes = std::fs::read(path).map_err(|e| format!("read {path:?}: {e}"))?;
192    if bytes.len() < 12 || &bytes[..8] != FRAME_MAGIC {
193        return Err("bad frame magic (not a QCAL artifact)".into());
194    }
195    let prov_len = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;
196    let payload_start = 12usize
197        .checked_add(prov_len)
198        .filter(|&e| e <= bytes.len())
199        .ok_or("provenance length out of range")?;
200    let prov: MiniProvenance = ciborium::from_reader(&bytes[12..payload_start])
201        .map_err(|e| format!("provenance CBOR: {e}"))?;
202    if prov.kind != "KvDictionary" {
203        return Err(format!(
204            "not a KV-dictionary artifact (kind={:?})",
205            prov.kind
206        ));
207    }
208    if !prov.passed {
209        return Err("artifact did NOT pass its ΔPPL gate — refusing (fail-closed)".into());
210    }
211    let mut info = install_from_cbor(&bytes[payload_start..])?;
212    info.delta_ppl = prov.delta_ppl;
213    Ok(info)
214}
215
216/// **User switch — ON.** Load a certified dictionary from `path` and turn the dict-coded KV cache on
217/// (`QUALIA_LLM_KV_DICT`). One call for "use the small KV cache". Fail-closed via [`load_certified`].
218/// Take effect at the NEXT model load (the cache layout is chosen then), like the int8 toggle.
219pub fn activate(path: &std::path::Path) -> Result<CertInfo, String> {
220    let info = load_certified(path)?;
221    crate::llm_bench::set_kv_dict(true);
222    Ok(info)
223}
224
225/// **User switch — OFF.** Turn the dict-coded KV cache off and drop the installed dictionaries; the next
226/// model load uses the default f32/int8 cache.
227pub fn deactivate() {
228    crate::llm_bench::set_kv_dict(false);
229    disable();
230    clear();
231}
232
233/// Whether the dict-coded KV cache is currently the active choice: the toggle is on AND a dictionary is
234/// installed. (`QUALIA_LLM_KV_DICT` / [`crate::llm_bench::set_kv_dict`] / this read-back = the 3-way
235/// user switch, mirroring speculative-decode and int8-KV.)
236pub fn dict_active() -> bool {
237    crate::llm_bench::kv_dict_enabled() && installed_meta().is_some()
238}
239
240/// Reconstruct each of the `n_kv` head vectors in `proj` (length ≥ `n_kv * head_dim`) through this
241/// layer's dictionary, in place. No-op (one atomic load) when disabled, when the layer has no
242/// dictionary, or on a head_dim mismatch — so the caller stores the original vector unchanged.
243#[inline]
244pub fn reconstruct_kv(layer: usize, k_not_v: bool, proj: &mut [f32], n_kv: usize, head_dim: usize) {
245    if !ENABLED.load(Ordering::Relaxed) || head_dim == 0 {
246        return;
247    }
248    let Ok(g) = rt().lock() else {
249        return;
250    };
251    let Some(rt) = g.as_ref() else {
252        return;
253    };
254    let dicts = if k_not_v { &rt.k } else { &rt.v };
255    let Some(Some(dict)) = dicts.get(layer) else {
256        return;
257    };
258    if dict.dim != head_dim {
259        return;
260    }
261    for h in 0..n_kv {
262        let s = h * head_dim;
263        if s + head_dim > proj.len() {
264            break;
265        }
266        let code = dict.encode(&proj[s..s + head_dim], rt.sparsity);
267        let recon = dict.reconstruct(&code);
268        proj[s..s + head_dim].copy_from_slice(&recon);
269    }
270}