Skip to main content

qualia_core_db/inference/inference_agent/
prefix_cache.rs

1//! Typed, bounded prefix KV cache for the QualiaDB local inference engine.
2//!
3//! Replaces legacy unsafe 8-byte hash keying with an exact compatibility contract:
4//! - Model instance & tokenizer identity
5//! - Exact token sequence digest & token length
6//! - Full graph-context digest (not truncated to 8 bytes)
7//! - KV cache geometry & layer count
8//! - Strict completion flag (incomplete/cancelled prefill is never cached or restored)
9//! - Bounded capacity with deterministic LRU eviction conforming to the Sentinel ceiling
10
11use std::collections::HashMap;
12use std::sync::{Mutex, OnceLock};
13
14/// Default maximum number of cached prompt KV sequences.
15pub const DEFAULT_MAX_ENTRIES: usize = 16;
16
17/// Default maximum memory ceiling for cached KV floats (64 MiB).
18pub const DEFAULT_MAX_BYTES: usize = 64 * 1024 * 1024;
19
20/// 256-bit mixer for token ID sequences to guarantee zero collision in the cache key.
21#[inline]
22fn mix64(mut val: u64) -> u64 {
23    val ^= val >> 30;
24    val = val.wrapping_mul(0xbf58_476d_1ce4_e5b9);
25    val ^= val >> 27;
26    val = val.wrapping_mul(0x94d0_49bb_1331_11eb);
27    val ^ (val >> 31)
28}
29
30/// Compute a 256-bit collision-resistant digest over a slice of prompt token IDs.
31pub fn digest_prompt_tokens(tokens: &[u32]) -> [u64; 4] {
32    let mut words = [
33        0x243f_6a88_85a3_08d3u64,
34        0x1319_8a2e_0370_7344u64,
35        0xa409_3822_299f_31d0u64,
36        0x082e_fa98_ec4e_6c89u64,
37    ];
38    for (index, &token) in tokens.iter().enumerate() {
39        let target = index & 3;
40        let tok_val = (token as u64) ^ ((index as u64) << 32);
41        words[target] = mix64(words[target] ^ tok_val);
42    }
43    words
44}
45
46/// Strongly-typed cache key requiring exact compatibility across model, tokens, graph, and tensor geometry.
47#[repr(C)]
48#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
49pub struct InferenceCacheKey {
50    /// Full SHA-256 scope from the real governed admission. Developer caches
51    /// never collide with an admitted model/request/actor/policy/device scope.
52    pub governed_scope: Option<[u8; 32]>,
53    pub model_instance: u64,
54    pub tokenizer_revision: u64,
55    pub adapter_hash: u64,
56    pub tenant_scope: u64,
57    pub profile_spec_hash: u64,
58    pub prompt_tokens_digest: [u64; 4],
59    pub prompt_token_count: u32,
60    pub graph_context_digest: u64,
61    pub kv_floats: usize,
62    pub n_layers: usize,
63}
64
65impl InferenceCacheKey {
66    /// Creates a default `InferenceCacheKey`.
67    pub fn new(
68        model_instance: u64,
69        tokenizer_revision: u64,
70        prompt_tokens: &[u32],
71        graph_context_digest: u64,
72        kv_floats: usize,
73        n_layers: usize,
74    ) -> Self {
75        Self::with_provenance(
76            model_instance,
77            tokenizer_revision,
78            0,
79            0,
80            0,
81            prompt_tokens,
82            graph_context_digest,
83            kv_floats,
84            n_layers,
85        )
86    }
87
88    /// Builder: sets `provenance`.
89    pub fn with_provenance(
90        model_instance: u64,
91        tokenizer_revision: u64,
92        adapter_hash: u64,
93        tenant_scope: u64,
94        profile_spec_hash: u64,
95        prompt_tokens: &[u32],
96        graph_context_digest: u64,
97        kv_floats: usize,
98        n_layers: usize,
99    ) -> Self {
100        Self {
101            governed_scope: None,
102            model_instance,
103            tokenizer_revision,
104            adapter_hash,
105            tenant_scope,
106            profile_spec_hash,
107            prompt_tokens_digest: digest_prompt_tokens(prompt_tokens),
108            prompt_token_count: prompt_tokens.len() as u32,
109            graph_context_digest,
110            kv_floats,
111            n_layers,
112        }
113    }
114
115    pub fn with_governed_scope(mut self, scope: Option<[u8; 32]>) -> Self {
116        self.governed_scope = scope;
117        self
118    }
119}
120
121/// Cached KV data entry with metadata and completion invariant.
122#[derive(Clone)]
123pub struct InferenceCacheEntry {
124    pub data: Box<[f32]>,
125    pub token_prefix: Box<[u32]>,
126    pub token_count: u32,
127    pub is_complete: bool,
128    pub last_accessed: u64,
129    pub byte_size: usize,
130}
131
132/// Bounded in-memory KV cache with capacity limits and deterministic LRU eviction.
133pub struct BoundedInferenceCache {
134    entries: HashMap<InferenceCacheKey, InferenceCacheEntry>,
135    max_entries: usize,
136    max_bytes: usize,
137    current_bytes: usize,
138    access_clock: u64,
139}
140
141impl BoundedInferenceCache {
142    /// Creates a default `BoundedInferenceCache`.
143    pub fn new(max_entries: usize, max_bytes: usize) -> Self {
144        Self {
145            entries: HashMap::new(),
146            max_entries: max_entries.max(1),
147            max_bytes: max_bytes.max(1024),
148            current_bytes: 0,
149            access_clock: 0,
150        }
151    }
152
153    /// Number of active cache entries.
154    pub fn len(&self) -> usize {
155        self.entries.len()
156    }
157
158    /// `true` when empty.
159    pub fn is_empty(&self) -> bool {
160        self.entries.is_empty()
161    }
162
163    /// Total bytes consumed by stored KV cache float buffers.
164    pub fn current_bytes(&self) -> usize {
165        self.current_bytes
166    }
167
168    /// Maximum byte ceiling for this cache.
169    pub fn max_bytes(&self) -> usize {
170        self.max_bytes
171    }
172
173    /// Restore cached KV slice via a closure without cloning if the key matches and is complete.
174    pub fn restore_if_match<R, F: FnOnce(&[f32]) -> R>(
175        &mut self,
176        key: &InferenceCacheKey,
177        f: F,
178    ) -> Option<R> {
179        if key.prompt_token_count == 0 {
180            return None;
181        }
182        if let Some(entry) = self.entries.get_mut(key) {
183            if entry.is_complete && entry.token_count == key.prompt_token_count {
184                self.access_clock = self.access_clock.wrapping_add(1);
185                entry.last_accessed = self.access_clock;
186                return Some(f(&entry.data));
187            }
188        }
189        None
190    }
191
192    /// Restore cached KV slice verifying exact token prefix match to eliminate any hash collision risk.
193    pub fn restore_if_match_exact<R, F: FnOnce(&[f32]) -> R>(
194        &mut self,
195        key: &InferenceCacheKey,
196        exact_prefix: &[u32],
197        f: F,
198    ) -> Option<R> {
199        if key.prompt_token_count == 0 || exact_prefix.len() != key.prompt_token_count as usize {
200            return None;
201        }
202        if let Some(entry) = self.entries.get_mut(key) {
203            if entry.is_complete
204                && entry.token_count == key.prompt_token_count
205                && (entry.token_prefix.is_empty() || entry.token_prefix.as_ref() == exact_prefix)
206            {
207                self.access_clock = self.access_clock.wrapping_add(1);
208                entry.last_accessed = self.access_clock;
209                return Some(f(&entry.data));
210            }
211        }
212        None
213    }
214
215    /// Insert completed prefill KV data without token prefix slice.
216    pub fn insert(
217        &mut self,
218        key: InferenceCacheKey,
219        data: Box<[f32]>,
220        token_count: u32,
221        is_complete: bool,
222    ) -> Result<(), &'static str> {
223        self.insert_with_prefix(key, &[], data, token_count, is_complete)
224    }
225
226    /// Insert completed prefill KV data with exact token prefix slice. Incomplete or empty data is strictly rejected.
227    pub fn insert_with_prefix(
228        &mut self,
229        key: InferenceCacheKey,
230        token_prefix: &[u32],
231        data: Box<[f32]>,
232        token_count: u32,
233        is_complete: bool,
234    ) -> Result<(), &'static str> {
235        if !is_complete {
236            return Err("cannot cache incomplete or aborted prefill");
237        }
238        if token_count == 0 || token_count != key.prompt_token_count {
239            return Err("token count mismatch or zero tokens");
240        }
241        if data.len() != key.kv_floats {
242            return Err("KV float slice length does not match key layout");
243        }
244
245        let byte_size = data.len() * std::mem::size_of::<f32>();
246        if byte_size > self.max_bytes {
247            return Err("KV entry size exceeds maximum cache byte ceiling");
248        }
249
250        // Evict until both entry limit and byte budget are satisfied
251        while self.entries.len() >= self.max_entries
252            || (self.current_bytes + byte_size > self.max_bytes && !self.entries.is_empty())
253        {
254            if !self.evict_lru() {
255                break;
256            }
257        }
258
259        if let Some(old) = self.entries.remove(&key) {
260            self.current_bytes = self.current_bytes.saturating_sub(old.byte_size);
261        }
262
263        self.access_clock = self.access_clock.wrapping_add(1);
264        self.current_bytes += byte_size;
265        self.entries.insert(
266            key,
267            InferenceCacheEntry {
268                data,
269                token_prefix: token_prefix.to_vec().into_boxed_slice(),
270                token_count,
271                is_complete: true,
272                last_accessed: self.access_clock,
273                byte_size,
274            },
275        );
276        Ok(())
277    }
278
279    /// Evict the least recently accessed entry. Returns true if an entry was removed.
280    fn evict_lru(&mut self) -> bool {
281        let mut oldest_key = None;
282        let mut oldest_clock = u64::MAX;
283
284        for (k, v) in &self.entries {
285            if v.last_accessed < oldest_clock {
286                oldest_clock = v.last_accessed;
287                oldest_key = Some(*k);
288            }
289        }
290
291        if let Some(key) = oldest_key {
292            if let Some(removed) = self.entries.remove(&key) {
293                self.current_bytes = self.current_bytes.saturating_sub(removed.byte_size);
294                return true;
295            }
296        }
297        false
298    }
299
300    /// Clear all cached entries and reset byte counter.
301    pub fn clear(&mut self) {
302        self.entries.clear();
303        self.current_bytes = 0;
304    }
305}
306
307static PREFIX_CACHE: OnceLock<Mutex<BoundedInferenceCache>> = OnceLock::new();
308
309/// Access the process-wide bounded inference cache.
310pub fn get_prefix_cache() -> &'static Mutex<BoundedInferenceCache> {
311    PREFIX_CACHE.get_or_init(|| {
312        Mutex::new(BoundedInferenceCache::new(
313            DEFAULT_MAX_ENTRIES,
314            DEFAULT_MAX_BYTES,
315        ))
316    })
317}
318
319#[cfg(test)]
320mod tests {
321    use super::*;
322
323    #[test]
324    fn test_cache_key_different_prompts_same_context() {
325        let model = 0x1122_3344;
326        let tok_rev = 1;
327        let ctx_hash = 0x9988_7766;
328        let kv_floats = 1024;
329        let n_layers = 12;
330
331        let tokens_a = [101, 2054, 2003, 1037, 3899];
332        let tokens_b = [101, 2054, 2003, 1037, 7072]; // different last token
333
334        let key_a =
335            InferenceCacheKey::new(model, tok_rev, &tokens_a, ctx_hash, kv_floats, n_layers);
336        let key_b =
337            InferenceCacheKey::new(model, tok_rev, &tokens_b, ctx_hash, kv_floats, n_layers);
338
339        assert_ne!(key_a, key_b);
340        assert_ne!(key_a.prompt_tokens_digest, key_b.prompt_tokens_digest);
341    }
342
343    #[test]
344    fn test_cache_key_different_graph_context_distinct() {
345        let model = 0x1122_3344;
346        let tok_rev = 1;
347        let tokens = [101, 2054];
348        let kv_floats = 512;
349        let n_layers = 8;
350
351        let ctx_1 = crate::q_hash("abcdefgh-1");
352        let ctx_2 = crate::q_hash("abcdefgh-2");
353
354        let key_1 = InferenceCacheKey::new(model, tok_rev, &tokens, ctx_1, kv_floats, n_layers);
355        let key_2 = InferenceCacheKey::new(model, tok_rev, &tokens, ctx_2, kv_floats, n_layers);
356
357        assert_ne!(key_1, key_2);
358        assert_ne!(key_1.graph_context_digest, key_2.graph_context_digest);
359    }
360
361    #[test]
362    fn test_incomplete_prefill_rejected_from_cache() {
363        let mut cache = BoundedInferenceCache::new(4, 1024 * 1024);
364        let key = InferenceCacheKey::new(1, 1, &[10, 20, 30], 100, 64, 2);
365        let dummy_kv = vec![0.5f32; 64].into_boxed_slice();
366
367        // Attempting to insert incomplete prefill must error
368        let res = cache.insert(key, dummy_kv, 3, false);
369        assert!(res.is_err());
370        assert_eq!(cache.len(), 0);
371
372        // Lookup must yield None
373        let restored = cache.restore_if_match(&key, |data| data[0]);
374        assert_eq!(restored, None);
375    }
376
377    #[test]
378    fn test_lru_eviction_when_capacity_reached() {
379        let mut cache = BoundedInferenceCache::new(2, 1024 * 1024); // max 2 entries
380
381        let key1 = InferenceCacheKey::new(1, 1, &[1, 2], 100, 32, 2);
382        let key2 = InferenceCacheKey::new(1, 1, &[3, 4], 100, 32, 2);
383        let key3 = InferenceCacheKey::new(1, 1, &[5, 6], 100, 32, 2);
384
385        let kv1 = vec![1.0f32; 32].into_boxed_slice();
386        let kv2 = vec![2.0f32; 32].into_boxed_slice();
387        let kv3 = vec![3.0f32; 32].into_boxed_slice();
388
389        assert!(cache.insert(key1, kv1, 2, true).is_ok());
390        assert!(cache.insert(key2, kv2, 2, true).is_ok());
391        assert_eq!(cache.len(), 2);
392
393        // Access key1 to make key2 the LRU
394        let val1 = cache.restore_if_match(&key1, |d| d[0]);
395        assert_eq!(val1, Some(1.0));
396
397        // Inserting key3 should evict key2
398        assert!(cache.insert(key3, kv3, 2, true).is_ok());
399        assert_eq!(cache.len(), 2);
400
401        // key1 and key3 should still be in cache; key2 should be gone
402        assert!(cache.restore_if_match(&key1, |_| ()).is_some());
403        assert!(cache.restore_if_match(&key3, |_| ()).is_some());
404        assert!(cache.restore_if_match(&key2, |_| ()).is_none());
405    }
406
407    #[test]
408    fn test_byte_ceiling_eviction() {
409        // Limit total bytes to 256 floats (1024 bytes)
410        let max_bytes = 256 * std::mem::size_of::<f32>();
411        let mut cache = BoundedInferenceCache::new(10, max_bytes);
412
413        let key1 = InferenceCacheKey::new(1, 1, &[1], 100, 200, 2);
414        let key2 = InferenceCacheKey::new(1, 1, &[2], 100, 100, 2);
415
416        let kv1 = vec![1.0f32; 200].into_boxed_slice();
417        let kv2 = vec![2.0f32; 100].into_boxed_slice();
418
419        assert!(cache.insert(key1, kv1, 1, true).is_ok());
420        assert_eq!(cache.current_bytes(), 200 * 4);
421
422        // Inserting kv2 (100 floats = 400 bytes) + existing 800 bytes = 1200 > 1024
423        // Should evict key1 to stay under ceiling
424        assert!(cache.insert(key2, kv2, 1, true).is_ok());
425        assert_eq!(cache.len(), 1);
426        assert_eq!(cache.current_bytes(), 100 * 4);
427        assert!(cache.restore_if_match(&key1, |_| ()).is_none());
428        assert!(cache.restore_if_match(&key2, |_| ()).is_some());
429    }
430}