1use std::collections::HashMap;
12use std::sync::{Mutex, OnceLock};
13
14pub const DEFAULT_MAX_ENTRIES: usize = 16;
16
17pub const DEFAULT_MAX_BYTES: usize = 64 * 1024 * 1024;
19
20#[inline]
22fn mix64(mut val: u64) -> u64 {
23 val ^= val >> 30;
24 val = val.wrapping_mul(0xbf58_476d_1ce4_e5b9);
25 val ^= val >> 27;
26 val = val.wrapping_mul(0x94d0_49bb_1331_11eb);
27 val ^ (val >> 31)
28}
29
30pub fn digest_prompt_tokens(tokens: &[u32]) -> [u64; 4] {
32 let mut words = [
33 0x243f_6a88_85a3_08d3u64,
34 0x1319_8a2e_0370_7344u64,
35 0xa409_3822_299f_31d0u64,
36 0x082e_fa98_ec4e_6c89u64,
37 ];
38 for (index, &token) in tokens.iter().enumerate() {
39 let target = index & 3;
40 let tok_val = (token as u64) ^ ((index as u64) << 32);
41 words[target] = mix64(words[target] ^ tok_val);
42 }
43 words
44}
45
46#[repr(C)]
48#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
49pub struct InferenceCacheKey {
50 pub governed_scope: Option<[u8; 32]>,
53 pub model_instance: u64,
54 pub tokenizer_revision: u64,
55 pub adapter_hash: u64,
56 pub tenant_scope: u64,
57 pub profile_spec_hash: u64,
58 pub prompt_tokens_digest: [u64; 4],
59 pub prompt_token_count: u32,
60 pub graph_context_digest: u64,
61 pub kv_floats: usize,
62 pub n_layers: usize,
63}
64
65impl InferenceCacheKey {
66 pub fn new(
68 model_instance: u64,
69 tokenizer_revision: u64,
70 prompt_tokens: &[u32],
71 graph_context_digest: u64,
72 kv_floats: usize,
73 n_layers: usize,
74 ) -> Self {
75 Self::with_provenance(
76 model_instance,
77 tokenizer_revision,
78 0,
79 0,
80 0,
81 prompt_tokens,
82 graph_context_digest,
83 kv_floats,
84 n_layers,
85 )
86 }
87
88 pub fn with_provenance(
90 model_instance: u64,
91 tokenizer_revision: u64,
92 adapter_hash: u64,
93 tenant_scope: u64,
94 profile_spec_hash: u64,
95 prompt_tokens: &[u32],
96 graph_context_digest: u64,
97 kv_floats: usize,
98 n_layers: usize,
99 ) -> Self {
100 Self {
101 governed_scope: None,
102 model_instance,
103 tokenizer_revision,
104 adapter_hash,
105 tenant_scope,
106 profile_spec_hash,
107 prompt_tokens_digest: digest_prompt_tokens(prompt_tokens),
108 prompt_token_count: prompt_tokens.len() as u32,
109 graph_context_digest,
110 kv_floats,
111 n_layers,
112 }
113 }
114
115 pub fn with_governed_scope(mut self, scope: Option<[u8; 32]>) -> Self {
116 self.governed_scope = scope;
117 self
118 }
119}
120
121#[derive(Clone)]
123pub struct InferenceCacheEntry {
124 pub data: Box<[f32]>,
125 pub token_prefix: Box<[u32]>,
126 pub token_count: u32,
127 pub is_complete: bool,
128 pub last_accessed: u64,
129 pub byte_size: usize,
130}
131
132pub struct BoundedInferenceCache {
134 entries: HashMap<InferenceCacheKey, InferenceCacheEntry>,
135 max_entries: usize,
136 max_bytes: usize,
137 current_bytes: usize,
138 access_clock: u64,
139}
140
141impl BoundedInferenceCache {
142 pub fn new(max_entries: usize, max_bytes: usize) -> Self {
144 Self {
145 entries: HashMap::new(),
146 max_entries: max_entries.max(1),
147 max_bytes: max_bytes.max(1024),
148 current_bytes: 0,
149 access_clock: 0,
150 }
151 }
152
153 pub fn len(&self) -> usize {
155 self.entries.len()
156 }
157
158 pub fn is_empty(&self) -> bool {
160 self.entries.is_empty()
161 }
162
163 pub fn current_bytes(&self) -> usize {
165 self.current_bytes
166 }
167
168 pub fn max_bytes(&self) -> usize {
170 self.max_bytes
171 }
172
173 pub fn restore_if_match<R, F: FnOnce(&[f32]) -> R>(
175 &mut self,
176 key: &InferenceCacheKey,
177 f: F,
178 ) -> Option<R> {
179 if key.prompt_token_count == 0 {
180 return None;
181 }
182 if let Some(entry) = self.entries.get_mut(key) {
183 if entry.is_complete && entry.token_count == key.prompt_token_count {
184 self.access_clock = self.access_clock.wrapping_add(1);
185 entry.last_accessed = self.access_clock;
186 return Some(f(&entry.data));
187 }
188 }
189 None
190 }
191
192 pub fn restore_if_match_exact<R, F: FnOnce(&[f32]) -> R>(
194 &mut self,
195 key: &InferenceCacheKey,
196 exact_prefix: &[u32],
197 f: F,
198 ) -> Option<R> {
199 if key.prompt_token_count == 0 || exact_prefix.len() != key.prompt_token_count as usize {
200 return None;
201 }
202 if let Some(entry) = self.entries.get_mut(key) {
203 if entry.is_complete
204 && entry.token_count == key.prompt_token_count
205 && (entry.token_prefix.is_empty() || entry.token_prefix.as_ref() == exact_prefix)
206 {
207 self.access_clock = self.access_clock.wrapping_add(1);
208 entry.last_accessed = self.access_clock;
209 return Some(f(&entry.data));
210 }
211 }
212 None
213 }
214
215 pub fn insert(
217 &mut self,
218 key: InferenceCacheKey,
219 data: Box<[f32]>,
220 token_count: u32,
221 is_complete: bool,
222 ) -> Result<(), &'static str> {
223 self.insert_with_prefix(key, &[], data, token_count, is_complete)
224 }
225
226 pub fn insert_with_prefix(
228 &mut self,
229 key: InferenceCacheKey,
230 token_prefix: &[u32],
231 data: Box<[f32]>,
232 token_count: u32,
233 is_complete: bool,
234 ) -> Result<(), &'static str> {
235 if !is_complete {
236 return Err("cannot cache incomplete or aborted prefill");
237 }
238 if token_count == 0 || token_count != key.prompt_token_count {
239 return Err("token count mismatch or zero tokens");
240 }
241 if data.len() != key.kv_floats {
242 return Err("KV float slice length does not match key layout");
243 }
244
245 let byte_size = data.len() * std::mem::size_of::<f32>();
246 if byte_size > self.max_bytes {
247 return Err("KV entry size exceeds maximum cache byte ceiling");
248 }
249
250 while self.entries.len() >= self.max_entries
252 || (self.current_bytes + byte_size > self.max_bytes && !self.entries.is_empty())
253 {
254 if !self.evict_lru() {
255 break;
256 }
257 }
258
259 if let Some(old) = self.entries.remove(&key) {
260 self.current_bytes = self.current_bytes.saturating_sub(old.byte_size);
261 }
262
263 self.access_clock = self.access_clock.wrapping_add(1);
264 self.current_bytes += byte_size;
265 self.entries.insert(
266 key,
267 InferenceCacheEntry {
268 data,
269 token_prefix: token_prefix.to_vec().into_boxed_slice(),
270 token_count,
271 is_complete: true,
272 last_accessed: self.access_clock,
273 byte_size,
274 },
275 );
276 Ok(())
277 }
278
279 fn evict_lru(&mut self) -> bool {
281 let mut oldest_key = None;
282 let mut oldest_clock = u64::MAX;
283
284 for (k, v) in &self.entries {
285 if v.last_accessed < oldest_clock {
286 oldest_clock = v.last_accessed;
287 oldest_key = Some(*k);
288 }
289 }
290
291 if let Some(key) = oldest_key {
292 if let Some(removed) = self.entries.remove(&key) {
293 self.current_bytes = self.current_bytes.saturating_sub(removed.byte_size);
294 return true;
295 }
296 }
297 false
298 }
299
300 pub fn clear(&mut self) {
302 self.entries.clear();
303 self.current_bytes = 0;
304 }
305}
306
307static PREFIX_CACHE: OnceLock<Mutex<BoundedInferenceCache>> = OnceLock::new();
308
309pub fn get_prefix_cache() -> &'static Mutex<BoundedInferenceCache> {
311 PREFIX_CACHE.get_or_init(|| {
312 Mutex::new(BoundedInferenceCache::new(
313 DEFAULT_MAX_ENTRIES,
314 DEFAULT_MAX_BYTES,
315 ))
316 })
317}
318
319#[cfg(test)]
320mod tests {
321 use super::*;
322
323 #[test]
324 fn test_cache_key_different_prompts_same_context() {
325 let model = 0x1122_3344;
326 let tok_rev = 1;
327 let ctx_hash = 0x9988_7766;
328 let kv_floats = 1024;
329 let n_layers = 12;
330
331 let tokens_a = [101, 2054, 2003, 1037, 3899];
332 let tokens_b = [101, 2054, 2003, 1037, 7072]; let key_a =
335 InferenceCacheKey::new(model, tok_rev, &tokens_a, ctx_hash, kv_floats, n_layers);
336 let key_b =
337 InferenceCacheKey::new(model, tok_rev, &tokens_b, ctx_hash, kv_floats, n_layers);
338
339 assert_ne!(key_a, key_b);
340 assert_ne!(key_a.prompt_tokens_digest, key_b.prompt_tokens_digest);
341 }
342
343 #[test]
344 fn test_cache_key_different_graph_context_distinct() {
345 let model = 0x1122_3344;
346 let tok_rev = 1;
347 let tokens = [101, 2054];
348 let kv_floats = 512;
349 let n_layers = 8;
350
351 let ctx_1 = crate::q_hash("abcdefgh-1");
352 let ctx_2 = crate::q_hash("abcdefgh-2");
353
354 let key_1 = InferenceCacheKey::new(model, tok_rev, &tokens, ctx_1, kv_floats, n_layers);
355 let key_2 = InferenceCacheKey::new(model, tok_rev, &tokens, ctx_2, kv_floats, n_layers);
356
357 assert_ne!(key_1, key_2);
358 assert_ne!(key_1.graph_context_digest, key_2.graph_context_digest);
359 }
360
361 #[test]
362 fn test_incomplete_prefill_rejected_from_cache() {
363 let mut cache = BoundedInferenceCache::new(4, 1024 * 1024);
364 let key = InferenceCacheKey::new(1, 1, &[10, 20, 30], 100, 64, 2);
365 let dummy_kv = vec![0.5f32; 64].into_boxed_slice();
366
367 let res = cache.insert(key, dummy_kv, 3, false);
369 assert!(res.is_err());
370 assert_eq!(cache.len(), 0);
371
372 let restored = cache.restore_if_match(&key, |data| data[0]);
374 assert_eq!(restored, None);
375 }
376
377 #[test]
378 fn test_lru_eviction_when_capacity_reached() {
379 let mut cache = BoundedInferenceCache::new(2, 1024 * 1024); let key1 = InferenceCacheKey::new(1, 1, &[1, 2], 100, 32, 2);
382 let key2 = InferenceCacheKey::new(1, 1, &[3, 4], 100, 32, 2);
383 let key3 = InferenceCacheKey::new(1, 1, &[5, 6], 100, 32, 2);
384
385 let kv1 = vec![1.0f32; 32].into_boxed_slice();
386 let kv2 = vec![2.0f32; 32].into_boxed_slice();
387 let kv3 = vec![3.0f32; 32].into_boxed_slice();
388
389 assert!(cache.insert(key1, kv1, 2, true).is_ok());
390 assert!(cache.insert(key2, kv2, 2, true).is_ok());
391 assert_eq!(cache.len(), 2);
392
393 let val1 = cache.restore_if_match(&key1, |d| d[0]);
395 assert_eq!(val1, Some(1.0));
396
397 assert!(cache.insert(key3, kv3, 2, true).is_ok());
399 assert_eq!(cache.len(), 2);
400
401 assert!(cache.restore_if_match(&key1, |_| ()).is_some());
403 assert!(cache.restore_if_match(&key3, |_| ()).is_some());
404 assert!(cache.restore_if_match(&key2, |_| ()).is_none());
405 }
406
407 #[test]
408 fn test_byte_ceiling_eviction() {
409 let max_bytes = 256 * std::mem::size_of::<f32>();
411 let mut cache = BoundedInferenceCache::new(10, max_bytes);
412
413 let key1 = InferenceCacheKey::new(1, 1, &[1], 100, 200, 2);
414 let key2 = InferenceCacheKey::new(1, 1, &[2], 100, 100, 2);
415
416 let kv1 = vec![1.0f32; 200].into_boxed_slice();
417 let kv2 = vec![2.0f32; 100].into_boxed_slice();
418
419 assert!(cache.insert(key1, kv1, 1, true).is_ok());
420 assert_eq!(cache.current_bytes(), 200 * 4);
421
422 assert!(cache.insert(key2, kv2, 1, true).is_ok());
425 assert_eq!(cache.len(), 1);
426 assert_eq!(cache.current_bytes(), 100 * 4);
427 assert!(cache.restore_if_match(&key1, |_| ()).is_none());
428 assert!(cache.restore_if_match(&key2, |_| ()).is_some());
429 }
430}