Skip to main content

qualia_core_db/inference/
semantic_skills.rs

1//! A9 — Semantic Skills: Vectors, Embeddings & Scratchpads.
2//!
3//! First-class vector cosine distance, in-process text embedding, semantic
4//! search, and ephemeral scratchpad memory for agent workflows.
5//!
6//! ## Design
7//!
8//! - **Vector**: Fixed-size f32 vector with cosine distance.
9//! - **VectorStore**: A bounded collection of vectors with semantic search
10//!   (k-nearest neighbors by cosine similarity).
11//! - **TextEmbedder**: In-process text embedding using FNV-1a hashing into
12//!  a fixed-dimensional vector. This is a lightweight embedding suitable for
13//!  semantic deduplication and approximate search — not a replacement for
14//!  neural embeddings.
15//! - **Scratchpad**: Ephemeral key-value memory for agent workflows. Bounded,
16//!  with TTL-based expiry.
17//!
18//! ## Integration
19//!
20//! - Uses A5 (`blackboard`) for observable state channels.
21//! - Designed for zero-heap on hot paths (search uses caller-supplied buffers).
22
23use std::collections::HashMap;
24
25// ── Vector ─────────────────────────────────────────────────────────────────
26
27/// Default embedding dimensionality.
28pub const EMBED_DIM: usize = 256;
29/// Maximum vectors in a store.
30pub const MAX_VECTORS: usize = 4096;
31/// Maximum scratchpad entries.
32pub const MAX_SCRATCHPAD_ENTRIES: usize = 256;
33/// Default scratchpad TTL (in ticks).
34pub const DEFAULT_SCRATCHPAD_TTL: u32 = 1000;
35
36/// A fixed-size vector.
37#[derive(Debug, Clone, PartialEq)]
38pub struct Vector {
39    pub dims: Vec<f32>,
40}
41
42impl Vector {
43    /// Vector over an existing dimension buffer.
44    pub fn new(dims: Vec<f32>) -> Self {
45        Self { dims }
46    }
47
48    /// Zero vector of `dim` dimensions.
49    pub fn zeros(dim: usize) -> Self {
50        Self {
51            dims: vec![0.0; dim],
52        }
53    }
54
55    /// Vector dimensionality.
56    pub fn dim(&self) -> usize {
57        self.dims.len()
58    }
59
60    /// Compute the L2 norm.
61    pub fn norm(&self) -> f32 {
62        self.dims.iter().map(|v| v * v).sum::<f32>().sqrt()
63    }
64
65    /// Compute the dot product with another vector.
66    pub fn dot(&self, other: &Vector) -> f32 {
67        self.dims
68            .iter()
69            .zip(other.dims.iter())
70            .map(|(a, b)| a * b)
71            .sum()
72    }
73
74    /// Compute the cosine similarity with another vector.
75    /// Returns 0.0 if either vector has zero norm.
76    pub fn cosine_similarity(&self, other: &Vector) -> f32 {
77        let norm_product = self.norm() * other.norm();
78        if norm_product == 0.0 {
79            return 0.0;
80        }
81        self.dot(other) / norm_product
82    }
83
84    /// Compute the cosine distance (1 - similarity).
85    pub fn cosine_distance(&self, other: &Vector) -> f32 {
86        1.0 - self.cosine_similarity(other)
87    }
88
89    /// Normalize the vector in place.
90    pub fn normalize(&mut self) {
91        let norm = self.norm();
92        if norm > 0.0 {
93            for v in &mut self.dims {
94                *v /= norm;
95            }
96        }
97    }
98}
99
100// ── Text Embedder ──────────────────────────────────────────────────────────
101
102/// A lightweight in-process text embedder using FNV-1a hashing.
103/// Maps text into a fixed-dimensional vector by hashing character n-grams
104/// into vector dimensions. This is suitable for semantic deduplication and
105/// approximate search, not as a replacement for neural embeddings.
106pub struct TextEmbedder {
107    dim: usize,
108    ngram_size: usize,
109}
110
111impl TextEmbedder {
112    /// Text embedder producing `dim`-d vectors from `ngram_size`
113    /// character n-grams.
114    pub fn new(dim: usize, ngram_size: usize) -> Self {
115        Self { dim, ngram_size }
116    }
117
118    /// Default embedder (`EMBED_DIM`, trigram n-grams).
119    pub fn default() -> Self {
120        Self::new(EMBED_DIM, 3)
121    }
122
123    /// Embed text into a fixed-dimensional vector.
124    /// Uses character n-gram hashing: each n-gram is hashed and contributes
125    /// to a dimension. Positive contributions for presence, weighted by
126    /// frequency.
127    pub fn embed(&self, text: &str) -> Vector {
128        let mut dims = vec![0.0f32; self.dim];
129        let chars: Vec<char> = text.to_lowercase().chars().collect();
130        if chars.len() < self.ngram_size {
131            // For very short text, hash the whole string.
132            let h = fnv1a_hash(text.as_bytes());
133            dims[(h as usize) % self.dim] += 1.0;
134            return Vector { dims };
135        }
136        for i in 0..=chars.len() - self.ngram_size {
137            let ngram: String = chars[i..i + self.ngram_size].iter().collect();
138            let h = fnv1a_hash(ngram.as_bytes());
139            let idx = (h as usize) % self.dim;
140            dims[idx] += 1.0;
141        }
142        let mut v = Vector { dims };
143        v.normalize();
144        v
145    }
146}
147
148/// FNV-1a hash (32-bit).
149fn fnv1a_hash(data: &[u8]) -> u32 {
150    let mut hash: u32 = 0x811C_9DC5;
151    for &b in data {
152        hash ^= b as u32;
153        hash = hash.wrapping_mul(0x0100_0193);
154    }
155    hash
156}
157
158// ── Vector Store ───────────────────────────────────────────────────────────
159
160/// A stored vector with an associated key and optional metadata.
161#[derive(Debug, Clone)]
162pub struct StoredVector {
163    pub key: String,
164    pub vector: Vector,
165    pub metadata: Option<String>,
166}
167
168/// A bounded vector store with semantic search.
169pub struct VectorStore {
170    vectors: Vec<StoredVector>,
171    embedder: TextEmbedder,
172}
173
174impl VectorStore {
175    /// Empty vector store (default embedder).
176    pub fn new() -> Self {
177        Self {
178            vectors: Vec::new(),
179            embedder: TextEmbedder::default(),
180        }
181    }
182
183    /// Vector store using a custom `embedder`.
184    pub fn with_embedder(embedder: TextEmbedder) -> Self {
185        Self {
186            vectors: Vec::new(),
187            embedder,
188        }
189    }
190
191    /// Add a vector to the store. Returns false if the store is full.
192    pub fn add(&mut self, key: &str, vector: Vector, metadata: Option<String>) -> bool {
193        if self.vectors.len() >= MAX_VECTORS {
194            return false;
195        }
196        self.vectors.push(StoredVector {
197            key: key.to_string(),
198            vector,
199            metadata,
200        });
201        true
202    }
203
204    /// Add text to the store by embedding it.
205    pub fn add_text(&mut self, key: &str, text: &str, metadata: Option<String>) -> bool {
206        let vector = self.embedder.embed(text);
207        self.add(key, vector, metadata)
208    }
209
210    /// Search for the k nearest neighbors by cosine similarity.
211    /// Returns results sorted by descending similarity.
212    pub fn search(&self, query: &Vector, k: usize) -> Vec<SearchResult> {
213        let mut results: Vec<SearchResult> = self
214            .vectors
215            .iter()
216            .map(|sv| SearchResult {
217                key: sv.key.clone(),
218                similarity: query.cosine_similarity(&sv.vector),
219                metadata: sv.metadata.clone(),
220            })
221            .collect();
222        results.sort_by(|a, b| {
223            b.similarity
224                .partial_cmp(&a.similarity)
225                .unwrap_or(std::cmp::Ordering::Equal)
226        });
227        results.truncate(k);
228        results
229    }
230
231    /// Search by text (embeds the query first).
232    pub fn search_text(&self, query_text: &str, k: usize) -> Vec<SearchResult> {
233        let query = self.embedder.embed(query_text);
234        self.search(&query, k)
235    }
236
237    /// Get a vector by key.
238    pub fn get(&self, key: &str) -> Option<&StoredVector> {
239        self.vectors.iter().find(|v| v.key == key)
240    }
241
242    /// Remove a vector by key.
243    pub fn remove(&mut self, key: &str) -> bool {
244        let len_before = self.vectors.len();
245        self.vectors.retain(|v| v.key != key);
246        self.vectors.len() != len_before
247    }
248
249    /// Number of stored vectors.
250    pub fn len(&self) -> usize {
251        self.vectors.len()
252    }
253
254    /// Is the store empty?
255    pub fn is_empty(&self) -> bool {
256        self.vectors.is_empty()
257    }
258
259    /// Clear all vectors.
260    pub fn clear(&mut self) {
261        self.vectors.clear();
262    }
263}
264
265impl Default for VectorStore {
266    /// Default is the empty store.
267    fn default() -> Self {
268        Self::new()
269    }
270}
271
272/// A search result from the vector store.
273#[derive(Debug, Clone)]
274pub struct SearchResult {
275    pub key: String,
276    pub similarity: f32,
277    pub metadata: Option<String>,
278}
279
280// ── Scratchpad ─────────────────────────────────────────────────────────────
281
282/// An ephemeral scratchpad entry with TTL.
283#[derive(Debug, Clone)]
284pub struct ScratchpadEntry {
285    pub value: String,
286    pub tick: u32,
287    pub ttl: u32,
288}
289
290impl ScratchpadEntry {
291    /// Scratchpad entry expiring `ttl` ticks after `tick`.
292    pub fn new(value: String, tick: u32, ttl: u32) -> Self {
293        Self { value, tick, ttl }
294    }
295
296    /// Is this entry expired at the given tick?
297    pub fn is_expired(&self, current_tick: u32) -> bool {
298        current_tick > self.tick + self.ttl
299    }
300}
301
302/// An ephemeral key-value scratchpad with TTL-based expiry.
303/// Designed for agent workflows that need temporary memory between steps.
304pub struct Scratchpad {
305    entries: HashMap<String, ScratchpadEntry>,
306    default_ttl: u32,
307    current_tick: u32,
308}
309
310impl Scratchpad {
311    /// Scratchpad where entries expire after `default_ttl` ticks.
312    pub fn new(default_ttl: u32) -> Self {
313        Self {
314            entries: HashMap::new(),
315            default_ttl,
316            current_tick: 0,
317        }
318    }
319
320    /// Default scratchpad (default TTL).
321    pub fn default() -> Self {
322        Self::new(DEFAULT_SCRATCHPAD_TTL)
323    }
324
325    /// Advance the tick counter.
326    pub fn tick(&mut self) {
327        self.current_tick += 1;
328        // Evict expired entries.
329        let tick = self.current_tick;
330        self.entries.retain(|_, e| !e.is_expired(tick));
331    }
332
333    /// Set a key-value pair with the default TTL.
334    pub fn set(&mut self, key: &str, value: &str) -> Result<(), ScratchpadError> {
335        self.set_with_ttl(key, value, self.default_ttl)
336    }
337
338    /// Set a key-value pair with a custom TTL.
339    pub fn set_with_ttl(
340        &mut self,
341        key: &str,
342        value: &str,
343        ttl: u32,
344    ) -> Result<(), ScratchpadError> {
345        if self.entries.len() >= MAX_SCRATCHPAD_ENTRIES && !self.entries.contains_key(key) {
346            return Err(ScratchpadError::Full);
347        }
348        self.entries.insert(
349            key.to_string(),
350            ScratchpadEntry::new(value.to_string(), self.current_tick, ttl),
351        );
352        Ok(())
353    }
354
355    /// Get a value by key. Returns None if the key doesn't exist or is expired.
356    pub fn get(&self, key: &str) -> Option<&str> {
357        let entry = self.entries.get(key)?;
358        if entry.is_expired(self.current_tick) {
359            return None;
360        }
361        Some(entry.value.as_str())
362    }
363
364    /// Remove a key.
365    pub fn remove(&mut self, key: &str) -> bool {
366        self.entries.remove(key).is_some()
367    }
368
369    /// Clear all entries.
370    pub fn clear(&mut self) {
371        self.entries.clear();
372    }
373
374    /// Number of entries (including expired ones not yet evicted).
375    pub fn len(&self) -> usize {
376        self.entries.len()
377    }
378
379    /// Is the scratchpad empty?
380    pub fn is_empty(&self) -> bool {
381        self.entries.is_empty()
382    }
383
384    /// Get the current tick.
385    pub fn current_tick(&self) -> u32 {
386        self.current_tick
387    }
388
389    /// Evict all expired entries immediately.
390    pub fn evict_expired(&mut self) -> usize {
391        let tick = self.current_tick;
392        let len_before = self.entries.len();
393        self.entries.retain(|_, e| !e.is_expired(tick));
394        len_before - self.entries.len()
395    }
396}
397
398/// Errors from the scratchpad.
399#[derive(Debug, Clone, PartialEq, Eq)]
400pub enum ScratchpadError {
401    /// The scratchpad is full.
402    Full,
403}
404
405// ── Tests ──────────────────────────────────────────────────────────────────
406
407#[cfg(test)]
408mod tests {
409    use super::*;
410
411    #[test]
412    fn vector_cosine_similarity_identical() {
413        let v = Vector::new(vec![1.0, 2.0, 3.0]);
414        let sim = v.cosine_similarity(&v);
415        assert!((sim - 1.0).abs() < 1e-5);
416    }
417
418    #[test]
419    fn vector_cosine_similarity_orthogonal() {
420        let v1 = Vector::new(vec![1.0, 0.0]);
421        let v2 = Vector::new(vec![0.0, 1.0]);
422        let sim = v1.cosine_similarity(&v2);
423        assert!(sim.abs() < 1e-5);
424    }
425
426    #[test]
427    fn vector_cosine_similarity_opposite() {
428        let v1 = Vector::new(vec![1.0, 0.0]);
429        let v2 = Vector::new(vec![-1.0, 0.0]);
430        let sim = v1.cosine_similarity(&v2);
431        assert!((sim - (-1.0)).abs() < 1e-5);
432    }
433
434    #[test]
435    fn vector_cosine_distance() {
436        let v1 = Vector::new(vec![1.0, 0.0]);
437        let v2 = Vector::new(vec![1.0, 0.0]);
438        let dist = v1.cosine_distance(&v2);
439        assert!(dist.abs() < 1e-5);
440    }
441
442    #[test]
443    fn vector_normalize() {
444        let mut v = Vector::new(vec![3.0, 4.0]);
445        v.normalize();
446        assert!((v.norm() - 1.0).abs() < 1e-5);
447    }
448
449    #[test]
450    fn vector_zero_norm_similarity() {
451        let v1 = Vector::new(vec![0.0, 0.0]);
452        let v2 = Vector::new(vec![1.0, 0.0]);
453        let sim = v1.cosine_similarity(&v2);
454        assert_eq!(sim, 0.0);
455    }
456
457    #[test]
458    fn text_embedder_basic() {
459        let embedder = TextEmbedder::default();
460        let v1 = embedder.embed("hello world");
461        let v2 = embedder.embed("hello world");
462        assert_eq!(v1, v2);
463        assert_eq!(v1.dim(), EMBED_DIM);
464    }
465
466    #[test]
467    fn text_embedder_similar_texts() {
468        let embedder = TextEmbedder::default();
469        let v1 = embedder.embed("the quick brown fox");
470        let v2 = embedder.embed("the quick brown fox");
471        let v3 = embedder.embed("completely different text");
472        let sim_same = v1.cosine_similarity(&v2);
473        let sim_diff = v1.cosine_similarity(&v3);
474        assert!(sim_same > sim_diff);
475    }
476
477    #[test]
478    fn text_embedder_normalizes() {
479        let embedder = TextEmbedder::default();
480        let v = embedder.embed("test text");
481        assert!((v.norm() - 1.0).abs() < 1e-5);
482    }
483
484    #[test]
485    fn vector_store_add_and_search() {
486        let mut store = VectorStore::new();
487        store.add_text("a", "hello world", None);
488        store.add_text("b", "goodbye world", None);
489        store.add_text("c", "hello there", None);
490        let results = store.search_text("hello world", 2);
491        assert_eq!(results.len(), 2);
492        assert_eq!(results[0].key, "a"); // Best match.
493    }
494
495    #[test]
496    fn vector_store_get_and_remove() {
497        let mut store = VectorStore::new();
498        store.add_text("key1", "test", Some("meta".to_string()));
499        assert!(store.get("key1").is_some());
500        assert!(store.remove("key1"));
501        assert!(store.get("key1").is_none());
502    }
503
504    #[test]
505    fn vector_store_capacity() {
506        let embedder = TextEmbedder::new(8, 2);
507        let mut store = VectorStore::with_embedder(embedder);
508        for i in 0..MAX_VECTORS {
509            assert!(store.add_text(&format!("k{i}"), &format!("text{i}"), None));
510        }
511        // Adding one more should fail.
512        assert!(!store.add_text("overflow", "text", None));
513    }
514
515    #[test]
516    fn vector_store_clear() {
517        let mut store = VectorStore::new();
518        store.add_text("a", "test", None);
519        store.add_text("b", "test2", None);
520        assert_eq!(store.len(), 2);
521        store.clear();
522        assert!(store.is_empty());
523    }
524
525    #[test]
526    fn scratchpad_set_and_get() {
527        let mut pad = Scratchpad::default();
528        pad.set("key", "value").unwrap();
529        assert_eq!(pad.get("key"), Some("value"));
530    }
531
532    #[test]
533    fn scratchpad_remove() {
534        let mut pad = Scratchpad::default();
535        pad.set("key", "value").unwrap();
536        assert!(pad.remove("key"));
537        assert_eq!(pad.get("key"), None);
538    }
539
540    #[test]
541    fn scratchpad_ttl_expiry() {
542        let mut pad = Scratchpad::new(5);
543        pad.set("key", "value").unwrap();
544        assert_eq!(pad.get("key"), Some("value"));
545        // Advance ticks past TTL.
546        for _ in 0..6 {
547            pad.tick();
548        }
549        assert_eq!(pad.get("key"), None);
550    }
551
552    #[test]
553    fn scratchpad_ttl_not_expired() {
554        let mut pad = Scratchpad::new(10);
555        pad.set("key", "value").unwrap();
556        pad.tick();
557        pad.tick();
558        assert_eq!(pad.get("key"), Some("value"));
559    }
560
561    #[test]
562    fn scratchpad_evict_expired() {
563        let mut pad = Scratchpad::new(2);
564        pad.set("a", "1").unwrap();
565        pad.set("b", "2").unwrap();
566        // Advance tick past TTL — tick() auto-evicts.
567        pad.tick();
568        pad.tick();
569        pad.tick();
570        // All entries should have been auto-evicted by tick().
571        assert!(pad.is_empty());
572        // Manual evict should find nothing.
573        let evicted = pad.evict_expired();
574        assert_eq!(evicted, 0);
575    }
576
577    #[test]
578    fn scratchpad_manual_evict_without_tick() {
579        let mut pad = Scratchpad::new(2);
580        pad.set("a", "1").unwrap();
581        // Advance tick manually (without eviction).
582        pad.current_tick = 5;
583        // Now manually evict — entries should be expired.
584        let evicted = pad.evict_expired();
585        assert_eq!(evicted, 1);
586        assert!(pad.is_empty());
587    }
588
589    #[test]
590    fn scratchpad_clear() {
591        let mut pad = Scratchpad::default();
592        pad.set("a", "1").unwrap();
593        pad.set("b", "2").unwrap();
594        pad.clear();
595        assert!(pad.is_empty());
596    }
597
598    #[test]
599    fn scratchpad_overwrite() {
600        let mut pad = Scratchpad::default();
601        pad.set("key", "old").unwrap();
602        pad.set("key", "new").unwrap();
603        assert_eq!(pad.get("key"), Some("new"));
604        assert_eq!(pad.len(), 1);
605    }
606
607    #[test]
608    fn scratchpad_capacity() {
609        let mut pad = Scratchpad::default();
610        for i in 0..MAX_SCRATCHPAD_ENTRIES {
611            pad.set(&format!("k{i}"), "v").unwrap();
612        }
613        let result = pad.set("overflow", "v");
614        assert_eq!(result, Err(ScratchpadError::Full));
615    }
616
617    #[test]
618    fn scratchpad_current_tick() {
619        let mut pad = Scratchpad::default();
620        assert_eq!(pad.current_tick(), 0);
621        pad.tick();
622        assert_eq!(pad.current_tick(), 1);
623        pad.tick();
624        assert_eq!(pad.current_tick(), 2);
625    }
626
627    #[test]
628    fn search_result_sorted_by_similarity() {
629        let mut store = VectorStore::new();
630        let embedder = TextEmbedder::default();
631        store.add("a", embedder.embed("hello world"), None);
632        store.add("b", embedder.embed("hello there"), None);
633        store.add("c", embedder.embed("completely different"), None);
634        let query = embedder.embed("hello world");
635        let results = store.search(&query, 3);
636        assert!(results[0].similarity >= results[1].similarity);
637        assert!(results[1].similarity >= results[2].similarity);
638    }
639}