qualia_core_db/inference/
semantic_skills.rs1use std::collections::HashMap;
24
25pub const EMBED_DIM: usize = 256;
29pub const MAX_VECTORS: usize = 4096;
31pub const MAX_SCRATCHPAD_ENTRIES: usize = 256;
33pub const DEFAULT_SCRATCHPAD_TTL: u32 = 1000;
35
36#[derive(Debug, Clone, PartialEq)]
38pub struct Vector {
39 pub dims: Vec<f32>,
40}
41
42impl Vector {
43 pub fn new(dims: Vec<f32>) -> Self {
45 Self { dims }
46 }
47
48 pub fn zeros(dim: usize) -> Self {
50 Self {
51 dims: vec![0.0; dim],
52 }
53 }
54
55 pub fn dim(&self) -> usize {
57 self.dims.len()
58 }
59
60 pub fn norm(&self) -> f32 {
62 self.dims.iter().map(|v| v * v).sum::<f32>().sqrt()
63 }
64
65 pub fn dot(&self, other: &Vector) -> f32 {
67 self.dims
68 .iter()
69 .zip(other.dims.iter())
70 .map(|(a, b)| a * b)
71 .sum()
72 }
73
74 pub fn cosine_similarity(&self, other: &Vector) -> f32 {
77 let norm_product = self.norm() * other.norm();
78 if norm_product == 0.0 {
79 return 0.0;
80 }
81 self.dot(other) / norm_product
82 }
83
84 pub fn cosine_distance(&self, other: &Vector) -> f32 {
86 1.0 - self.cosine_similarity(other)
87 }
88
89 pub fn normalize(&mut self) {
91 let norm = self.norm();
92 if norm > 0.0 {
93 for v in &mut self.dims {
94 *v /= norm;
95 }
96 }
97 }
98}
99
100pub struct TextEmbedder {
107 dim: usize,
108 ngram_size: usize,
109}
110
111impl TextEmbedder {
112 pub fn new(dim: usize, ngram_size: usize) -> Self {
115 Self { dim, ngram_size }
116 }
117
118 pub fn default() -> Self {
120 Self::new(EMBED_DIM, 3)
121 }
122
123 pub fn embed(&self, text: &str) -> Vector {
128 let mut dims = vec![0.0f32; self.dim];
129 let chars: Vec<char> = text.to_lowercase().chars().collect();
130 if chars.len() < self.ngram_size {
131 let h = fnv1a_hash(text.as_bytes());
133 dims[(h as usize) % self.dim] += 1.0;
134 return Vector { dims };
135 }
136 for i in 0..=chars.len() - self.ngram_size {
137 let ngram: String = chars[i..i + self.ngram_size].iter().collect();
138 let h = fnv1a_hash(ngram.as_bytes());
139 let idx = (h as usize) % self.dim;
140 dims[idx] += 1.0;
141 }
142 let mut v = Vector { dims };
143 v.normalize();
144 v
145 }
146}
147
148fn fnv1a_hash(data: &[u8]) -> u32 {
150 let mut hash: u32 = 0x811C_9DC5;
151 for &b in data {
152 hash ^= b as u32;
153 hash = hash.wrapping_mul(0x0100_0193);
154 }
155 hash
156}
157
158#[derive(Debug, Clone)]
162pub struct StoredVector {
163 pub key: String,
164 pub vector: Vector,
165 pub metadata: Option<String>,
166}
167
168pub struct VectorStore {
170 vectors: Vec<StoredVector>,
171 embedder: TextEmbedder,
172}
173
174impl VectorStore {
175 pub fn new() -> Self {
177 Self {
178 vectors: Vec::new(),
179 embedder: TextEmbedder::default(),
180 }
181 }
182
183 pub fn with_embedder(embedder: TextEmbedder) -> Self {
185 Self {
186 vectors: Vec::new(),
187 embedder,
188 }
189 }
190
191 pub fn add(&mut self, key: &str, vector: Vector, metadata: Option<String>) -> bool {
193 if self.vectors.len() >= MAX_VECTORS {
194 return false;
195 }
196 self.vectors.push(StoredVector {
197 key: key.to_string(),
198 vector,
199 metadata,
200 });
201 true
202 }
203
204 pub fn add_text(&mut self, key: &str, text: &str, metadata: Option<String>) -> bool {
206 let vector = self.embedder.embed(text);
207 self.add(key, vector, metadata)
208 }
209
210 pub fn search(&self, query: &Vector, k: usize) -> Vec<SearchResult> {
213 let mut results: Vec<SearchResult> = self
214 .vectors
215 .iter()
216 .map(|sv| SearchResult {
217 key: sv.key.clone(),
218 similarity: query.cosine_similarity(&sv.vector),
219 metadata: sv.metadata.clone(),
220 })
221 .collect();
222 results.sort_by(|a, b| {
223 b.similarity
224 .partial_cmp(&a.similarity)
225 .unwrap_or(std::cmp::Ordering::Equal)
226 });
227 results.truncate(k);
228 results
229 }
230
231 pub fn search_text(&self, query_text: &str, k: usize) -> Vec<SearchResult> {
233 let query = self.embedder.embed(query_text);
234 self.search(&query, k)
235 }
236
237 pub fn get(&self, key: &str) -> Option<&StoredVector> {
239 self.vectors.iter().find(|v| v.key == key)
240 }
241
242 pub fn remove(&mut self, key: &str) -> bool {
244 let len_before = self.vectors.len();
245 self.vectors.retain(|v| v.key != key);
246 self.vectors.len() != len_before
247 }
248
249 pub fn len(&self) -> usize {
251 self.vectors.len()
252 }
253
254 pub fn is_empty(&self) -> bool {
256 self.vectors.is_empty()
257 }
258
259 pub fn clear(&mut self) {
261 self.vectors.clear();
262 }
263}
264
265impl Default for VectorStore {
266 fn default() -> Self {
268 Self::new()
269 }
270}
271
272#[derive(Debug, Clone)]
274pub struct SearchResult {
275 pub key: String,
276 pub similarity: f32,
277 pub metadata: Option<String>,
278}
279
280#[derive(Debug, Clone)]
284pub struct ScratchpadEntry {
285 pub value: String,
286 pub tick: u32,
287 pub ttl: u32,
288}
289
290impl ScratchpadEntry {
291 pub fn new(value: String, tick: u32, ttl: u32) -> Self {
293 Self { value, tick, ttl }
294 }
295
296 pub fn is_expired(&self, current_tick: u32) -> bool {
298 current_tick > self.tick + self.ttl
299 }
300}
301
302pub struct Scratchpad {
305 entries: HashMap<String, ScratchpadEntry>,
306 default_ttl: u32,
307 current_tick: u32,
308}
309
310impl Scratchpad {
311 pub fn new(default_ttl: u32) -> Self {
313 Self {
314 entries: HashMap::new(),
315 default_ttl,
316 current_tick: 0,
317 }
318 }
319
320 pub fn default() -> Self {
322 Self::new(DEFAULT_SCRATCHPAD_TTL)
323 }
324
325 pub fn tick(&mut self) {
327 self.current_tick += 1;
328 let tick = self.current_tick;
330 self.entries.retain(|_, e| !e.is_expired(tick));
331 }
332
333 pub fn set(&mut self, key: &str, value: &str) -> Result<(), ScratchpadError> {
335 self.set_with_ttl(key, value, self.default_ttl)
336 }
337
338 pub fn set_with_ttl(
340 &mut self,
341 key: &str,
342 value: &str,
343 ttl: u32,
344 ) -> Result<(), ScratchpadError> {
345 if self.entries.len() >= MAX_SCRATCHPAD_ENTRIES && !self.entries.contains_key(key) {
346 return Err(ScratchpadError::Full);
347 }
348 self.entries.insert(
349 key.to_string(),
350 ScratchpadEntry::new(value.to_string(), self.current_tick, ttl),
351 );
352 Ok(())
353 }
354
355 pub fn get(&self, key: &str) -> Option<&str> {
357 let entry = self.entries.get(key)?;
358 if entry.is_expired(self.current_tick) {
359 return None;
360 }
361 Some(entry.value.as_str())
362 }
363
364 pub fn remove(&mut self, key: &str) -> bool {
366 self.entries.remove(key).is_some()
367 }
368
369 pub fn clear(&mut self) {
371 self.entries.clear();
372 }
373
374 pub fn len(&self) -> usize {
376 self.entries.len()
377 }
378
379 pub fn is_empty(&self) -> bool {
381 self.entries.is_empty()
382 }
383
384 pub fn current_tick(&self) -> u32 {
386 self.current_tick
387 }
388
389 pub fn evict_expired(&mut self) -> usize {
391 let tick = self.current_tick;
392 let len_before = self.entries.len();
393 self.entries.retain(|_, e| !e.is_expired(tick));
394 len_before - self.entries.len()
395 }
396}
397
398#[derive(Debug, Clone, PartialEq, Eq)]
400pub enum ScratchpadError {
401 Full,
403}
404
405#[cfg(test)]
408mod tests {
409 use super::*;
410
411 #[test]
412 fn vector_cosine_similarity_identical() {
413 let v = Vector::new(vec![1.0, 2.0, 3.0]);
414 let sim = v.cosine_similarity(&v);
415 assert!((sim - 1.0).abs() < 1e-5);
416 }
417
418 #[test]
419 fn vector_cosine_similarity_orthogonal() {
420 let v1 = Vector::new(vec![1.0, 0.0]);
421 let v2 = Vector::new(vec![0.0, 1.0]);
422 let sim = v1.cosine_similarity(&v2);
423 assert!(sim.abs() < 1e-5);
424 }
425
426 #[test]
427 fn vector_cosine_similarity_opposite() {
428 let v1 = Vector::new(vec![1.0, 0.0]);
429 let v2 = Vector::new(vec![-1.0, 0.0]);
430 let sim = v1.cosine_similarity(&v2);
431 assert!((sim - (-1.0)).abs() < 1e-5);
432 }
433
434 #[test]
435 fn vector_cosine_distance() {
436 let v1 = Vector::new(vec![1.0, 0.0]);
437 let v2 = Vector::new(vec![1.0, 0.0]);
438 let dist = v1.cosine_distance(&v2);
439 assert!(dist.abs() < 1e-5);
440 }
441
442 #[test]
443 fn vector_normalize() {
444 let mut v = Vector::new(vec![3.0, 4.0]);
445 v.normalize();
446 assert!((v.norm() - 1.0).abs() < 1e-5);
447 }
448
449 #[test]
450 fn vector_zero_norm_similarity() {
451 let v1 = Vector::new(vec![0.0, 0.0]);
452 let v2 = Vector::new(vec![1.0, 0.0]);
453 let sim = v1.cosine_similarity(&v2);
454 assert_eq!(sim, 0.0);
455 }
456
457 #[test]
458 fn text_embedder_basic() {
459 let embedder = TextEmbedder::default();
460 let v1 = embedder.embed("hello world");
461 let v2 = embedder.embed("hello world");
462 assert_eq!(v1, v2);
463 assert_eq!(v1.dim(), EMBED_DIM);
464 }
465
466 #[test]
467 fn text_embedder_similar_texts() {
468 let embedder = TextEmbedder::default();
469 let v1 = embedder.embed("the quick brown fox");
470 let v2 = embedder.embed("the quick brown fox");
471 let v3 = embedder.embed("completely different text");
472 let sim_same = v1.cosine_similarity(&v2);
473 let sim_diff = v1.cosine_similarity(&v3);
474 assert!(sim_same > sim_diff);
475 }
476
477 #[test]
478 fn text_embedder_normalizes() {
479 let embedder = TextEmbedder::default();
480 let v = embedder.embed("test text");
481 assert!((v.norm() - 1.0).abs() < 1e-5);
482 }
483
484 #[test]
485 fn vector_store_add_and_search() {
486 let mut store = VectorStore::new();
487 store.add_text("a", "hello world", None);
488 store.add_text("b", "goodbye world", None);
489 store.add_text("c", "hello there", None);
490 let results = store.search_text("hello world", 2);
491 assert_eq!(results.len(), 2);
492 assert_eq!(results[0].key, "a"); }
494
495 #[test]
496 fn vector_store_get_and_remove() {
497 let mut store = VectorStore::new();
498 store.add_text("key1", "test", Some("meta".to_string()));
499 assert!(store.get("key1").is_some());
500 assert!(store.remove("key1"));
501 assert!(store.get("key1").is_none());
502 }
503
504 #[test]
505 fn vector_store_capacity() {
506 let embedder = TextEmbedder::new(8, 2);
507 let mut store = VectorStore::with_embedder(embedder);
508 for i in 0..MAX_VECTORS {
509 assert!(store.add_text(&format!("k{i}"), &format!("text{i}"), None));
510 }
511 assert!(!store.add_text("overflow", "text", None));
513 }
514
515 #[test]
516 fn vector_store_clear() {
517 let mut store = VectorStore::new();
518 store.add_text("a", "test", None);
519 store.add_text("b", "test2", None);
520 assert_eq!(store.len(), 2);
521 store.clear();
522 assert!(store.is_empty());
523 }
524
525 #[test]
526 fn scratchpad_set_and_get() {
527 let mut pad = Scratchpad::default();
528 pad.set("key", "value").unwrap();
529 assert_eq!(pad.get("key"), Some("value"));
530 }
531
532 #[test]
533 fn scratchpad_remove() {
534 let mut pad = Scratchpad::default();
535 pad.set("key", "value").unwrap();
536 assert!(pad.remove("key"));
537 assert_eq!(pad.get("key"), None);
538 }
539
540 #[test]
541 fn scratchpad_ttl_expiry() {
542 let mut pad = Scratchpad::new(5);
543 pad.set("key", "value").unwrap();
544 assert_eq!(pad.get("key"), Some("value"));
545 for _ in 0..6 {
547 pad.tick();
548 }
549 assert_eq!(pad.get("key"), None);
550 }
551
552 #[test]
553 fn scratchpad_ttl_not_expired() {
554 let mut pad = Scratchpad::new(10);
555 pad.set("key", "value").unwrap();
556 pad.tick();
557 pad.tick();
558 assert_eq!(pad.get("key"), Some("value"));
559 }
560
561 #[test]
562 fn scratchpad_evict_expired() {
563 let mut pad = Scratchpad::new(2);
564 pad.set("a", "1").unwrap();
565 pad.set("b", "2").unwrap();
566 pad.tick();
568 pad.tick();
569 pad.tick();
570 assert!(pad.is_empty());
572 let evicted = pad.evict_expired();
574 assert_eq!(evicted, 0);
575 }
576
577 #[test]
578 fn scratchpad_manual_evict_without_tick() {
579 let mut pad = Scratchpad::new(2);
580 pad.set("a", "1").unwrap();
581 pad.current_tick = 5;
583 let evicted = pad.evict_expired();
585 assert_eq!(evicted, 1);
586 assert!(pad.is_empty());
587 }
588
589 #[test]
590 fn scratchpad_clear() {
591 let mut pad = Scratchpad::default();
592 pad.set("a", "1").unwrap();
593 pad.set("b", "2").unwrap();
594 pad.clear();
595 assert!(pad.is_empty());
596 }
597
598 #[test]
599 fn scratchpad_overwrite() {
600 let mut pad = Scratchpad::default();
601 pad.set("key", "old").unwrap();
602 pad.set("key", "new").unwrap();
603 assert_eq!(pad.get("key"), Some("new"));
604 assert_eq!(pad.len(), 1);
605 }
606
607 #[test]
608 fn scratchpad_capacity() {
609 let mut pad = Scratchpad::default();
610 for i in 0..MAX_SCRATCHPAD_ENTRIES {
611 pad.set(&format!("k{i}"), "v").unwrap();
612 }
613 let result = pad.set("overflow", "v");
614 assert_eq!(result, Err(ScratchpadError::Full));
615 }
616
617 #[test]
618 fn scratchpad_current_tick() {
619 let mut pad = Scratchpad::default();
620 assert_eq!(pad.current_tick(), 0);
621 pad.tick();
622 assert_eq!(pad.current_tick(), 1);
623 pad.tick();
624 assert_eq!(pad.current_tick(), 2);
625 }
626
627 #[test]
628 fn search_result_sorted_by_similarity() {
629 let mut store = VectorStore::new();
630 let embedder = TextEmbedder::default();
631 store.add("a", embedder.embed("hello world"), None);
632 store.add("b", embedder.embed("hello there"), None);
633 store.add("c", embedder.embed("completely different"), None);
634 let query = embedder.embed("hello world");
635 let results = store.search(&query, 3);
636 assert!(results[0].similarity >= results[1].similarity);
637 assert!(results[1].similarity >= results[2].similarity);
638 }
639}