qualia_core_db/foundation/
topology_draft.rs1use crate::compute_universe::{TopologyDraftBatch, MAX_DRAFT_LEN};
4use crate::gguf_sharder::GgufTokenizer;
5
6pub struct TopologyDraftMapper<'a> {
8 tokenizer: &'a GgufTokenizer,
9}
10
11impl<'a> TopologyDraftMapper<'a> {
12 pub fn new(tokenizer: &'a GgufTokenizer) -> Self {
14 Self { tokenizer }
15 }
16
17 pub fn concept_to_token_id(&self, concept_hash: u64) -> u32 {
19 let probe = format!("q42:{:016x}", concept_hash);
20 let ids = self.tokenizer.encode(&probe);
21 if let Some(&id) = ids.first() {
22 return id;
23 }
24 (concept_hash as u32) % self.tokenizer.vocab_len().max(1)
25 }
26
27 pub fn fill_draft_batch(&self, concept_hashes: &[u64], gamma: usize) -> TopologyDraftBatch {
29 let gamma = gamma.clamp(1, MAX_DRAFT_LEN).min(concept_hashes.len());
30 let mut batch = TopologyDraftBatch::empty();
31 for i in 0..gamma {
32 batch.concept_hashes[i] = concept_hashes[i];
33 batch.draft_ids[i] = self.concept_to_token_id(concept_hashes[i]);
34 }
35 batch.draft_len = gamma as u8;
36 batch
37 }
38}
39
40#[cfg(test)]
41mod tests {
42 use super::*;
43
44 #[test]
45 fn mapper_is_stable_for_same_hash() {
46 let tok = GgufTokenizer::default();
47 let mapper = TopologyDraftMapper::new(&tok);
48 let a = mapper.concept_to_token_id(0xDEAD_BEEF);
49 let b = mapper.concept_to_token_id(0xDEAD_BEEF);
50 assert_eq!(a, b);
51 }
52
53 #[test]
54 fn fill_draft_batch_respects_gamma() {
55 let tok = GgufTokenizer::default();
56 let mapper = TopologyDraftMapper::new(&tok);
57 let batch = mapper.fill_draft_batch(&[1, 2, 3, 4], 3);
58 assert_eq!(batch.draft_len, 3);
59 assert_ne!(batch.draft_ids[0], 0);
60 }
61}