Skip to main content

qualia_core_db/inference/gguf_sharder/
tokenizer.rs

1//! `GgufTokenizer` — vocabulary + BOS/EOS + BPE merges parsed from a GGUF KV
2//! section (or a compact P64 section), with encode/decode, chat-template handling,
3//! and the generation stop-token set.
4
5use super::gguf_skip_value;
6use std::collections::HashMap;
7use std::sync::OnceLock;
8
9mod decode;
10mod prepared;
11pub mod hf_json;
12mod pretokenizer;
13pub use pretokenizer::{PretokenError, PretokenSpan};
14
15/// GPT-2 `bytes_to_unicode` table — maps raw bytes to BPE merge symbols.
16fn gpt2_byte_to_unicode(byte: u8) -> char {
17    static TABLE: OnceLock<[char; 256]> = OnceLock::new();
18    TABLE.get_or_init(|| {
19        let mut bs: Vec<u32> = (b'!'..=b'~')
20            .chain(b'\xA1'..=b'\xAC')
21            .chain(b'\xAE'..=b'\xFF')
22            .map(|b| b as u32)
23            .collect();
24        let mut cs = bs.clone();
25        let mut n = 0u32;
26        for b in 0u32..256 {
27            if !bs.contains(&b) {
28                bs.push(b);
29                cs.push(256 + n);
30                n += 1;
31            }
32        }
33        let mut out = ['\0'; 256];
34        for (b, c) in bs.into_iter().zip(cs) {
35            out[b as usize] = char::from_u32(c).unwrap_or('\u{FFFD}');
36        }
37        out
38    })[byte as usize]
39}
40
41fn gpt2_unicode_to_byte(symbol: char) -> Option<u8> {
42    (0u16..=255)
43        .find(|byte| gpt2_byte_to_unicode(*byte as u8) == symbol)
44        .map(|byte| byte as u8)
45}
46
47/// Max stop-token ids kept on the tokenizer (eos + chat-end family + extras).
48pub const MAX_STOP_TOKEN_IDS: usize = 8;
49
50/// Vocabulary and BOS/EOS metadata extracted from a GGUF KV section.
51/// Used by `infer_local_model()` to encode prompts and decode output token IDs.
52pub struct GgufTokenizer {
53    /// Token ID → string (index = token ID).
54    pub vocab: Vec<String>,
55    pub bos_token_id: u32,
56    pub eos_token_id: u32,
57    /// `tokenizer.ggml.add_bos_token` — prepend BOS before prompt tokens when true.
58    pub add_bos_token: bool,
59    /// `tokenizer.ggml.pre` — e.g. `smollm`, `gpt2`; drives pretokenization.
60    pub pre_type: String,
61    /// BPE merge ranks: `(left_symbol, right_symbol)` in ascending rank order.
62    merge_pairs: Vec<(String, String)>,
63    /// Cold-built pair fingerprint -> rank index. A detected fingerprint collision disables the
64    /// index and preserves the exact linear oracle.
65    merge_rank_index: HashMap<u64, usize>,
66    merge_rank_collision: bool,
67    /// Fast vocab lookup for BPE tail + legacy greedy path.
68    pub(super) token_to_id_map: HashMap<String, u32>,
69    /// Special tokens (`<|…|>`, etc.) sorted longest-first for atomic matching.
70    special_tokens: Vec<(String, u32)>,
71    /// (token_string, token_id) sorted by descending byte length — legacy greedy fallback.
72    pub(super) token_to_id: Vec<(String, u32)>,
73    /// Decode stop set: always includes `eos_token_id`, plus chat-end specials when
74    /// present in vocab (`<|eot_id|>`, `<|im_end|>`, `<end_of_turn>`, …). Fixed array
75    /// so the hot path does not allocate.
76    stop_token_ids: [u32; MAX_STOP_TOKEN_IDS],
77    stop_token_count: u8,
78}
79
80/// Chat-template family, detected from the special tokens a model's vocab carries. Instruct models
81/// must have their prompt wrapped in this template (with an assistant-turn cue) or they degenerate
82/// (emit EOS immediately, or repeat) — a raw prompt gives the model no "your turn to answer" signal.
83#[derive(Clone, Copy, PartialEq, Eq, Debug)]
84pub enum ChatFamily {
85    /// `<|im_start|>role\n…<|im_end|>` — Qwen2 / SmolLM2 / many instruct models.
86    ChatMl,
87    /// `<|start_header_id|>role<|end_header_id|>\n\n…<|eot_id|>` — Llama-3.x.
88    Llama3,
89    /// `<start_of_turn>role\n…<end_of_turn>` — Gemma 1/2/3 (no system role).
90    Gemma,
91    /// `<|turn>role\n…<turn|>` — Gemma 4 instruct (also uses `<|channel>` tool channel).
92    Gemma4,
93    /// No recognised chat specials — the raw prompt is used unchanged.
94    None,
95}
96
97impl ChatFamily {
98    /// Static string.
99    pub const fn as_str(&self) -> &'static str {
100        match self {
101            Self::ChatMl => "chatml",
102            Self::Llama3 => "llama3",
103            Self::Gemma => "gemma",
104            Self::Gemma4 => "gemma4",
105            Self::None => "none",
106        }
107    }
108}
109
110impl Default for GgufTokenizer {
111    /// 256-entry byte-level fallback tokenizer — used when no GGUF is loaded.
112    fn default() -> Self {
113        let vocab: Vec<String> = (0u32..256)
114            .map(|b| {
115                let c = b as u8;
116                if c.is_ascii_graphic() || c == b' ' {
117                    (c as char).to_string()
118                } else {
119                    format!("<0x{:02X}>", b)
120                }
121            })
122            .collect();
123        let mut t2id: Vec<(String, u32)> = vocab
124            .iter()
125            .enumerate()
126            .map(|(i, s)| (s.clone(), i as u32))
127            .collect();
128        t2id.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
129        let token_to_id_map: HashMap<String, u32> =
130            t2id.iter().map(|(s, id)| (s.clone(), *id)).collect();
131        let mut tok = Self {
132            vocab,
133            bos_token_id: 1,
134            eos_token_id: 2,
135            add_bos_token: true,
136            pre_type: String::new(),
137            merge_pairs: Vec::new(),
138            merge_rank_index: HashMap::new(),
139            merge_rank_collision: false,
140            token_to_id_map,
141            special_tokens: Vec::new(),
142            token_to_id: t2id,
143            stop_token_ids: [0; MAX_STOP_TOKEN_IDS],
144            stop_token_count: 0,
145        };
146        tok.rebuild_stop_token_ids();
147        tok
148    }
149}
150
151impl GgufTokenizer {
152    /// Parse vocab + BOS/EOS from a memory-mapped GGUF v2/v3 file.
153    /// Falls back to `Default` (byte-level) on any parse error.
154    pub fn from_gguf(mmap: &[u8]) -> Self {
155        Self::try_parse(mmap).unwrap_or_default()
156    }
157
158    fn try_parse(mmap: &[u8]) -> Option<Self> {
159        if mmap.len() < 24 || &mmap[0..4] != b"GGUF" {
160            return None;
161        }
162        let version = u32::from_le_bytes(mmap[4..8].try_into().ok()?);
163        if version < 2 {
164            return None;
165        } // only v2/v3 have u64 string lengths
166        let kv_count = u64::from_le_bytes(mmap[16..24].try_into().ok()?);
167        let mut pos = 24usize;
168        let mut vocab: Option<Vec<String>> = None;
169        let mut merges_raw: Option<Vec<String>> = None;
170        let mut bos_id: Option<u32> = None;
171        let mut eos_id: Option<u32> = None;
172        let mut add_bos: Option<bool> = None;
173        let mut pre_type: Option<String> = None;
174
175        for _ in 0..kv_count {
176            if pos + 8 > mmap.len() {
177                break;
178            }
179            let klen = u64::from_le_bytes(mmap[pos..pos + 8].try_into().ok()?) as usize;
180            pos += 8;
181            if pos + klen > mmap.len() {
182                break;
183            }
184            let key = std::str::from_utf8(&mmap[pos..pos + klen]).unwrap_or("");
185            pos += klen;
186            if pos + 4 > mmap.len() {
187                break;
188            }
189            let vtype = u32::from_le_bytes(mmap[pos..pos + 4].try_into().ok()?);
190            pos += 4;
191            match key {
192                "tokenizer.ggml.tokens" => {
193                    vocab = Self::read_string_array(mmap, &mut pos, vtype);
194                }
195                "tokenizer.ggml.merges" => {
196                    merges_raw = Self::read_string_array(mmap, &mut pos, vtype);
197                }
198                "tokenizer.ggml.bos_token_id" => {
199                    bos_id = Self::read_u32_val(mmap, &mut pos, vtype);
200                }
201                "tokenizer.ggml.eos_token_id" => {
202                    eos_id = Self::read_u32_val(mmap, &mut pos, vtype);
203                }
204                "tokenizer.ggml.add_bos_token" => {
205                    add_bos = Self::read_bool_val(mmap, &mut pos, vtype);
206                }
207                "tokenizer.ggml.pre" => {
208                    pre_type = Self::read_string_val(mmap, &mut pos, vtype);
209                }
210                _ => {
211                    if Self::skip_value(mmap, &mut pos, vtype).is_none() {
212                        break;
213                    }
214                }
215            }
216        }
217
218        let v = vocab?;
219        Some(Self::from_raw_components(
220            v,
221            merges_raw.as_deref(),
222            bos_id,
223            eos_id,
224            add_bos,
225            pre_type,
226        ))
227    }
228
229    /// Construct tokenizer from parsed vocabulary components.
230    pub fn from_raw_components(
231        vocab: Vec<String>,
232        merges_raw: Option<&[String]>,
233        bos_id: Option<u32>,
234        eos_id: Option<u32>,
235        add_bos: Option<bool>,
236        pre_type: Option<String>,
237    ) -> Self {
238        let bos = bos_id.unwrap_or(1);
239        let eos = eos_id.unwrap_or(2);
240        let mut t2id: Vec<(String, u32)> = vocab
241            .iter()
242            .enumerate()
243            .map(|(i, s)| (s.clone(), i as u32))
244            .collect();
245        t2id.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
246        let token_to_id_map: HashMap<String, u32> =
247            t2id.iter().map(|(s, id)| (s.clone(), *id)).collect();
248        let mut special_tokens: Vec<(String, u32)> = vocab
249            .iter()
250            .enumerate()
251            .filter(|(_, s)| s.starts_with('<') && s.ends_with('>'))
252            .map(|(i, s)| (s.clone(), i as u32))
253            .collect();
254        special_tokens.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
255        let merge_pairs = Self::parse_merge_pairs(merges_raw);
256        let (merge_rank_index, merge_rank_collision) = Self::build_merge_rank_index(&merge_pairs);
257        let default_add_bos = bos_id.is_some()
258            || vocab
259                .get(bos as usize)
260                .is_some_and(|s| s.starts_with('<') && s.ends_with('>'))
261            || vocab.len() <= 256;
262        let mut tok = Self {
263            vocab,
264            bos_token_id: bos,
265            eos_token_id: eos,
266            add_bos_token: add_bos.unwrap_or(default_add_bos),
267            pre_type: pre_type.unwrap_or_default(),
268            merge_pairs,
269            merge_rank_index,
270            merge_rank_collision,
271            token_to_id_map,
272            special_tokens,
273            token_to_id: t2id,
274            stop_token_ids: [0; MAX_STOP_TOKEN_IDS],
275            stop_token_count: 0,
276        };
277        tok.rebuild_stop_token_ids();
278        tok
279    }
280
281    /// Parse Hugging Face `tokenizer.json` format into a `GgufTokenizer`.
282    pub fn from_hf_json(json_str: &str) -> Option<Self> {
283        hf_json::parse_hf_tokenizer_json(json_str)
284    }
285
286    /// Phase 4 v3 / v2: serialize the tokenizer into a compact, contiguous P64 section (no page
287    /// alignment needed). Fields: vocab / merges / bos / eos / add_bos / pre, plus (v2) the
288    /// stop-token set so decode does not re-guess chat ends. Derived maps are rebuilt by
289    /// `from_p64_section()`. Heap use here is load-time only.
290    pub fn to_p64_section(&self) -> Vec<u8> {
291        let mut out = Vec::with_capacity(1 << 20);
292        out.extend_from_slice(b"Q42T");
293        out.extend_from_slice(&2u16.to_le_bytes()); // section version (v2 = stop tokens)
294        out.extend_from_slice(&0u16.to_le_bytes()); // flags
295        out.extend_from_slice(&self.bos_token_id.to_le_bytes());
296        out.extend_from_slice(&self.eos_token_id.to_le_bytes());
297        out.push(self.add_bos_token as u8);
298        out.extend_from_slice(&[0u8; 3]);
299        let put_str = |o: &mut Vec<u8>, s: &str| {
300            o.extend_from_slice(&(s.len() as u32).to_le_bytes());
301            o.extend_from_slice(s.as_bytes());
302        };
303        put_str(&mut out, &self.pre_type);
304        out.extend_from_slice(&(self.vocab.len() as u32).to_le_bytes());
305        for t in &self.vocab {
306            put_str(&mut out, t);
307        }
308        out.extend_from_slice(&(self.merge_pairs.len() as u32).to_le_bytes());
309        for (l, r) in &self.merge_pairs {
310            put_str(&mut out, l);
311            put_str(&mut out, r);
312        }
313        // v2: stop-token set (eos + chat ends). Fixed 8 slots, count in first byte.
314        out.push(self.stop_token_count);
315        out.extend_from_slice(&[0u8; 3]);
316        for id in &self.stop_token_ids {
317            out.extend_from_slice(&id.to_le_bytes());
318        }
319        out
320    }
321
322    /// Compatibility alias for the historical pre-P64 method name.
323    #[deprecated(note = "use to_p64_section")]
324    pub fn to_q42_section(&self) -> Vec<u8> {
325        self.to_p64_section()
326    }
327
328    /// Phase 4 v3: rebuild a tokenizer from a P64 tokenizer section — bypasses GGUF KV string-key
329    /// parsing entirely. Fully bounds-checked (the section is untrusted input). Returns `None` on any
330    /// malformed field.
331    pub fn from_p64_section(data: &[u8]) -> Option<Self> {
332        let mut p = 0usize;
333        let take = |p: &mut usize, n: usize| -> Option<&[u8]> {
334            let end = p.checked_add(n)?;
335            if end > data.len() {
336                return None;
337            }
338            let s = &data[*p..end];
339            *p = end;
340            Some(s)
341        };
342        let take_u32 = |p: &mut usize| -> Option<u32> {
343            Some(u32::from_le_bytes(take(p, 4)?.try_into().ok()?))
344        };
345        let take_str = |p: &mut usize| -> Option<String> {
346            let len = take_u32(p)? as usize;
347            Some(String::from_utf8_lossy(take(p, len)?).into_owned())
348        };
349        if take(&mut p, 4)? != b"Q42T" {
350            return None;
351        }
352        let ver = u16::from_le_bytes(take(&mut p, 2)?.try_into().ok()?);
353        let _flags = take(&mut p, 2)?;
354        let bos = take_u32(&mut p)?;
355        let eos = take_u32(&mut p)?;
356        let add_bos = take(&mut p, 1)?[0] != 0;
357        let _pad = take(&mut p, 3)?;
358        let pre_type = take_str(&mut p)?;
359        let n_vocab = take_u32(&mut p)? as usize;
360        if n_vocab > 1_000_000 {
361            return None;
362        }
363        let mut vocab = Vec::with_capacity(n_vocab);
364        for _ in 0..n_vocab {
365            vocab.push(take_str(&mut p)?);
366        }
367        let n_merges = take_u32(&mut p)? as usize;
368        if n_merges > 5_000_000 {
369            return None;
370        }
371        let mut merge_pairs = Vec::with_capacity(n_merges);
372        for _ in 0..n_merges {
373            let l = take_str(&mut p)?;
374            let r = take_str(&mut p)?;
375            merge_pairs.push((l, r));
376        }
377        // v2 optional trailer: stop_count + pad3 + 8×u32. v1 rebuilds from vocab specials.
378        let mut stored_stops: Option<([u32; MAX_STOP_TOKEN_IDS], u8)> = None;
379        if ver >= 2 && p + 4 + MAX_STOP_TOKEN_IDS * 4 <= data.len() {
380            let count = take(&mut p, 1)?[0].min(MAX_STOP_TOKEN_IDS as u8);
381            let _ = take(&mut p, 3)?;
382            let mut ids = [0u32; MAX_STOP_TOKEN_IDS];
383            for i in 0..MAX_STOP_TOKEN_IDS {
384                ids[i] = take_u32(&mut p)?;
385            }
386            stored_stops = Some((ids, count));
387        }
388        // Rebuild the derived maps exactly as `try_parse` does, so encode/decode are identical.
389        let mut t2id: Vec<(String, u32)> = vocab
390            .iter()
391            .enumerate()
392            .map(|(i, s)| (s.clone(), i as u32))
393            .collect();
394        t2id.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
395        let token_to_id_map: HashMap<String, u32> =
396            t2id.iter().map(|(s, id)| (s.clone(), *id)).collect();
397        let mut special_tokens: Vec<(String, u32)> = vocab
398            .iter()
399            .enumerate()
400            .filter(|(_, s)| s.starts_with('<') && s.ends_with('>'))
401            .map(|(i, s)| (s.clone(), i as u32))
402            .collect();
403        special_tokens.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
404        let mut tok = Self {
405            vocab,
406            bos_token_id: bos,
407            eos_token_id: eos,
408            add_bos_token: add_bos,
409            pre_type,
410            merge_pairs,
411            merge_rank_index: HashMap::new(),
412            merge_rank_collision: false,
413            token_to_id_map,
414            special_tokens,
415            token_to_id: t2id,
416            stop_token_ids: [0; MAX_STOP_TOKEN_IDS],
417            stop_token_count: 0,
418        };
419        (tok.merge_rank_index, tok.merge_rank_collision) =
420            Self::build_merge_rank_index(&tok.merge_pairs);
421        if let Some((ids, count)) = stored_stops {
422            tok.stop_token_ids = ids;
423            tok.stop_token_count = count;
424            // Always ensure eos is present even if a stale helper omitted it.
425            if !tok.is_stop_token(eos) {
426                tok.rebuild_stop_token_ids();
427            }
428        } else {
429            tok.rebuild_stop_token_ids();
430        }
431        Some(tok)
432    }
433
434    /// Rebuild the decode stop set from `eos_token_id` + known chat-end specials in vocab.
435    /// Call after any mutation of eos / token_to_id_map (load paths do this automatically).
436    pub fn rebuild_stop_token_ids(&mut self) {
437        let mut ids = [0u32; MAX_STOP_TOKEN_IDS];
438        let mut n = 0usize;
439        let mut push = |id: u32| {
440            if n >= MAX_STOP_TOKEN_IDS {
441                return;
442            }
443            if ids[..n].contains(&id) {
444                return;
445            }
446            ids[n] = id;
447            n += 1;
448        };
449        push(self.eos_token_id);
450        // Chat / instruct end-of-turn tokens. Missing from vocab → no-op.
451        // Without these, Llama-3 keeps past <|eot_id|> into pretraining-style continuation.
452        const CHAT_ENDS: &[&str] = &[
453            "<|endoftext|>",
454            "<|eot_id|>",
455            "<|im_end|>",
456            "<end_of_turn>",
457            "<turn|>", // Gemma 4 turn close
458            "<|end_of_text|>",
459            "</s>",
460            "<|end|>",
461        ];
462        for s in CHAT_ENDS {
463            if let Some(&id) = self.token_to_id_map.get(*s) {
464                push(id);
465            }
466        }
467        // Also scan special_tokens + full token_to_id_map for end-of-turn markers
468        // (SentencePiece / BPE may store them with leading space or ▁ prefixes).
469        let looks_like_chat_end = |n: &str| -> bool {
470            let l = n.to_ascii_lowercase();
471            l == "<|im_end|>"
472                || l == "<|eot_id|>"
473                || l == "<|endoftext|>"
474                || l == "<end_of_turn>"
475                || l == "</s>"
476                || l.ends_with("im_end|>")
477                || l.ends_with("eot_id|>")
478                || l.contains("end_of_turn")
479                || l.contains("im_end")
480                || l.contains("eot_id")
481        };
482        for (name, id) in &self.special_tokens {
483            if looks_like_chat_end(name) {
484                push(*id);
485            }
486        }
487        for (name, id) in &self.token_to_id_map {
488            if looks_like_chat_end(name) {
489                push(*id);
490            }
491        }
492        self.stop_token_ids = ids;
493        self.stop_token_count = n as u8;
494    }
495
496    /// Whether `id` is a generation stop token (eos and/or chat end-of-turn).
497    #[inline]
498    pub fn is_stop_token(&self, id: u32) -> bool {
499        let n = self.stop_token_count as usize;
500        self.stop_token_ids[..n].contains(&id)
501    }
502
503    /// Slice of active stop-token ids (for logging / q42 export).
504    pub fn stop_tokens(&self) -> &[u32] {
505        &self.stop_token_ids[..self.stop_token_count as usize]
506    }
507
508    /// Merge extra stop ids (e.g. from a model's canonical `.q42` metadata) into the stop set.
509    /// Does not allocate; drops overflow past [`MAX_STOP_TOKEN_IDS`].
510    pub fn merge_stop_token_ids(&mut self, extra: &[u32]) {
511        let mut n = self.stop_token_count as usize;
512        for &id in extra {
513            if n >= MAX_STOP_TOKEN_IDS {
514                break;
515            }
516            if self.stop_token_ids[..n].contains(&id) {
517                continue;
518            }
519            self.stop_token_ids[n] = id;
520            n += 1;
521        }
522        // Always keep eos.
523        if !self.stop_token_ids[..n].contains(&self.eos_token_id) && n < MAX_STOP_TOKEN_IDS {
524            self.stop_token_ids[n] = self.eos_token_id;
525            n += 1;
526        }
527        self.stop_token_count = n as u8;
528    }
529
530    /// Tokenize `text`, prepending `bos_token_id()` when `add_bos_token()` is set and absent.
531    pub fn encode_prompt(&self, text: &str) -> Vec<u32> {
532        let mut ids = self.encode(text);
533        if self.add_bos_token && ids.first().copied() != Some(self.bos_token_id) {
534            let mut with_bos = Vec::with_capacity(ids.len().saturating_add(1));
535            with_bos.push(self.bos_token_id);
536            with_bos.append(&mut ids);
537            with_bos
538        } else {
539            ids
540        }
541    }
542
543    /// Detect this model's chat-template family from the special tokens present in its vocab.
544    pub fn chat_family(&self) -> ChatFamily {
545        if self.token_to_id_map.contains_key("<|im_start|>") {
546            ChatFamily::ChatMl
547        } else if self.token_to_id_map.contains_key("<|start_header_id|>") {
548            ChatFamily::Llama3
549        } else if self.token_to_id_map.contains_key("<|turn>")
550            || self.token_to_id_map.contains_key("<turn|>")
551        {
552            // Gemma 4 (before classic Gemma — classic uses <start_of_turn>).
553            ChatFamily::Gemma4
554        } else if self.token_to_id_map.contains_key("<start_of_turn>") {
555            ChatFamily::Gemma
556        } else {
557            ChatFamily::None
558        }
559    }
560
561    /// Wrap a user prompt (and optional system message) in the model's chat template, cueing the
562    /// assistant turn so an instruct model answers instead of degenerating. The tokenizer BOS is
563    /// still prepended by `encode_prompt()`; it is NOT embedded here (avoids a double BOS). Returns
564    /// the raw prompt unchanged when no chat family is recognised.
565    pub fn apply_chat_template(&self, system: Option<&str>, user: &str) -> String {
566        match self.chat_family() {
567            ChatFamily::ChatMl => {
568                let mut s = String::new();
569                if let Some(sys) = system {
570                    s.push_str("<|im_start|>system\n");
571                    s.push_str(sys);
572                    s.push_str("<|im_end|>\n");
573                }
574                s.push_str("<|im_start|>user\n");
575                s.push_str(user);
576                s.push_str("<|im_end|>\n<|im_start|>assistant\n");
577                s
578            }
579            ChatFamily::Llama3 => {
580                let mut s = String::new();
581                if let Some(sys) = system {
582                    s.push_str("<|start_header_id|>system<|end_header_id|>\n\n");
583                    s.push_str(sys);
584                    s.push_str("<|eot_id|>");
585                }
586                s.push_str("<|start_header_id|>user<|end_header_id|>\n\n");
587                s.push_str(user);
588                s.push_str("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n");
589                s
590            }
591            ChatFamily::Gemma => {
592                // Gemma has no system role; fold any system text into the user turn.
593                let mut s = String::from("<start_of_turn>user\n");
594                if let Some(sys) = system {
595                    s.push_str(sys);
596                    s.push_str("\n\n");
597                }
598                s.push_str(user);
599                s.push_str("<end_of_turn>\n<start_of_turn>model\n");
600                s
601            }
602            ChatFamily::Gemma4 => {
603                // Gemma 4 instruct: paired <|turn>…<turn|> markers (see GGUF chat_template).
604                // No dedicated system role — fold system into the user turn. BOS is added by encode.
605                let mut s = String::from("<|turn>user\n");
606                if let Some(sys) = system {
607                    s.push_str(sys);
608                    s.push_str("\n\n");
609                }
610                s.push_str(user);
611                s.push_str("<turn|><|turn>model\n");
612                s
613            }
614            ChatFamily::None => user.to_string(),
615        }
616    }
617
618    /// Apply the model's chat template (if any), then tokenize (+BOS per `add_bos_token`). This is
619    /// the path for interactive chat/instruct inference; `encode_prompt()` stays the raw-completion
620    /// path. Chat models without a recognised family fall back to the raw prompt.
621    pub fn encode_chat_prompt(&self, user: &str) -> Vec<u32> {
622        let templated = self.apply_chat_template(None, user);
623        self.encode_prompt(&templated)
624    }
625
626    /// Apply the model's chat template with explicit system and user roles, then tokenize.
627    pub fn encode_chat_roles(&self, system: Option<&str>, user: &str) -> Vec<u32> {
628        let templated = self.apply_chat_template(system, user);
629        self.encode_prompt(&templated)
630    }
631
632    /// Format token IDs for diagnostic logging (MC3f).
633    pub fn format_ids_for_log(ids: &[u32]) -> String {
634        let mut s = String::from("[");
635        for (i, &id) in ids.iter().enumerate() {
636            if i > 0 {
637                s.push_str(", ");
638            }
639            if i >= 64 {
640                s.push_str("…");
641                break;
642            }
643            s.push_str(&id.to_string());
644        }
645        s.push(']');
646        s
647    }
648
649    /// Greedy longest-match tokenisation; falls back to single-byte encoding.
650    pub fn encode(&self, text: &str) -> Vec<u32> {
651        if self.uses_bpe() {
652            return self.encode_bpe(text);
653        }
654        self.encode_greedy(text)
655    }
656
657    fn uses_bpe(&self) -> bool {
658        !self.merge_pairs.is_empty()
659            || matches!(
660                self.pre_type.as_str(),
661                "smollm" | "gpt2" | "mpt" | "olmo" | "jais" | "llama3"
662            )
663    }
664
665    fn encode_greedy(&self, text: &str) -> Vec<u32> {
666        let mut normalized = String::new();
667        let text_to_scan = if self.token_to_id_map.contains_key("\u{2581}") && text.contains(' ') {
668            normalized.reserve(text.len() + 16);
669            let mut in_special = false;
670            for ch in text.chars() {
671                if ch == '<' {
672                    in_special = true;
673                    normalized.push(ch);
674                } else if ch == '>' && in_special {
675                    in_special = false;
676                    normalized.push(ch);
677                } else if ch == ' ' && !in_special {
678                    normalized.push('\u{2581}');
679                } else {
680                    normalized.push(ch);
681                }
682            }
683            normalized.as_str()
684        } else {
685            text
686        };
687        let mut ids = Vec::new();
688        let mut remaining = text_to_scan;
689        while !remaining.is_empty() {
690            let mut matched = false;
691            for (token, id) in &self.token_to_id {
692                if remaining.starts_with(token.as_str()) {
693                    ids.push(*id);
694                    remaining = &remaining[token.len()..];
695                    matched = true;
696                    break;
697                }
698            }
699            if !matched {
700                let b = remaining.as_bytes()[0];
701                ids.push(b as u32);
702                let step = remaining.chars().next().map(|c| c.len_utf8()).unwrap_or(1);
703                remaining = &remaining[step..];
704            }
705        }
706        ids
707    }
708
709    /// BPE encode with special-token atomicity + smollm/gpt2 pretokenization.
710    fn encode_bpe(&self, text: &str) -> Vec<u32> {
711        let mut ids = Vec::new();
712        // One bounded span workspace replaces regex captures and one String per piece.
713        let mut spans = vec![PretokenSpan::default(); text.len().max(1)];
714        let mut remaining = text;
715        while !remaining.is_empty() {
716            let mut matched_special = false;
717            for (tok, id) in &self.special_tokens {
718                if remaining.starts_with(tok.as_str()) {
719                    ids.push(*id);
720                    remaining = &remaining[tok.len()..];
721                    matched_special = true;
722                    break;
723                }
724            }
725            if matched_special {
726                continue;
727            }
728            let mut next_special = remaining.len();
729            for (tok, _) in &self.special_tokens {
730                if let Some(pos) = remaining.find(tok.as_str()) {
731                    next_special = next_special.min(pos);
732                }
733            }
734            let segment = &remaining[..next_special];
735            if !segment.is_empty() {
736                let count = pretokenizer::pretokenize_for(&self.pre_type, segment, &mut spans)
737                    .expect("span workspace covers the segment byte length");
738                for span in &spans[..count] {
739                    ids.extend(self.bpe_piece(span.get(segment).unwrap()));
740                }
741            }
742            remaining = &remaining[next_special..];
743        }
744        ids
745    }
746
747    /// Regex-free llama.cpp `LLAMA_VOCAB_PRE_TYPE_SMOLLM`-compatible borrowed-span split.
748    pub fn pretokenize_into(
749        &self,
750        text: &str,
751        out: &mut [PretokenSpan],
752    ) -> Result<usize, PretokenError> {
753        pretokenizer::pretokenize_into(text, out)
754    }
755
756    fn bpe_piece(&self, piece: &str) -> Vec<u32> {
757        if piece.is_empty() {
758            return Vec::new();
759        }
760        if self.token_to_id_map.contains_key("\u{2581}") && piece.contains(' ') {
761            let sp_word = piece.replace(' ', "\u{2581}");
762            if let Some(&id) = self.token_to_id_map.get(sp_word.as_str()) {
763                return vec![id];
764            }
765            let mut symbols: Vec<String> = sp_word.chars().map(|c| c.to_string()).collect();
766            loop {
767                let mut best_rank: Option<usize> = None;
768                let mut best_idx = 0usize;
769                for i in 0..symbols.len().saturating_sub(1) {
770                    if let Some(rank) = self.merge_rank_str(&symbols[i], &symbols[i + 1]) {
771                        if best_rank.is_none() || rank < best_rank.unwrap() {
772                            best_rank = Some(rank);
773                            best_idx = i;
774                        }
775                    }
776                }
777                let Some(_rank) = best_rank else { break };
778                let merged = format!("{}{}", symbols[best_idx], symbols[best_idx + 1]);
779                symbols[best_idx] = merged;
780                symbols.remove(best_idx + 1);
781            }
782            let mut ids = Vec::with_capacity(symbols.len());
783            for sym in symbols {
784                if let Some(&id) = self.token_to_id_map.get(sym.as_str()) {
785                    ids.push(id);
786                } else {
787                    for ch in sym.chars() {
788                        let s = ch.to_string();
789                        if let Some(&id) = self.token_to_id_map.get(s.as_str()) {
790                            ids.push(id);
791                        }
792                    }
793                }
794            }
795            return ids;
796        }
797        if let Some(&id) = self.token_to_id_map.get(piece) {
798            return vec![id];
799        }
800        let word: String = piece.bytes().map(gpt2_byte_to_unicode).collect();
801        if let Some(&id) = self.token_to_id_map.get(word.as_str()) {
802            return vec![id];
803        }
804        let mut symbols: Vec<String> = word.chars().map(|c| c.to_string()).collect();
805        if symbols.is_empty() {
806            return Vec::new();
807        }
808        loop {
809            let mut best_rank: Option<usize> = None;
810            let mut best_idx = 0usize;
811            for i in 0..symbols.len().saturating_sub(1) {
812                if let Some(rank) = self.merge_rank_str(&symbols[i], &symbols[i + 1]) {
813                    if best_rank.is_none() || rank < best_rank.unwrap() {
814                        best_rank = Some(rank);
815                        best_idx = i;
816                    }
817                }
818            }
819            let Some(_rank) = best_rank else { break };
820            let merged = format!("{}{}", symbols[best_idx], symbols[best_idx + 1]);
821            symbols[best_idx] = merged;
822            symbols.remove(best_idx + 1);
823        }
824        let mut ids = Vec::with_capacity(symbols.len());
825        for sym in symbols {
826            if let Some(&id) = self.token_to_id_map.get(sym.as_str()) {
827                ids.push(id);
828            } else {
829                for ch in sym.chars() {
830                    let s = ch.to_string();
831                    if let Some(&id) = self.token_to_id_map.get(s.as_str()) {
832                        ids.push(id);
833                    }
834                }
835            }
836        }
837        ids
838    }
839
840    fn merge_rank_str(&self, left: &str, right: &str) -> Option<usize> {
841        if !self.merge_rank_collision {
842            let fingerprint = Self::merge_pair_fingerprint(left, right);
843            if let Some(&rank) = self.merge_rank_index.get(&fingerprint) {
844                let pair = self.merge_pairs.get(rank)?;
845                if pair.0 == left && pair.1 == right {
846                    return Some(rank);
847                }
848                // Defensive exactness if an index built by an older serialized source collides.
849                return self
850                    .merge_pairs
851                    .iter()
852                    .position(|(l, r)| l == left && r == right);
853            }
854            return None;
855        }
856        self.merge_pairs
857            .iter()
858            .position(|(l, r)| l == left && r == right)
859    }
860
861    fn merge_pair_fingerprint(left: &str, right: &str) -> u64 {
862        let mut hash = 0xcbf29ce484222325u64;
863        for byte in (left.len() as u64)
864            .to_le_bytes()
865            .into_iter()
866            .chain(left.bytes())
867            .chain((right.len() as u64).to_le_bytes())
868            .chain(right.bytes())
869        {
870            hash ^= byte as u64;
871            hash = hash.wrapping_mul(0x100000001b3);
872        }
873        hash
874    }
875
876    fn build_merge_rank_index(pairs: &[(String, String)]) -> (HashMap<u64, usize>, bool) {
877        let mut index: HashMap<u64, usize> = HashMap::with_capacity(pairs.len());
878        let mut collision = false;
879        for (rank, (left, right)) in pairs.iter().enumerate() {
880            let fingerprint = Self::merge_pair_fingerprint(left, right);
881            if let Some(&existing) = index.get(&fingerprint) {
882                if pairs[existing].0 != *left || pairs[existing].1 != *right {
883                    collision = true;
884                }
885                continue;
886            }
887            index.insert(fingerprint, rank);
888        }
889        (index, collision)
890    }
891
892    fn parse_merge_pairs(merges: Option<&[String]>) -> Vec<(String, String)> {
893        let Some(merges) = merges else {
894            return Vec::new();
895        };
896        let mut pairs = Vec::with_capacity(merges.len());
897        for merge in merges {
898            if let Some((a, b)) = merge.split_once(' ') {
899                pairs.push((a.to_string(), b.to_string()));
900            }
901        }
902        pairs
903    }
904
905    fn uses_gpt2_byte_decoder(&self) -> bool {
906        matches!(
907            self.pre_type.to_ascii_lowercase().as_str(),
908            "gpt2"
909                | "smollm"
910                | "qwen2"
911                | "qwen35"
912                | "llama-bpe"
913                | "deepseek-llm"
914                | "deepseek-coder"
915                | "falcon"
916                | "starcoder"
917        )
918    }
919
920    fn append_decoded_token_bytes(&self, out: &mut Vec<u8>, s: &str) {
921        if s.len() == 6 && s.starts_with("<0x") && s.ends_with('>') {
922            if let Ok(byte) = u8::from_str_radix(&s[3..5], 16) {
923                out.push(byte);
924            }
925        } else if self.uses_gpt2_byte_decoder() {
926            for symbol in s.chars() {
927                if let Some(byte) = gpt2_unicode_to_byte(symbol) {
928                    out.push(byte);
929                } else {
930                    let mut encoded = [0u8; 4];
931                    out.extend_from_slice(symbol.encode_utf8(&mut encoded).as_bytes());
932                }
933            }
934        } else if let Some(rest) = s.strip_prefix('\u{2581}') {
935            out.push(b' ');
936            out.extend_from_slice(rest.as_bytes());
937        } else if let Some(rest) = s.strip_prefix('\u{0120}') {
938            out.push(b' ');
939            out.extend_from_slice(rest.as_bytes());
940        } else {
941            out.extend_from_slice(s.as_bytes());
942        }
943    }
944
945    /// Append one vocabulary token to `out` with BPE / SentencePiece space normalization.
946    #[allow(dead_code)]
947    fn append_decoded_token(out: &mut String, s: &str) {
948        if let Some(rest) = s.strip_prefix('\u{2581}') {
949            out.push(' ');
950            out.push_str(rest);
951        } else if let Some(rest) = s.strip_prefix('\u{0120}') {
952            // GPT-2 / Llama / SmolLM BPE space marker (Ġ).
953            out.push(' ');
954            out.push_str(rest);
955        } else if s.len() == 6 && s.starts_with("<0x") && s.ends_with('>') {
956            if let Ok(b) = u8::from_str_radix(&s[3..5], 16) {
957                out.push(b as char);
958            }
959        } else {
960            out.push_str(s);
961        }
962    }
963
964    /// Map token IDs → strings, joining without separator.
965    /// Converts SentencePiece `▁` and GPT-2 BPE `Ġ` → space; `<0x##>` → raw byte.
966    pub fn decode(&self, ids: &[u32]) -> String {
967        let mut out = Vec::new();
968        for &id in ids {
969            let s = self
970                .vocab
971                .get(id as usize)
972                .map(|s| s.as_str())
973                .unwrap_or("");
974            self.append_decoded_token_bytes(&mut out, s);
975        }
976        String::from_utf8_lossy(&out).into_owned()
977    }
978
979    /// Decode one token into the exact byte piece used by comparator APIs.
980    ///
981    /// This allocates and is intended for cold diagnostics, receipts, and corpus comparison,
982    /// never for the token-forward hot path.
983    pub fn decode_token_bytes_cold(&self, id: u32) -> Vec<u8> {
984        let mut out = Vec::new();
985        if let Some(token) = self.vocab.get(id as usize) {
986            self.append_decoded_token_bytes(&mut out, token);
987        }
988        out
989    }
990
991    /// `vocab` length.
992    pub fn vocab_len(&self) -> u32 {
993        self.vocab.len() as u32
994    }
995
996    /// Number of BPE merges loaded from GGUF (diagnostic).
997    pub fn merge_count(&self) -> usize {
998        self.merge_pairs.len()
999    }
1000
1001    // ── internal KV parsers ──────────────────────────────────────────────────
1002
1003    fn read_string_array(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<Vec<String>> {
1004        if vtype != 9 {
1005            Self::skip_value(mmap, pos, vtype)?;
1006            return None;
1007        }
1008        if *pos + 12 > mmap.len() {
1009            return None;
1010        }
1011        let etype = u32::from_le_bytes(mmap[*pos..*pos + 4].try_into().ok()?);
1012        *pos += 4;
1013        let count = u64::from_le_bytes(mmap[*pos..*pos + 8].try_into().ok()?) as usize;
1014        *pos += 8;
1015        if etype != 8 {
1016            return None;
1017        } // must be STRING array
1018        let mut result = Vec::with_capacity(count.min(256_000));
1019        for _ in 0..count {
1020            if *pos + 8 > mmap.len() {
1021                break;
1022            }
1023            let slen = u64::from_le_bytes(mmap[*pos..*pos + 8].try_into().ok()?) as usize;
1024            *pos += 8;
1025            if *pos + slen > mmap.len() {
1026                break;
1027            }
1028            let s = std::str::from_utf8(&mmap[*pos..*pos + slen])
1029                .unwrap_or("<?>")
1030                .to_string();
1031            *pos += slen;
1032            result.push(s);
1033        }
1034        Some(result)
1035    }
1036
1037    fn read_u32_val(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<u32> {
1038        if vtype == 4 {
1039            if *pos + 4 > mmap.len() {
1040                return None;
1041            }
1042            let v = u32::from_le_bytes(mmap[*pos..*pos + 4].try_into().ok()?);
1043            *pos += 4;
1044            Some(v)
1045        } else {
1046            Self::skip_value(mmap, pos, vtype)?;
1047            None
1048        }
1049    }
1050
1051    fn read_string_val(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<String> {
1052        if vtype == 8 {
1053            if *pos + 8 > mmap.len() {
1054                return None;
1055            }
1056            let slen = u64::from_le_bytes(mmap[*pos..*pos + 8].try_into().ok()?) as usize;
1057            *pos += 8;
1058            if *pos + slen > mmap.len() {
1059                return None;
1060            }
1061            let s = std::str::from_utf8(&mmap[*pos..*pos + slen])
1062                .unwrap_or("")
1063                .to_string();
1064            *pos += slen;
1065            Some(s)
1066        } else {
1067            Self::skip_value(mmap, pos, vtype)?;
1068            None
1069        }
1070    }
1071
1072    fn read_bool_val(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<bool> {
1073        if vtype == 7 {
1074            if *pos + 1 > mmap.len() {
1075                return None;
1076            }
1077            let b = mmap[*pos];
1078            *pos += 1;
1079            Some(b != 0)
1080        } else {
1081            Self::skip_value(mmap, pos, vtype)?;
1082            None
1083        }
1084    }
1085
1086    fn skip_value(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<()> {
1087        gguf_skip_value(mmap, pos, vtype)
1088    }
1089}