1use super::gguf_skip_value;
6use std::collections::HashMap;
7use std::sync::OnceLock;
8
9mod decode;
10mod prepared;
11pub mod hf_json;
12mod pretokenizer;
13pub use pretokenizer::{PretokenError, PretokenSpan};
14
15fn gpt2_byte_to_unicode(byte: u8) -> char {
17 static TABLE: OnceLock<[char; 256]> = OnceLock::new();
18 TABLE.get_or_init(|| {
19 let mut bs: Vec<u32> = (b'!'..=b'~')
20 .chain(b'\xA1'..=b'\xAC')
21 .chain(b'\xAE'..=b'\xFF')
22 .map(|b| b as u32)
23 .collect();
24 let mut cs = bs.clone();
25 let mut n = 0u32;
26 for b in 0u32..256 {
27 if !bs.contains(&b) {
28 bs.push(b);
29 cs.push(256 + n);
30 n += 1;
31 }
32 }
33 let mut out = ['\0'; 256];
34 for (b, c) in bs.into_iter().zip(cs) {
35 out[b as usize] = char::from_u32(c).unwrap_or('\u{FFFD}');
36 }
37 out
38 })[byte as usize]
39}
40
41fn gpt2_unicode_to_byte(symbol: char) -> Option<u8> {
42 (0u16..=255)
43 .find(|byte| gpt2_byte_to_unicode(*byte as u8) == symbol)
44 .map(|byte| byte as u8)
45}
46
47pub const MAX_STOP_TOKEN_IDS: usize = 8;
49
50pub struct GgufTokenizer {
53 pub vocab: Vec<String>,
55 pub bos_token_id: u32,
56 pub eos_token_id: u32,
57 pub add_bos_token: bool,
59 pub pre_type: String,
61 merge_pairs: Vec<(String, String)>,
63 merge_rank_index: HashMap<u64, usize>,
66 merge_rank_collision: bool,
67 pub(super) token_to_id_map: HashMap<String, u32>,
69 special_tokens: Vec<(String, u32)>,
71 pub(super) token_to_id: Vec<(String, u32)>,
73 stop_token_ids: [u32; MAX_STOP_TOKEN_IDS],
77 stop_token_count: u8,
78}
79
80#[derive(Clone, Copy, PartialEq, Eq, Debug)]
84pub enum ChatFamily {
85 ChatMl,
87 Llama3,
89 Gemma,
91 Gemma4,
93 None,
95}
96
97impl ChatFamily {
98 pub const fn as_str(&self) -> &'static str {
100 match self {
101 Self::ChatMl => "chatml",
102 Self::Llama3 => "llama3",
103 Self::Gemma => "gemma",
104 Self::Gemma4 => "gemma4",
105 Self::None => "none",
106 }
107 }
108}
109
110impl Default for GgufTokenizer {
111 fn default() -> Self {
113 let vocab: Vec<String> = (0u32..256)
114 .map(|b| {
115 let c = b as u8;
116 if c.is_ascii_graphic() || c == b' ' {
117 (c as char).to_string()
118 } else {
119 format!("<0x{:02X}>", b)
120 }
121 })
122 .collect();
123 let mut t2id: Vec<(String, u32)> = vocab
124 .iter()
125 .enumerate()
126 .map(|(i, s)| (s.clone(), i as u32))
127 .collect();
128 t2id.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
129 let token_to_id_map: HashMap<String, u32> =
130 t2id.iter().map(|(s, id)| (s.clone(), *id)).collect();
131 let mut tok = Self {
132 vocab,
133 bos_token_id: 1,
134 eos_token_id: 2,
135 add_bos_token: true,
136 pre_type: String::new(),
137 merge_pairs: Vec::new(),
138 merge_rank_index: HashMap::new(),
139 merge_rank_collision: false,
140 token_to_id_map,
141 special_tokens: Vec::new(),
142 token_to_id: t2id,
143 stop_token_ids: [0; MAX_STOP_TOKEN_IDS],
144 stop_token_count: 0,
145 };
146 tok.rebuild_stop_token_ids();
147 tok
148 }
149}
150
151impl GgufTokenizer {
152 pub fn from_gguf(mmap: &[u8]) -> Self {
155 Self::try_parse(mmap).unwrap_or_default()
156 }
157
158 fn try_parse(mmap: &[u8]) -> Option<Self> {
159 if mmap.len() < 24 || &mmap[0..4] != b"GGUF" {
160 return None;
161 }
162 let version = u32::from_le_bytes(mmap[4..8].try_into().ok()?);
163 if version < 2 {
164 return None;
165 } let kv_count = u64::from_le_bytes(mmap[16..24].try_into().ok()?);
167 let mut pos = 24usize;
168 let mut vocab: Option<Vec<String>> = None;
169 let mut merges_raw: Option<Vec<String>> = None;
170 let mut bos_id: Option<u32> = None;
171 let mut eos_id: Option<u32> = None;
172 let mut add_bos: Option<bool> = None;
173 let mut pre_type: Option<String> = None;
174
175 for _ in 0..kv_count {
176 if pos + 8 > mmap.len() {
177 break;
178 }
179 let klen = u64::from_le_bytes(mmap[pos..pos + 8].try_into().ok()?) as usize;
180 pos += 8;
181 if pos + klen > mmap.len() {
182 break;
183 }
184 let key = std::str::from_utf8(&mmap[pos..pos + klen]).unwrap_or("");
185 pos += klen;
186 if pos + 4 > mmap.len() {
187 break;
188 }
189 let vtype = u32::from_le_bytes(mmap[pos..pos + 4].try_into().ok()?);
190 pos += 4;
191 match key {
192 "tokenizer.ggml.tokens" => {
193 vocab = Self::read_string_array(mmap, &mut pos, vtype);
194 }
195 "tokenizer.ggml.merges" => {
196 merges_raw = Self::read_string_array(mmap, &mut pos, vtype);
197 }
198 "tokenizer.ggml.bos_token_id" => {
199 bos_id = Self::read_u32_val(mmap, &mut pos, vtype);
200 }
201 "tokenizer.ggml.eos_token_id" => {
202 eos_id = Self::read_u32_val(mmap, &mut pos, vtype);
203 }
204 "tokenizer.ggml.add_bos_token" => {
205 add_bos = Self::read_bool_val(mmap, &mut pos, vtype);
206 }
207 "tokenizer.ggml.pre" => {
208 pre_type = Self::read_string_val(mmap, &mut pos, vtype);
209 }
210 _ => {
211 if Self::skip_value(mmap, &mut pos, vtype).is_none() {
212 break;
213 }
214 }
215 }
216 }
217
218 let v = vocab?;
219 Some(Self::from_raw_components(
220 v,
221 merges_raw.as_deref(),
222 bos_id,
223 eos_id,
224 add_bos,
225 pre_type,
226 ))
227 }
228
229 pub fn from_raw_components(
231 vocab: Vec<String>,
232 merges_raw: Option<&[String]>,
233 bos_id: Option<u32>,
234 eos_id: Option<u32>,
235 add_bos: Option<bool>,
236 pre_type: Option<String>,
237 ) -> Self {
238 let bos = bos_id.unwrap_or(1);
239 let eos = eos_id.unwrap_or(2);
240 let mut t2id: Vec<(String, u32)> = vocab
241 .iter()
242 .enumerate()
243 .map(|(i, s)| (s.clone(), i as u32))
244 .collect();
245 t2id.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
246 let token_to_id_map: HashMap<String, u32> =
247 t2id.iter().map(|(s, id)| (s.clone(), *id)).collect();
248 let mut special_tokens: Vec<(String, u32)> = vocab
249 .iter()
250 .enumerate()
251 .filter(|(_, s)| s.starts_with('<') && s.ends_with('>'))
252 .map(|(i, s)| (s.clone(), i as u32))
253 .collect();
254 special_tokens.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
255 let merge_pairs = Self::parse_merge_pairs(merges_raw);
256 let (merge_rank_index, merge_rank_collision) = Self::build_merge_rank_index(&merge_pairs);
257 let default_add_bos = bos_id.is_some()
258 || vocab
259 .get(bos as usize)
260 .is_some_and(|s| s.starts_with('<') && s.ends_with('>'))
261 || vocab.len() <= 256;
262 let mut tok = Self {
263 vocab,
264 bos_token_id: bos,
265 eos_token_id: eos,
266 add_bos_token: add_bos.unwrap_or(default_add_bos),
267 pre_type: pre_type.unwrap_or_default(),
268 merge_pairs,
269 merge_rank_index,
270 merge_rank_collision,
271 token_to_id_map,
272 special_tokens,
273 token_to_id: t2id,
274 stop_token_ids: [0; MAX_STOP_TOKEN_IDS],
275 stop_token_count: 0,
276 };
277 tok.rebuild_stop_token_ids();
278 tok
279 }
280
281 pub fn from_hf_json(json_str: &str) -> Option<Self> {
283 hf_json::parse_hf_tokenizer_json(json_str)
284 }
285
286 pub fn to_p64_section(&self) -> Vec<u8> {
291 let mut out = Vec::with_capacity(1 << 20);
292 out.extend_from_slice(b"Q42T");
293 out.extend_from_slice(&2u16.to_le_bytes()); out.extend_from_slice(&0u16.to_le_bytes()); out.extend_from_slice(&self.bos_token_id.to_le_bytes());
296 out.extend_from_slice(&self.eos_token_id.to_le_bytes());
297 out.push(self.add_bos_token as u8);
298 out.extend_from_slice(&[0u8; 3]);
299 let put_str = |o: &mut Vec<u8>, s: &str| {
300 o.extend_from_slice(&(s.len() as u32).to_le_bytes());
301 o.extend_from_slice(s.as_bytes());
302 };
303 put_str(&mut out, &self.pre_type);
304 out.extend_from_slice(&(self.vocab.len() as u32).to_le_bytes());
305 for t in &self.vocab {
306 put_str(&mut out, t);
307 }
308 out.extend_from_slice(&(self.merge_pairs.len() as u32).to_le_bytes());
309 for (l, r) in &self.merge_pairs {
310 put_str(&mut out, l);
311 put_str(&mut out, r);
312 }
313 out.push(self.stop_token_count);
315 out.extend_from_slice(&[0u8; 3]);
316 for id in &self.stop_token_ids {
317 out.extend_from_slice(&id.to_le_bytes());
318 }
319 out
320 }
321
322 #[deprecated(note = "use to_p64_section")]
324 pub fn to_q42_section(&self) -> Vec<u8> {
325 self.to_p64_section()
326 }
327
328 pub fn from_p64_section(data: &[u8]) -> Option<Self> {
332 let mut p = 0usize;
333 let take = |p: &mut usize, n: usize| -> Option<&[u8]> {
334 let end = p.checked_add(n)?;
335 if end > data.len() {
336 return None;
337 }
338 let s = &data[*p..end];
339 *p = end;
340 Some(s)
341 };
342 let take_u32 = |p: &mut usize| -> Option<u32> {
343 Some(u32::from_le_bytes(take(p, 4)?.try_into().ok()?))
344 };
345 let take_str = |p: &mut usize| -> Option<String> {
346 let len = take_u32(p)? as usize;
347 Some(String::from_utf8_lossy(take(p, len)?).into_owned())
348 };
349 if take(&mut p, 4)? != b"Q42T" {
350 return None;
351 }
352 let ver = u16::from_le_bytes(take(&mut p, 2)?.try_into().ok()?);
353 let _flags = take(&mut p, 2)?;
354 let bos = take_u32(&mut p)?;
355 let eos = take_u32(&mut p)?;
356 let add_bos = take(&mut p, 1)?[0] != 0;
357 let _pad = take(&mut p, 3)?;
358 let pre_type = take_str(&mut p)?;
359 let n_vocab = take_u32(&mut p)? as usize;
360 if n_vocab > 1_000_000 {
361 return None;
362 }
363 let mut vocab = Vec::with_capacity(n_vocab);
364 for _ in 0..n_vocab {
365 vocab.push(take_str(&mut p)?);
366 }
367 let n_merges = take_u32(&mut p)? as usize;
368 if n_merges > 5_000_000 {
369 return None;
370 }
371 let mut merge_pairs = Vec::with_capacity(n_merges);
372 for _ in 0..n_merges {
373 let l = take_str(&mut p)?;
374 let r = take_str(&mut p)?;
375 merge_pairs.push((l, r));
376 }
377 let mut stored_stops: Option<([u32; MAX_STOP_TOKEN_IDS], u8)> = None;
379 if ver >= 2 && p + 4 + MAX_STOP_TOKEN_IDS * 4 <= data.len() {
380 let count = take(&mut p, 1)?[0].min(MAX_STOP_TOKEN_IDS as u8);
381 let _ = take(&mut p, 3)?;
382 let mut ids = [0u32; MAX_STOP_TOKEN_IDS];
383 for i in 0..MAX_STOP_TOKEN_IDS {
384 ids[i] = take_u32(&mut p)?;
385 }
386 stored_stops = Some((ids, count));
387 }
388 let mut t2id: Vec<(String, u32)> = vocab
390 .iter()
391 .enumerate()
392 .map(|(i, s)| (s.clone(), i as u32))
393 .collect();
394 t2id.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
395 let token_to_id_map: HashMap<String, u32> =
396 t2id.iter().map(|(s, id)| (s.clone(), *id)).collect();
397 let mut special_tokens: Vec<(String, u32)> = vocab
398 .iter()
399 .enumerate()
400 .filter(|(_, s)| s.starts_with('<') && s.ends_with('>'))
401 .map(|(i, s)| (s.clone(), i as u32))
402 .collect();
403 special_tokens.sort_by(|a, b| b.0.len().cmp(&a.0.len()));
404 let mut tok = Self {
405 vocab,
406 bos_token_id: bos,
407 eos_token_id: eos,
408 add_bos_token: add_bos,
409 pre_type,
410 merge_pairs,
411 merge_rank_index: HashMap::new(),
412 merge_rank_collision: false,
413 token_to_id_map,
414 special_tokens,
415 token_to_id: t2id,
416 stop_token_ids: [0; MAX_STOP_TOKEN_IDS],
417 stop_token_count: 0,
418 };
419 (tok.merge_rank_index, tok.merge_rank_collision) =
420 Self::build_merge_rank_index(&tok.merge_pairs);
421 if let Some((ids, count)) = stored_stops {
422 tok.stop_token_ids = ids;
423 tok.stop_token_count = count;
424 if !tok.is_stop_token(eos) {
426 tok.rebuild_stop_token_ids();
427 }
428 } else {
429 tok.rebuild_stop_token_ids();
430 }
431 Some(tok)
432 }
433
434 pub fn rebuild_stop_token_ids(&mut self) {
437 let mut ids = [0u32; MAX_STOP_TOKEN_IDS];
438 let mut n = 0usize;
439 let mut push = |id: u32| {
440 if n >= MAX_STOP_TOKEN_IDS {
441 return;
442 }
443 if ids[..n].contains(&id) {
444 return;
445 }
446 ids[n] = id;
447 n += 1;
448 };
449 push(self.eos_token_id);
450 const CHAT_ENDS: &[&str] = &[
453 "<|endoftext|>",
454 "<|eot_id|>",
455 "<|im_end|>",
456 "<end_of_turn>",
457 "<turn|>", "<|end_of_text|>",
459 "</s>",
460 "<|end|>",
461 ];
462 for s in CHAT_ENDS {
463 if let Some(&id) = self.token_to_id_map.get(*s) {
464 push(id);
465 }
466 }
467 let looks_like_chat_end = |n: &str| -> bool {
470 let l = n.to_ascii_lowercase();
471 l == "<|im_end|>"
472 || l == "<|eot_id|>"
473 || l == "<|endoftext|>"
474 || l == "<end_of_turn>"
475 || l == "</s>"
476 || l.ends_with("im_end|>")
477 || l.ends_with("eot_id|>")
478 || l.contains("end_of_turn")
479 || l.contains("im_end")
480 || l.contains("eot_id")
481 };
482 for (name, id) in &self.special_tokens {
483 if looks_like_chat_end(name) {
484 push(*id);
485 }
486 }
487 for (name, id) in &self.token_to_id_map {
488 if looks_like_chat_end(name) {
489 push(*id);
490 }
491 }
492 self.stop_token_ids = ids;
493 self.stop_token_count = n as u8;
494 }
495
496 #[inline]
498 pub fn is_stop_token(&self, id: u32) -> bool {
499 let n = self.stop_token_count as usize;
500 self.stop_token_ids[..n].contains(&id)
501 }
502
503 pub fn stop_tokens(&self) -> &[u32] {
505 &self.stop_token_ids[..self.stop_token_count as usize]
506 }
507
508 pub fn merge_stop_token_ids(&mut self, extra: &[u32]) {
511 let mut n = self.stop_token_count as usize;
512 for &id in extra {
513 if n >= MAX_STOP_TOKEN_IDS {
514 break;
515 }
516 if self.stop_token_ids[..n].contains(&id) {
517 continue;
518 }
519 self.stop_token_ids[n] = id;
520 n += 1;
521 }
522 if !self.stop_token_ids[..n].contains(&self.eos_token_id) && n < MAX_STOP_TOKEN_IDS {
524 self.stop_token_ids[n] = self.eos_token_id;
525 n += 1;
526 }
527 self.stop_token_count = n as u8;
528 }
529
530 pub fn encode_prompt(&self, text: &str) -> Vec<u32> {
532 let mut ids = self.encode(text);
533 if self.add_bos_token && ids.first().copied() != Some(self.bos_token_id) {
534 let mut with_bos = Vec::with_capacity(ids.len().saturating_add(1));
535 with_bos.push(self.bos_token_id);
536 with_bos.append(&mut ids);
537 with_bos
538 } else {
539 ids
540 }
541 }
542
543 pub fn chat_family(&self) -> ChatFamily {
545 if self.token_to_id_map.contains_key("<|im_start|>") {
546 ChatFamily::ChatMl
547 } else if self.token_to_id_map.contains_key("<|start_header_id|>") {
548 ChatFamily::Llama3
549 } else if self.token_to_id_map.contains_key("<|turn>")
550 || self.token_to_id_map.contains_key("<turn|>")
551 {
552 ChatFamily::Gemma4
554 } else if self.token_to_id_map.contains_key("<start_of_turn>") {
555 ChatFamily::Gemma
556 } else {
557 ChatFamily::None
558 }
559 }
560
561 pub fn apply_chat_template(&self, system: Option<&str>, user: &str) -> String {
566 match self.chat_family() {
567 ChatFamily::ChatMl => {
568 let mut s = String::new();
569 if let Some(sys) = system {
570 s.push_str("<|im_start|>system\n");
571 s.push_str(sys);
572 s.push_str("<|im_end|>\n");
573 }
574 s.push_str("<|im_start|>user\n");
575 s.push_str(user);
576 s.push_str("<|im_end|>\n<|im_start|>assistant\n");
577 s
578 }
579 ChatFamily::Llama3 => {
580 let mut s = String::new();
581 if let Some(sys) = system {
582 s.push_str("<|start_header_id|>system<|end_header_id|>\n\n");
583 s.push_str(sys);
584 s.push_str("<|eot_id|>");
585 }
586 s.push_str("<|start_header_id|>user<|end_header_id|>\n\n");
587 s.push_str(user);
588 s.push_str("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n");
589 s
590 }
591 ChatFamily::Gemma => {
592 let mut s = String::from("<start_of_turn>user\n");
594 if let Some(sys) = system {
595 s.push_str(sys);
596 s.push_str("\n\n");
597 }
598 s.push_str(user);
599 s.push_str("<end_of_turn>\n<start_of_turn>model\n");
600 s
601 }
602 ChatFamily::Gemma4 => {
603 let mut s = String::from("<|turn>user\n");
606 if let Some(sys) = system {
607 s.push_str(sys);
608 s.push_str("\n\n");
609 }
610 s.push_str(user);
611 s.push_str("<turn|><|turn>model\n");
612 s
613 }
614 ChatFamily::None => user.to_string(),
615 }
616 }
617
618 pub fn encode_chat_prompt(&self, user: &str) -> Vec<u32> {
622 let templated = self.apply_chat_template(None, user);
623 self.encode_prompt(&templated)
624 }
625
626 pub fn encode_chat_roles(&self, system: Option<&str>, user: &str) -> Vec<u32> {
628 let templated = self.apply_chat_template(system, user);
629 self.encode_prompt(&templated)
630 }
631
632 pub fn format_ids_for_log(ids: &[u32]) -> String {
634 let mut s = String::from("[");
635 for (i, &id) in ids.iter().enumerate() {
636 if i > 0 {
637 s.push_str(", ");
638 }
639 if i >= 64 {
640 s.push_str("…");
641 break;
642 }
643 s.push_str(&id.to_string());
644 }
645 s.push(']');
646 s
647 }
648
649 pub fn encode(&self, text: &str) -> Vec<u32> {
651 if self.uses_bpe() {
652 return self.encode_bpe(text);
653 }
654 self.encode_greedy(text)
655 }
656
657 fn uses_bpe(&self) -> bool {
658 !self.merge_pairs.is_empty()
659 || matches!(
660 self.pre_type.as_str(),
661 "smollm" | "gpt2" | "mpt" | "olmo" | "jais" | "llama3"
662 )
663 }
664
665 fn encode_greedy(&self, text: &str) -> Vec<u32> {
666 let mut normalized = String::new();
667 let text_to_scan = if self.token_to_id_map.contains_key("\u{2581}") && text.contains(' ') {
668 normalized.reserve(text.len() + 16);
669 let mut in_special = false;
670 for ch in text.chars() {
671 if ch == '<' {
672 in_special = true;
673 normalized.push(ch);
674 } else if ch == '>' && in_special {
675 in_special = false;
676 normalized.push(ch);
677 } else if ch == ' ' && !in_special {
678 normalized.push('\u{2581}');
679 } else {
680 normalized.push(ch);
681 }
682 }
683 normalized.as_str()
684 } else {
685 text
686 };
687 let mut ids = Vec::new();
688 let mut remaining = text_to_scan;
689 while !remaining.is_empty() {
690 let mut matched = false;
691 for (token, id) in &self.token_to_id {
692 if remaining.starts_with(token.as_str()) {
693 ids.push(*id);
694 remaining = &remaining[token.len()..];
695 matched = true;
696 break;
697 }
698 }
699 if !matched {
700 let b = remaining.as_bytes()[0];
701 ids.push(b as u32);
702 let step = remaining.chars().next().map(|c| c.len_utf8()).unwrap_or(1);
703 remaining = &remaining[step..];
704 }
705 }
706 ids
707 }
708
709 fn encode_bpe(&self, text: &str) -> Vec<u32> {
711 let mut ids = Vec::new();
712 let mut spans = vec![PretokenSpan::default(); text.len().max(1)];
714 let mut remaining = text;
715 while !remaining.is_empty() {
716 let mut matched_special = false;
717 for (tok, id) in &self.special_tokens {
718 if remaining.starts_with(tok.as_str()) {
719 ids.push(*id);
720 remaining = &remaining[tok.len()..];
721 matched_special = true;
722 break;
723 }
724 }
725 if matched_special {
726 continue;
727 }
728 let mut next_special = remaining.len();
729 for (tok, _) in &self.special_tokens {
730 if let Some(pos) = remaining.find(tok.as_str()) {
731 next_special = next_special.min(pos);
732 }
733 }
734 let segment = &remaining[..next_special];
735 if !segment.is_empty() {
736 let count = pretokenizer::pretokenize_for(&self.pre_type, segment, &mut spans)
737 .expect("span workspace covers the segment byte length");
738 for span in &spans[..count] {
739 ids.extend(self.bpe_piece(span.get(segment).unwrap()));
740 }
741 }
742 remaining = &remaining[next_special..];
743 }
744 ids
745 }
746
747 pub fn pretokenize_into(
749 &self,
750 text: &str,
751 out: &mut [PretokenSpan],
752 ) -> Result<usize, PretokenError> {
753 pretokenizer::pretokenize_into(text, out)
754 }
755
756 fn bpe_piece(&self, piece: &str) -> Vec<u32> {
757 if piece.is_empty() {
758 return Vec::new();
759 }
760 if self.token_to_id_map.contains_key("\u{2581}") && piece.contains(' ') {
761 let sp_word = piece.replace(' ', "\u{2581}");
762 if let Some(&id) = self.token_to_id_map.get(sp_word.as_str()) {
763 return vec![id];
764 }
765 let mut symbols: Vec<String> = sp_word.chars().map(|c| c.to_string()).collect();
766 loop {
767 let mut best_rank: Option<usize> = None;
768 let mut best_idx = 0usize;
769 for i in 0..symbols.len().saturating_sub(1) {
770 if let Some(rank) = self.merge_rank_str(&symbols[i], &symbols[i + 1]) {
771 if best_rank.is_none() || rank < best_rank.unwrap() {
772 best_rank = Some(rank);
773 best_idx = i;
774 }
775 }
776 }
777 let Some(_rank) = best_rank else { break };
778 let merged = format!("{}{}", symbols[best_idx], symbols[best_idx + 1]);
779 symbols[best_idx] = merged;
780 symbols.remove(best_idx + 1);
781 }
782 let mut ids = Vec::with_capacity(symbols.len());
783 for sym in symbols {
784 if let Some(&id) = self.token_to_id_map.get(sym.as_str()) {
785 ids.push(id);
786 } else {
787 for ch in sym.chars() {
788 let s = ch.to_string();
789 if let Some(&id) = self.token_to_id_map.get(s.as_str()) {
790 ids.push(id);
791 }
792 }
793 }
794 }
795 return ids;
796 }
797 if let Some(&id) = self.token_to_id_map.get(piece) {
798 return vec![id];
799 }
800 let word: String = piece.bytes().map(gpt2_byte_to_unicode).collect();
801 if let Some(&id) = self.token_to_id_map.get(word.as_str()) {
802 return vec![id];
803 }
804 let mut symbols: Vec<String> = word.chars().map(|c| c.to_string()).collect();
805 if symbols.is_empty() {
806 return Vec::new();
807 }
808 loop {
809 let mut best_rank: Option<usize> = None;
810 let mut best_idx = 0usize;
811 for i in 0..symbols.len().saturating_sub(1) {
812 if let Some(rank) = self.merge_rank_str(&symbols[i], &symbols[i + 1]) {
813 if best_rank.is_none() || rank < best_rank.unwrap() {
814 best_rank = Some(rank);
815 best_idx = i;
816 }
817 }
818 }
819 let Some(_rank) = best_rank else { break };
820 let merged = format!("{}{}", symbols[best_idx], symbols[best_idx + 1]);
821 symbols[best_idx] = merged;
822 symbols.remove(best_idx + 1);
823 }
824 let mut ids = Vec::with_capacity(symbols.len());
825 for sym in symbols {
826 if let Some(&id) = self.token_to_id_map.get(sym.as_str()) {
827 ids.push(id);
828 } else {
829 for ch in sym.chars() {
830 let s = ch.to_string();
831 if let Some(&id) = self.token_to_id_map.get(s.as_str()) {
832 ids.push(id);
833 }
834 }
835 }
836 }
837 ids
838 }
839
840 fn merge_rank_str(&self, left: &str, right: &str) -> Option<usize> {
841 if !self.merge_rank_collision {
842 let fingerprint = Self::merge_pair_fingerprint(left, right);
843 if let Some(&rank) = self.merge_rank_index.get(&fingerprint) {
844 let pair = self.merge_pairs.get(rank)?;
845 if pair.0 == left && pair.1 == right {
846 return Some(rank);
847 }
848 return self
850 .merge_pairs
851 .iter()
852 .position(|(l, r)| l == left && r == right);
853 }
854 return None;
855 }
856 self.merge_pairs
857 .iter()
858 .position(|(l, r)| l == left && r == right)
859 }
860
861 fn merge_pair_fingerprint(left: &str, right: &str) -> u64 {
862 let mut hash = 0xcbf29ce484222325u64;
863 for byte in (left.len() as u64)
864 .to_le_bytes()
865 .into_iter()
866 .chain(left.bytes())
867 .chain((right.len() as u64).to_le_bytes())
868 .chain(right.bytes())
869 {
870 hash ^= byte as u64;
871 hash = hash.wrapping_mul(0x100000001b3);
872 }
873 hash
874 }
875
876 fn build_merge_rank_index(pairs: &[(String, String)]) -> (HashMap<u64, usize>, bool) {
877 let mut index: HashMap<u64, usize> = HashMap::with_capacity(pairs.len());
878 let mut collision = false;
879 for (rank, (left, right)) in pairs.iter().enumerate() {
880 let fingerprint = Self::merge_pair_fingerprint(left, right);
881 if let Some(&existing) = index.get(&fingerprint) {
882 if pairs[existing].0 != *left || pairs[existing].1 != *right {
883 collision = true;
884 }
885 continue;
886 }
887 index.insert(fingerprint, rank);
888 }
889 (index, collision)
890 }
891
892 fn parse_merge_pairs(merges: Option<&[String]>) -> Vec<(String, String)> {
893 let Some(merges) = merges else {
894 return Vec::new();
895 };
896 let mut pairs = Vec::with_capacity(merges.len());
897 for merge in merges {
898 if let Some((a, b)) = merge.split_once(' ') {
899 pairs.push((a.to_string(), b.to_string()));
900 }
901 }
902 pairs
903 }
904
905 fn uses_gpt2_byte_decoder(&self) -> bool {
906 matches!(
907 self.pre_type.to_ascii_lowercase().as_str(),
908 "gpt2"
909 | "smollm"
910 | "qwen2"
911 | "qwen35"
912 | "llama-bpe"
913 | "deepseek-llm"
914 | "deepseek-coder"
915 | "falcon"
916 | "starcoder"
917 )
918 }
919
920 fn append_decoded_token_bytes(&self, out: &mut Vec<u8>, s: &str) {
921 if s.len() == 6 && s.starts_with("<0x") && s.ends_with('>') {
922 if let Ok(byte) = u8::from_str_radix(&s[3..5], 16) {
923 out.push(byte);
924 }
925 } else if self.uses_gpt2_byte_decoder() {
926 for symbol in s.chars() {
927 if let Some(byte) = gpt2_unicode_to_byte(symbol) {
928 out.push(byte);
929 } else {
930 let mut encoded = [0u8; 4];
931 out.extend_from_slice(symbol.encode_utf8(&mut encoded).as_bytes());
932 }
933 }
934 } else if let Some(rest) = s.strip_prefix('\u{2581}') {
935 out.push(b' ');
936 out.extend_from_slice(rest.as_bytes());
937 } else if let Some(rest) = s.strip_prefix('\u{0120}') {
938 out.push(b' ');
939 out.extend_from_slice(rest.as_bytes());
940 } else {
941 out.extend_from_slice(s.as_bytes());
942 }
943 }
944
945 #[allow(dead_code)]
947 fn append_decoded_token(out: &mut String, s: &str) {
948 if let Some(rest) = s.strip_prefix('\u{2581}') {
949 out.push(' ');
950 out.push_str(rest);
951 } else if let Some(rest) = s.strip_prefix('\u{0120}') {
952 out.push(' ');
954 out.push_str(rest);
955 } else if s.len() == 6 && s.starts_with("<0x") && s.ends_with('>') {
956 if let Ok(b) = u8::from_str_radix(&s[3..5], 16) {
957 out.push(b as char);
958 }
959 } else {
960 out.push_str(s);
961 }
962 }
963
964 pub fn decode(&self, ids: &[u32]) -> String {
967 let mut out = Vec::new();
968 for &id in ids {
969 let s = self
970 .vocab
971 .get(id as usize)
972 .map(|s| s.as_str())
973 .unwrap_or("");
974 self.append_decoded_token_bytes(&mut out, s);
975 }
976 String::from_utf8_lossy(&out).into_owned()
977 }
978
979 pub fn decode_token_bytes_cold(&self, id: u32) -> Vec<u8> {
984 let mut out = Vec::new();
985 if let Some(token) = self.vocab.get(id as usize) {
986 self.append_decoded_token_bytes(&mut out, token);
987 }
988 out
989 }
990
991 pub fn vocab_len(&self) -> u32 {
993 self.vocab.len() as u32
994 }
995
996 pub fn merge_count(&self) -> usize {
998 self.merge_pairs.len()
999 }
1000
1001 fn read_string_array(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<Vec<String>> {
1004 if vtype != 9 {
1005 Self::skip_value(mmap, pos, vtype)?;
1006 return None;
1007 }
1008 if *pos + 12 > mmap.len() {
1009 return None;
1010 }
1011 let etype = u32::from_le_bytes(mmap[*pos..*pos + 4].try_into().ok()?);
1012 *pos += 4;
1013 let count = u64::from_le_bytes(mmap[*pos..*pos + 8].try_into().ok()?) as usize;
1014 *pos += 8;
1015 if etype != 8 {
1016 return None;
1017 } let mut result = Vec::with_capacity(count.min(256_000));
1019 for _ in 0..count {
1020 if *pos + 8 > mmap.len() {
1021 break;
1022 }
1023 let slen = u64::from_le_bytes(mmap[*pos..*pos + 8].try_into().ok()?) as usize;
1024 *pos += 8;
1025 if *pos + slen > mmap.len() {
1026 break;
1027 }
1028 let s = std::str::from_utf8(&mmap[*pos..*pos + slen])
1029 .unwrap_or("<?>")
1030 .to_string();
1031 *pos += slen;
1032 result.push(s);
1033 }
1034 Some(result)
1035 }
1036
1037 fn read_u32_val(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<u32> {
1038 if vtype == 4 {
1039 if *pos + 4 > mmap.len() {
1040 return None;
1041 }
1042 let v = u32::from_le_bytes(mmap[*pos..*pos + 4].try_into().ok()?);
1043 *pos += 4;
1044 Some(v)
1045 } else {
1046 Self::skip_value(mmap, pos, vtype)?;
1047 None
1048 }
1049 }
1050
1051 fn read_string_val(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<String> {
1052 if vtype == 8 {
1053 if *pos + 8 > mmap.len() {
1054 return None;
1055 }
1056 let slen = u64::from_le_bytes(mmap[*pos..*pos + 8].try_into().ok()?) as usize;
1057 *pos += 8;
1058 if *pos + slen > mmap.len() {
1059 return None;
1060 }
1061 let s = std::str::from_utf8(&mmap[*pos..*pos + slen])
1062 .unwrap_or("")
1063 .to_string();
1064 *pos += slen;
1065 Some(s)
1066 } else {
1067 Self::skip_value(mmap, pos, vtype)?;
1068 None
1069 }
1070 }
1071
1072 fn read_bool_val(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<bool> {
1073 if vtype == 7 {
1074 if *pos + 1 > mmap.len() {
1075 return None;
1076 }
1077 let b = mmap[*pos];
1078 *pos += 1;
1079 Some(b != 0)
1080 } else {
1081 Self::skip_value(mmap, pos, vtype)?;
1082 None
1083 }
1084 }
1085
1086 fn skip_value(mmap: &[u8], pos: &mut usize, vtype: u32) -> Option<()> {
1087 gguf_skip_value(mmap, pos, vtype)
1088 }
1089}