qualia_core_db/inference/gguf_sharder/tokenizer/
decode.rs

1//! Caller-buffered token-piece decoding for autoregressive hot paths.
2
3use super::{gpt2_unicode_to_byte, GgufTokenizer};
4
5impl GgufTokenizer {
6    /// Decode one token into caller-owned storage.
7    ///
8    /// Returns `None` for an unknown token or when `out` is too small. A token's decoded byte
9    /// representation is never longer than its vocabulary string, so callers can use a bounded
10    /// stack buffer without intermediate heap allocation.
11    pub fn decode_token_bytes_into(&self, id: u32, out: &mut [u8]) -> Option<usize> {
12        let token = self.vocab.get(id as usize)?.as_str();
13        if out.len() < token.len() {
14            return None;
15        }
16
17        if token.len() == 6 && token.starts_with("<0x") && token.ends_with('>') {
18            let byte = u8::from_str_radix(&token[3..5], 16).ok()?;
19            out[0] = byte;
20            return Some(1);
21        }
22
23        if self.uses_gpt2_byte_decoder() {
24            let mut written = 0usize;
25            for symbol in token.chars() {
26                if let Some(byte) = gpt2_unicode_to_byte(symbol) {
27                    out[written] = byte;
28                    written += 1;
29                } else {
30                    let mut encoded = [0u8; 4];
31                    let bytes = symbol.encode_utf8(&mut encoded).as_bytes();
32                    out[written..written + bytes.len()].copy_from_slice(bytes);
33                    written += bytes.len();
34                }
35            }
36            return Some(written);
37        }
38
39        if let Some(rest) = token
40            .strip_prefix('\u{2581}')
41            .or_else(|| token.strip_prefix('\u{0120}'))
42        {
43            out[0] = b' ';
44            out[1..1 + rest.len()].copy_from_slice(rest.as_bytes());
45            return Some(1 + rest.len());
46        }
47
48        out[..token.len()].copy_from_slice(token.as_bytes());
49        Some(token.len())
50    }
51}
qualia_core_db/inference/gguf_sharder/tokenizer/decode.rs

qualia_core_db/inference/gguf_sharder/tokenizer/
decode.rs
