Skip to main content

qualia_core_db/nlp/
tokenize.rs

1//! UTF-8 tokenizer + sentence split. Offsets are byte spans.
2
3use super::span::DocSpan;
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6pub enum TokenKind {
7    Word,
8    Number,
9    Punct,
10    Other,
11}
12
13#[derive(Debug, Clone, PartialEq, Eq)]
14pub struct Token<'a> {
15    pub kind: TokenKind,
16    pub text: &'a str,
17    pub span: DocSpan,
18}
19
20#[derive(Debug, Clone, PartialEq, Eq)]
21pub struct Sentence {
22    pub span: DocSpan,
23}
24
25/// `?::tokenize`.
26pub fn tokenize(source: &str) -> Vec<Token<'_>> {
27    let mut out = Vec::new();
28    let bytes = source.as_bytes();
29    let mut i = 0;
30    while i < bytes.len() {
31        let ch = source[i..].chars().next().unwrap();
32        let len = ch.len_utf8();
33        if ch.is_whitespace() {
34            i += len;
35            continue;
36        }
37        if ch.is_ascii_digit() {
38            let start = i;
39            i += len;
40            while i < bytes.len() {
41                let c = source[i..].chars().next().unwrap();
42                if c.is_ascii_digit() || c == '.' || c == ',' {
43                    i += c.len_utf8();
44                } else {
45                    break;
46                }
47            }
48            out.push(Token {
49                kind: TokenKind::Number,
50                text: &source[start..i],
51                span: DocSpan::new(start as u32, i as u32),
52            });
53            continue;
54        }
55        if is_word_start(ch) {
56            let start = i;
57            i += len;
58            while i < bytes.len() {
59                let c = source[i..].chars().next().unwrap();
60                if is_word_cont(c) {
61                    i += c.len_utf8();
62                } else {
63                    break;
64                }
65            }
66            out.push(Token {
67                kind: TokenKind::Word,
68                text: &source[start..i],
69                span: DocSpan::new(start as u32, i as u32),
70            });
71            continue;
72        }
73        if is_punct_token(ch) {
74            out.push(Token {
75                kind: TokenKind::Punct,
76                text: &source[i..i + len],
77                span: DocSpan::new(i as u32, (i + len) as u32),
78            });
79            i += len;
80            continue;
81        }
82        out.push(Token {
83            kind: TokenKind::Other,
84            text: &source[i..i + len],
85            span: DocSpan::new(i as u32, (i + len) as u32),
86        });
87        i += len;
88    }
89    out
90}
91
92/// Splits `text` into sentences.
93pub fn split_sentences(source: &str) -> Vec<Sentence> {
94    let mut out = Vec::new();
95    let mut start = 0usize;
96    let bytes = source.as_bytes();
97    let mut i = 0;
98    while i < bytes.len() {
99        let ch = source[i..].chars().next().unwrap();
100        let len = ch.len_utf8();
101        let ender = is_sentence_ender(ch) && !is_ascii_decimal_dot(bytes, i, ch);
102        i += len;
103        if ender {
104            let end = i;
105            let slice = source[start..end].trim();
106            if !slice.is_empty() {
107                let trim_start = start + source[start..end].find(slice).unwrap_or(0);
108                out.push(Sentence {
109                    span: DocSpan::new(trim_start as u32, (trim_start + slice.len()) as u32),
110                });
111            }
112            start = i;
113        }
114    }
115    if start < source.len() {
116        let slice = source[start..].trim();
117        if !slice.is_empty() {
118            let trim_start = start + source[start..].find(slice).unwrap_or(0);
119            out.push(Sentence {
120                span: DocSpan::new(trim_start as u32, (trim_start + slice.len()) as u32),
121            });
122        }
123    }
124    out
125}
126
127fn is_word_start(ch: char) -> bool {
128    ch.is_alphabetic() || ch == '_' || ch == '\''
129}
130
131fn is_word_cont(ch: char) -> bool {
132    ch.is_alphanumeric() || ch == '_' || ch == '\'' || ch == '-'
133}
134
135/// ASCII `.?!` plus CJK/fullwidth terminators used by `split_sentences`.
136fn is_sentence_ender(ch: char) -> bool {
137    matches!(
138        ch,
139        '.' | '!' | '?' | '\n' | '\u{3002}' | '\u{FF01}' | '\u{FF1F}'
140    )
141}
142
143/// ASCII `.` between two ASCII digits is a decimal point, not a sentence end.
144fn is_ascii_decimal_dot(bytes: &[u8], i: usize, ch: char) -> bool {
145    if ch != '.' {
146        return false;
147    }
148    let prev_digit = i > 0 && bytes[i - 1].is_ascii_digit();
149    let next = i + 1;
150    let next_digit = next < bytes.len() && bytes[next].is_ascii_digit();
151    prev_digit && next_digit
152}
153
154fn is_punct_token(ch: char) -> bool {
155    ch.is_ascii_punctuation() || matches!(ch, '\u{3002}' | '\u{FF01}' | '\u{FF1F}')
156}
157
158#[cfg(test)]
159mod tests {
160    use super::*;
161
162    #[test]
163    fn tokenizes_words_and_numbers() {
164        let src = "North Spring recorded 12.5 mm.";
165        let toks = tokenize(src);
166        assert!(toks.iter().any(|t| t.text == "North"));
167        assert!(toks.iter().any(|t| t.text == "Spring"));
168        assert!(toks.iter().any(|t| t.text == "12.5"));
169        let north = toks.iter().find(|t| t.text == "North").unwrap();
170        assert_eq!(&src[north.span.as_range()], "North");
171    }
172
173    #[test]
174    fn splits_two_sentences() {
175        let src = "One. Two!";
176        let sents = split_sentences(src);
177        assert_eq!(sents.len(), 2);
178        assert_eq!(&src[sents[0].span.as_range()], "One.");
179        assert_eq!(&src[sents[1].span.as_range()], "Two!");
180    }
181
182    #[test]
183    fn splits_decimal_not_sentence() {
184        let src = "Recorded 12.5 mm. Next.";
185        let sents = split_sentences(src);
186        assert_eq!(sents.len(), 2, "decimal point must not split a sentence");
187        assert_eq!(&src[sents[0].span.as_range()], "Recorded 12.5 mm.");
188        assert_eq!(&src[sents[1].span.as_range()], "Next.");
189    }
190
191    #[test]
192    fn splits_unicode_terminators() {
193        let src = "你好。世界!";
194        let sents = split_sentences(src);
195        assert_eq!(sents.len(), 2);
196        assert_eq!(&src[sents[0].span.as_range()], "你好。");
197        assert_eq!(&src[sents[1].span.as_range()], "世界!");
198    }
199
200    #[test]
201    fn unicode_terminators_are_punct() {
202        let src = "你好。世界!吗?";
203        let toks = tokenize(src);
204        let punct: Vec<&str> = toks
205            .iter()
206            .filter(|t| t.kind == TokenKind::Punct)
207            .map(|t| t.text)
208            .collect();
209        assert_eq!(punct, vec!["。", "!", "?"]);
210        for t in toks.iter().filter(|t| t.kind == TokenKind::Punct) {
211            assert_eq!(&src[t.span.as_range()], t.text);
212        }
213    }
214}