qualia_core_db/nlp/
tokenize.rs1use super::span::DocSpan;
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6pub enum TokenKind {
7 Word,
8 Number,
9 Punct,
10 Other,
11}
12
13#[derive(Debug, Clone, PartialEq, Eq)]
14pub struct Token<'a> {
15 pub kind: TokenKind,
16 pub text: &'a str,
17 pub span: DocSpan,
18}
19
20#[derive(Debug, Clone, PartialEq, Eq)]
21pub struct Sentence {
22 pub span: DocSpan,
23}
24
25pub fn tokenize(source: &str) -> Vec<Token<'_>> {
27 let mut out = Vec::new();
28 let bytes = source.as_bytes();
29 let mut i = 0;
30 while i < bytes.len() {
31 let ch = source[i..].chars().next().unwrap();
32 let len = ch.len_utf8();
33 if ch.is_whitespace() {
34 i += len;
35 continue;
36 }
37 if ch.is_ascii_digit() {
38 let start = i;
39 i += len;
40 while i < bytes.len() {
41 let c = source[i..].chars().next().unwrap();
42 if c.is_ascii_digit() || c == '.' || c == ',' {
43 i += c.len_utf8();
44 } else {
45 break;
46 }
47 }
48 out.push(Token {
49 kind: TokenKind::Number,
50 text: &source[start..i],
51 span: DocSpan::new(start as u32, i as u32),
52 });
53 continue;
54 }
55 if is_word_start(ch) {
56 let start = i;
57 i += len;
58 while i < bytes.len() {
59 let c = source[i..].chars().next().unwrap();
60 if is_word_cont(c) {
61 i += c.len_utf8();
62 } else {
63 break;
64 }
65 }
66 out.push(Token {
67 kind: TokenKind::Word,
68 text: &source[start..i],
69 span: DocSpan::new(start as u32, i as u32),
70 });
71 continue;
72 }
73 if is_punct_token(ch) {
74 out.push(Token {
75 kind: TokenKind::Punct,
76 text: &source[i..i + len],
77 span: DocSpan::new(i as u32, (i + len) as u32),
78 });
79 i += len;
80 continue;
81 }
82 out.push(Token {
83 kind: TokenKind::Other,
84 text: &source[i..i + len],
85 span: DocSpan::new(i as u32, (i + len) as u32),
86 });
87 i += len;
88 }
89 out
90}
91
92pub fn split_sentences(source: &str) -> Vec<Sentence> {
94 let mut out = Vec::new();
95 let mut start = 0usize;
96 let bytes = source.as_bytes();
97 let mut i = 0;
98 while i < bytes.len() {
99 let ch = source[i..].chars().next().unwrap();
100 let len = ch.len_utf8();
101 let ender = is_sentence_ender(ch) && !is_ascii_decimal_dot(bytes, i, ch);
102 i += len;
103 if ender {
104 let end = i;
105 let slice = source[start..end].trim();
106 if !slice.is_empty() {
107 let trim_start = start + source[start..end].find(slice).unwrap_or(0);
108 out.push(Sentence {
109 span: DocSpan::new(trim_start as u32, (trim_start + slice.len()) as u32),
110 });
111 }
112 start = i;
113 }
114 }
115 if start < source.len() {
116 let slice = source[start..].trim();
117 if !slice.is_empty() {
118 let trim_start = start + source[start..].find(slice).unwrap_or(0);
119 out.push(Sentence {
120 span: DocSpan::new(trim_start as u32, (trim_start + slice.len()) as u32),
121 });
122 }
123 }
124 out
125}
126
127fn is_word_start(ch: char) -> bool {
128 ch.is_alphabetic() || ch == '_' || ch == '\''
129}
130
131fn is_word_cont(ch: char) -> bool {
132 ch.is_alphanumeric() || ch == '_' || ch == '\'' || ch == '-'
133}
134
135fn is_sentence_ender(ch: char) -> bool {
137 matches!(
138 ch,
139 '.' | '!' | '?' | '\n' | '\u{3002}' | '\u{FF01}' | '\u{FF1F}'
140 )
141}
142
143fn is_ascii_decimal_dot(bytes: &[u8], i: usize, ch: char) -> bool {
145 if ch != '.' {
146 return false;
147 }
148 let prev_digit = i > 0 && bytes[i - 1].is_ascii_digit();
149 let next = i + 1;
150 let next_digit = next < bytes.len() && bytes[next].is_ascii_digit();
151 prev_digit && next_digit
152}
153
154fn is_punct_token(ch: char) -> bool {
155 ch.is_ascii_punctuation() || matches!(ch, '\u{3002}' | '\u{FF01}' | '\u{FF1F}')
156}
157
158#[cfg(test)]
159mod tests {
160 use super::*;
161
162 #[test]
163 fn tokenizes_words_and_numbers() {
164 let src = "North Spring recorded 12.5 mm.";
165 let toks = tokenize(src);
166 assert!(toks.iter().any(|t| t.text == "North"));
167 assert!(toks.iter().any(|t| t.text == "Spring"));
168 assert!(toks.iter().any(|t| t.text == "12.5"));
169 let north = toks.iter().find(|t| t.text == "North").unwrap();
170 assert_eq!(&src[north.span.as_range()], "North");
171 }
172
173 #[test]
174 fn splits_two_sentences() {
175 let src = "One. Two!";
176 let sents = split_sentences(src);
177 assert_eq!(sents.len(), 2);
178 assert_eq!(&src[sents[0].span.as_range()], "One.");
179 assert_eq!(&src[sents[1].span.as_range()], "Two!");
180 }
181
182 #[test]
183 fn splits_decimal_not_sentence() {
184 let src = "Recorded 12.5 mm. Next.";
185 let sents = split_sentences(src);
186 assert_eq!(sents.len(), 2, "decimal point must not split a sentence");
187 assert_eq!(&src[sents[0].span.as_range()], "Recorded 12.5 mm.");
188 assert_eq!(&src[sents[1].span.as_range()], "Next.");
189 }
190
191 #[test]
192 fn splits_unicode_terminators() {
193 let src = "你好。世界!";
194 let sents = split_sentences(src);
195 assert_eq!(sents.len(), 2);
196 assert_eq!(&src[sents[0].span.as_range()], "你好。");
197 assert_eq!(&src[sents[1].span.as_range()], "世界!");
198 }
199
200 #[test]
201 fn unicode_terminators_are_punct() {
202 let src = "你好。世界!吗?";
203 let toks = tokenize(src);
204 let punct: Vec<&str> = toks
205 .iter()
206 .filter(|t| t.kind == TokenKind::Punct)
207 .map(|t| t.text)
208 .collect();
209 assert_eq!(punct, vec!["。", "!", "?"]);
210 for t in toks.iter().filter(|t| t.kind == TokenKind::Punct) {
211 assert_eq!(&src[t.span.as_range()], t.text);
212 }
213 }
214}