1use super::span::DocSpan;
4
5#[derive(Debug, Clone, PartialEq)]
6pub enum Normalized {
7 DateIso {
8 span: DocSpan,
9 yyyy_mm_dd: [u8; 10],
10 },
11 Number {
12 span: DocSpan,
13 value: f64,
14 unit: Option<&'static str>,
15 },
16}
17
18const KNOWN_UNITS: &[&str] = &[
20 "mm", "cm", "m", "km", "in", "ft", "yd", "mi", "mg", "g", "kg", "t", "lb", "oz", "ms", "s",
21 "min", "h", "d", "ml", "l", "gal", "Pa", "kPa", "MPa", "bar", "psi", "K", "°C", "°F", "Hz",
22 "kHz", "MHz", "GHz", "V", "A", "W", "kW", "MW", "J", "kJ", "cal", "kcal", "mol", "ppm", "ppb",
23 "rad", "deg", "m/s", "km/h", "mph",
24];
25
26pub fn normalize_dates_and_numbers(source: &str) -> Vec<Normalized> {
28 let mut out = Vec::new();
29 let bytes = source.as_bytes();
30 let mut i = 0;
31 while i < bytes.len() {
32 if let Some(norm) = try_iso_date(source, i) {
33 i = norm_end(&norm) as usize;
34 out.push(norm);
35 continue;
36 }
37 if let Some(norm) = try_number(source, i) {
38 i = norm_end(&norm) as usize;
39 out.push(norm);
40 continue;
41 }
42 i += source[i..]
43 .chars()
44 .next()
45 .map(|c| c.len_utf8())
46 .unwrap_or(1);
47 }
48 out
49}
50
51fn try_iso_date(source: &str, i: usize) -> Option<Normalized> {
52 let rest = &source[i..];
53 if rest.len() < 10 {
54 return None;
55 }
56 let cand = &rest.as_bytes()[..10];
57 if cand[4] != b'-' || cand[7] != b'-' {
58 return None;
59 }
60 if !cand[0..4].iter().all(|b| b.is_ascii_digit()) {
61 return None;
62 }
63 if !cand[5..7].iter().all(|b| b.is_ascii_digit()) {
64 return None;
65 }
66 if !cand[8..10].iter().all(|b| b.is_ascii_digit()) {
67 return None;
68 }
69 let y: u32 = std::str::from_utf8(&cand[0..4]).ok()?.parse().ok()?;
70 let m: u32 = std::str::from_utf8(&cand[5..7]).ok()?.parse().ok()?;
71 let d: u32 = std::str::from_utf8(&cand[8..10]).ok()?.parse().ok()?;
72 if !is_valid_gregorian_date(y, m, d) {
73 return None;
74 }
75 if i > 0 && source.as_bytes()[i - 1].is_ascii_digit() {
76 return None;
77 }
78 if i + 10 < source.len() && source.as_bytes()[i + 10].is_ascii_digit() {
79 return None;
80 }
81 let mut yyyy_mm_dd = [0u8; 10];
82 yyyy_mm_dd.copy_from_slice(cand);
83 Some(Normalized::DateIso {
84 span: DocSpan::new(i as u32, (i + 10) as u32),
85 yyyy_mm_dd,
86 })
87}
88
89fn try_number(source: &str, i: usize) -> Option<Normalized> {
90 let bytes = source.as_bytes();
91 if !bytes[i].is_ascii_digit() {
92 return None;
93 }
94 if i > 0 && bytes[i - 1].is_ascii_alphanumeric() {
95 return None;
96 }
97 let mut j = i;
98 while j < bytes.len() && bytes[j].is_ascii_digit() {
99 j += 1;
100 }
101 if j < bytes.len() && bytes[j] == b'.' {
102 let k = j + 1;
103 if k < bytes.len() && bytes[k].is_ascii_digit() {
104 j = k;
105 while j < bytes.len() && bytes[j].is_ascii_digit() {
106 j += 1;
107 }
108 }
109 }
110 let value: f64 = source[i..j].parse().ok()?;
111 let mut unit = None;
112 let mut end = j;
113 if j < source.len() {
115 let rest = &source[j..];
116 if rest.starts_with(' ') {
118 if let Some(u) = longest_known_unit(&rest[1..]) {
119 unit = Some(u);
120 end = j + 1 + u.len();
121 }
122 }
123 if unit.is_none() {
125 for &u in &["°C", "°F"] {
126 if rest.starts_with(u) {
127 unit = Some(u);
128 end = j + u.len();
129 break;
130 }
131 }
132 }
133 }
134 Some(Normalized::Number {
135 span: DocSpan::new(i as u32, end as u32),
136 value,
137 unit,
138 })
139}
140
141fn longest_known_unit(after: &str) -> Option<&'static str> {
143 let mut best: Option<&'static str> = None;
144 for &u in KNOWN_UNITS {
145 if !after.starts_with(u) {
146 continue;
147 }
148 let rest = &after[u.len()..];
149 if !rest.is_empty() && rest.as_bytes()[0].is_ascii_alphanumeric() {
150 continue;
151 }
152 if best.map_or(true, |b| u.len() > b.len()) {
153 best = Some(u);
154 }
155 }
156 best
157}
158
159fn is_leap_year(y: u32) -> bool {
160 y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)
161}
162
163fn days_in_month(y: u32, m: u32) -> Option<u32> {
164 Some(match m {
165 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
166 4 | 6 | 9 | 11 => 30,
167 2 => {
168 if is_leap_year(y) {
169 29
170 } else {
171 28
172 }
173 }
174 _ => return None,
175 })
176}
177
178fn is_valid_gregorian_date(y: u32, m: u32, d: u32) -> bool {
179 if y < 1000 {
180 return false;
181 }
182 days_in_month(y, m).is_some_and(|max| (1..=max).contains(&d))
183}
184
185fn norm_end(n: &Normalized) -> u32 {
186 match n {
187 Normalized::DateIso { span, .. } | Normalized::Number { span, .. } => span.end_utf8,
188 }
189}
190
191#[cfg(test)]
192mod tests {
193 use super::*;
194
195 #[test]
196 fn iso_date_and_mm() {
197 let src = "12.5 mm on 2026-08-15";
198 let n = normalize_dates_and_numbers(src);
199 assert!(n.iter().any(|x| matches!(x, Normalized::Number { value, unit: Some("mm"), .. } if (*value - 12.5).abs() < 1e-9)));
200 assert!(n.iter().any(|x| matches!(x, Normalized::DateIso { .. })));
201 }
202
203 #[test]
204 fn multi_unit_normalization() {
205 let src = "100 kg and 25 °C and 50 Hz";
206 let n = normalize_dates_and_numbers(src);
207 assert!(n.iter().any(|x| matches!(x, Normalized::Number { value, unit: Some("kg"), .. } if (*value - 100.0).abs() < 1e-9)));
208 assert!(n.iter().any(|x| matches!(x, Normalized::Number { value, unit: Some("°C"), .. } if (*value - 25.0).abs() < 1e-9)));
209 assert!(n.iter().any(|x| matches!(x, Normalized::Number { value, unit: Some("Hz"), .. } if (*value - 50.0).abs() < 1e-9)));
210 }
211
212 #[test]
213 fn no_unit_for_bare_numbers() {
214 let src = "There are 42 items.";
215 let n = normalize_dates_and_numbers(src);
216 assert!(n.iter().any(|x| matches!(x, Normalized::Number { unit: None, value, .. } if (*value - 42.0).abs() < 1e-9)));
217 }
218
219 fn unit_of(src: &str) -> Option<&'static str> {
220 normalize_dates_and_numbers(src)
221 .into_iter()
222 .find_map(|x| match x {
223 Normalized::Number { unit, .. } => unit,
224 _ => None,
225 })
226 }
227
228 #[test]
229 fn attached_degree_celsius_span() {
230 let src = "Recorded 25°C today";
231 let n = normalize_dates_and_numbers(src);
232 let found = n
233 .iter()
234 .find(|x| {
235 matches!(
236 x,
237 Normalized::Number {
238 unit: Some("°C"),
239 ..
240 }
241 )
242 })
243 .expect("attached °C");
244 match found {
245 Normalized::Number { span, value, unit } => {
246 assert!((value - 25.0).abs() < 1e-9);
247 assert_eq!(*unit, Some("°C"));
248 assert_eq!(&src[span.as_range()], "25°C");
249 }
250 _ => panic!("expected number"),
251 }
252 assert_eq!(unit_of("25 °C"), Some("°C"));
254 assert_eq!(unit_of("25 °F"), Some("°F"));
255 let f_src = "boils at 212°F";
256 let f = normalize_dates_and_numbers(f_src);
257 let f_found = f
258 .iter()
259 .find(|x| {
260 matches!(
261 x,
262 Normalized::Number {
263 unit: Some("°F"),
264 ..
265 }
266 )
267 })
268 .expect("attached °F");
269 if let Normalized::Number { span, .. } = f_found {
270 assert_eq!(&f_src[span.as_range()], "212°F");
271 }
272 }
273
274 #[test]
275 fn compound_unit_longest_match() {
276 assert_eq!(unit_of("10 m/s"), Some("m/s"));
277 assert_eq!(unit_of("10 km/h"), Some("km/h"));
278 assert_eq!(unit_of("10 ms"), Some("ms"));
279 assert_eq!(unit_of("10 m"), Some("m"));
280 }
281
282 fn date_labels(src: &str) -> Vec<String> {
283 normalize_dates_and_numbers(src)
284 .into_iter()
285 .filter_map(|x| match x {
286 Normalized::DateIso { yyyy_mm_dd, .. } => {
287 Some(String::from_utf8_lossy(&yyyy_mm_dd).into_owned())
288 }
289 _ => None,
290 })
291 .collect()
292 }
293
294 #[test]
295 fn rejects_invalid_calendar_dates() {
296 let src = "bad 2026-02-31 and 2023-02-29 and 2026-04-31 ok 2024-02-29 and 2026-08-15";
297 let dates = date_labels(src);
298 assert!(
299 !dates.iter().any(|d| d == "2026-02-31"),
300 "Feb 31 is not a calendar date: {dates:?}"
301 );
302 assert!(
303 !dates.iter().any(|d| d == "2023-02-29"),
304 "2023 is not a leap year: {dates:?}"
305 );
306 assert!(
307 !dates.iter().any(|d| d == "2026-04-31"),
308 "April has 30 days: {dates:?}"
309 );
310 assert!(
311 dates.iter().any(|d| d == "2024-02-29"),
312 "2024-02-29 is a leap day: {dates:?}"
313 );
314 assert!(
315 dates.iter().any(|d| d == "2026-08-15"),
316 "existing valid date must still parse: {dates:?}"
317 );
318 }
319}