Skip to main content

qualia_core_db/nlp/
normalize.rs

1//! Narrow date / number normalizers. Extended with QUDT unit vocabulary.
2
3use super::span::DocSpan;
4
5#[derive(Debug, Clone, PartialEq)]
6pub enum Normalized {
7    DateIso {
8        span: DocSpan,
9        yyyy_mm_dd: [u8; 10],
10    },
11    Number {
12        span: DocSpan,
13        value: f64,
14        unit: Option<&'static str>,
15    },
16}
17
18/// QUDT-compatible unit symbols recognized by the normalizer.
19const KNOWN_UNITS: &[&str] = &[
20    "mm", "cm", "m", "km", "in", "ft", "yd", "mi", "mg", "g", "kg", "t", "lb", "oz", "ms", "s",
21    "min", "h", "d", "ml", "l", "gal", "Pa", "kPa", "MPa", "bar", "psi", "K", "°C", "°F", "Hz",
22    "kHz", "MHz", "GHz", "V", "A", "W", "kW", "MW", "J", "kJ", "cal", "kcal", "mol", "ppm", "ppb",
23    "rad", "deg", "m/s", "km/h", "mph",
24];
25
26/// Normalizes dates and numbers.
27pub fn normalize_dates_and_numbers(source: &str) -> Vec<Normalized> {
28    let mut out = Vec::new();
29    let bytes = source.as_bytes();
30    let mut i = 0;
31    while i < bytes.len() {
32        if let Some(norm) = try_iso_date(source, i) {
33            i = norm_end(&norm) as usize;
34            out.push(norm);
35            continue;
36        }
37        if let Some(norm) = try_number(source, i) {
38            i = norm_end(&norm) as usize;
39            out.push(norm);
40            continue;
41        }
42        i += source[i..]
43            .chars()
44            .next()
45            .map(|c| c.len_utf8())
46            .unwrap_or(1);
47    }
48    out
49}
50
51fn try_iso_date(source: &str, i: usize) -> Option<Normalized> {
52    let rest = &source[i..];
53    if rest.len() < 10 {
54        return None;
55    }
56    let cand = &rest.as_bytes()[..10];
57    if cand[4] != b'-' || cand[7] != b'-' {
58        return None;
59    }
60    if !cand[0..4].iter().all(|b| b.is_ascii_digit()) {
61        return None;
62    }
63    if !cand[5..7].iter().all(|b| b.is_ascii_digit()) {
64        return None;
65    }
66    if !cand[8..10].iter().all(|b| b.is_ascii_digit()) {
67        return None;
68    }
69    let y: u32 = std::str::from_utf8(&cand[0..4]).ok()?.parse().ok()?;
70    let m: u32 = std::str::from_utf8(&cand[5..7]).ok()?.parse().ok()?;
71    let d: u32 = std::str::from_utf8(&cand[8..10]).ok()?.parse().ok()?;
72    if !is_valid_gregorian_date(y, m, d) {
73        return None;
74    }
75    if i > 0 && source.as_bytes()[i - 1].is_ascii_digit() {
76        return None;
77    }
78    if i + 10 < source.len() && source.as_bytes()[i + 10].is_ascii_digit() {
79        return None;
80    }
81    let mut yyyy_mm_dd = [0u8; 10];
82    yyyy_mm_dd.copy_from_slice(cand);
83    Some(Normalized::DateIso {
84        span: DocSpan::new(i as u32, (i + 10) as u32),
85        yyyy_mm_dd,
86    })
87}
88
89fn try_number(source: &str, i: usize) -> Option<Normalized> {
90    let bytes = source.as_bytes();
91    if !bytes[i].is_ascii_digit() {
92        return None;
93    }
94    if i > 0 && bytes[i - 1].is_ascii_alphanumeric() {
95        return None;
96    }
97    let mut j = i;
98    while j < bytes.len() && bytes[j].is_ascii_digit() {
99        j += 1;
100    }
101    if j < bytes.len() && bytes[j] == b'.' {
102        let k = j + 1;
103        if k < bytes.len() && bytes[k].is_ascii_digit() {
104            j = k;
105            while j < bytes.len() && bytes[j].is_ascii_digit() {
106                j += 1;
107            }
108        }
109    }
110    let value: f64 = source[i..j].parse().ok()?;
111    let mut unit = None;
112    let mut end = j;
113    // Try to match a known unit after the number (preceded by space or directly)
114    if j < source.len() {
115        let rest = &source[j..];
116        // Space-separated: longest KNOWN_UNITS prefix (so "m/s" wins over "m")
117        if rest.starts_with(' ') {
118            if let Some(u) = longest_known_unit(&rest[1..]) {
119                unit = Some(u);
120                end = j + 1 + u.len();
121            }
122        }
123        // Direct attachment (no space) for degree units; `°` is 2 UTF-8 bytes.
124        if unit.is_none() {
125            for &u in &["°C", "°F"] {
126                if rest.starts_with(u) {
127                    unit = Some(u);
128                    end = j + u.len();
129                    break;
130                }
131            }
132        }
133    }
134    Some(Normalized::Number {
135        span: DocSpan::new(i as u32, end as u32),
136        value,
137        unit,
138    })
139}
140
141/// Longest `KNOWN_UNITS` prefix of `after` that ends on a non-alphanumeric boundary.
142fn longest_known_unit(after: &str) -> Option<&'static str> {
143    let mut best: Option<&'static str> = None;
144    for &u in KNOWN_UNITS {
145        if !after.starts_with(u) {
146            continue;
147        }
148        let rest = &after[u.len()..];
149        if !rest.is_empty() && rest.as_bytes()[0].is_ascii_alphanumeric() {
150            continue;
151        }
152        if best.map_or(true, |b| u.len() > b.len()) {
153            best = Some(u);
154        }
155    }
156    best
157}
158
159fn is_leap_year(y: u32) -> bool {
160    y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)
161}
162
163fn days_in_month(y: u32, m: u32) -> Option<u32> {
164    Some(match m {
165        1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
166        4 | 6 | 9 | 11 => 30,
167        2 => {
168            if is_leap_year(y) {
169                29
170            } else {
171                28
172            }
173        }
174        _ => return None,
175    })
176}
177
178fn is_valid_gregorian_date(y: u32, m: u32, d: u32) -> bool {
179    if y < 1000 {
180        return false;
181    }
182    days_in_month(y, m).is_some_and(|max| (1..=max).contains(&d))
183}
184
185fn norm_end(n: &Normalized) -> u32 {
186    match n {
187        Normalized::DateIso { span, .. } | Normalized::Number { span, .. } => span.end_utf8,
188    }
189}
190
191#[cfg(test)]
192mod tests {
193    use super::*;
194
195    #[test]
196    fn iso_date_and_mm() {
197        let src = "12.5 mm on 2026-08-15";
198        let n = normalize_dates_and_numbers(src);
199        assert!(n.iter().any(|x| matches!(x, Normalized::Number { value, unit: Some("mm"), .. } if (*value - 12.5).abs() < 1e-9)));
200        assert!(n.iter().any(|x| matches!(x, Normalized::DateIso { .. })));
201    }
202
203    #[test]
204    fn multi_unit_normalization() {
205        let src = "100 kg and 25 °C and 50 Hz";
206        let n = normalize_dates_and_numbers(src);
207        assert!(n.iter().any(|x| matches!(x, Normalized::Number { value, unit: Some("kg"), .. } if (*value - 100.0).abs() < 1e-9)));
208        assert!(n.iter().any(|x| matches!(x, Normalized::Number { value, unit: Some("°C"), .. } if (*value - 25.0).abs() < 1e-9)));
209        assert!(n.iter().any(|x| matches!(x, Normalized::Number { value, unit: Some("Hz"), .. } if (*value - 50.0).abs() < 1e-9)));
210    }
211
212    #[test]
213    fn no_unit_for_bare_numbers() {
214        let src = "There are 42 items.";
215        let n = normalize_dates_and_numbers(src);
216        assert!(n.iter().any(|x| matches!(x, Normalized::Number { unit: None, value, .. } if (*value - 42.0).abs() < 1e-9)));
217    }
218
219    fn unit_of(src: &str) -> Option<&'static str> {
220        normalize_dates_and_numbers(src)
221            .into_iter()
222            .find_map(|x| match x {
223                Normalized::Number { unit, .. } => unit,
224                _ => None,
225            })
226    }
227
228    #[test]
229    fn attached_degree_celsius_span() {
230        let src = "Recorded 25°C today";
231        let n = normalize_dates_and_numbers(src);
232        let found = n
233            .iter()
234            .find(|x| {
235                matches!(
236                    x,
237                    Normalized::Number {
238                        unit: Some("°C"),
239                        ..
240                    }
241                )
242            })
243            .expect("attached °C");
244        match found {
245            Normalized::Number { span, value, unit } => {
246                assert!((value - 25.0).abs() < 1e-9);
247                assert_eq!(*unit, Some("°C"));
248                assert_eq!(&src[span.as_range()], "25°C");
249            }
250            _ => panic!("expected number"),
251        }
252        // Space-separated degree units still match via KNOWN_UNITS.
253        assert_eq!(unit_of("25 °C"), Some("°C"));
254        assert_eq!(unit_of("25 °F"), Some("°F"));
255        let f_src = "boils at 212°F";
256        let f = normalize_dates_and_numbers(f_src);
257        let f_found = f
258            .iter()
259            .find(|x| {
260                matches!(
261                    x,
262                    Normalized::Number {
263                        unit: Some("°F"),
264                        ..
265                    }
266                )
267            })
268            .expect("attached °F");
269        if let Normalized::Number { span, .. } = f_found {
270            assert_eq!(&f_src[span.as_range()], "212°F");
271        }
272    }
273
274    #[test]
275    fn compound_unit_longest_match() {
276        assert_eq!(unit_of("10 m/s"), Some("m/s"));
277        assert_eq!(unit_of("10 km/h"), Some("km/h"));
278        assert_eq!(unit_of("10 ms"), Some("ms"));
279        assert_eq!(unit_of("10 m"), Some("m"));
280    }
281
282    fn date_labels(src: &str) -> Vec<String> {
283        normalize_dates_and_numbers(src)
284            .into_iter()
285            .filter_map(|x| match x {
286                Normalized::DateIso { yyyy_mm_dd, .. } => {
287                    Some(String::from_utf8_lossy(&yyyy_mm_dd).into_owned())
288                }
289                _ => None,
290            })
291            .collect()
292    }
293
294    #[test]
295    fn rejects_invalid_calendar_dates() {
296        let src = "bad 2026-02-31 and 2023-02-29 and 2026-04-31 ok 2024-02-29 and 2026-08-15";
297        let dates = date_labels(src);
298        assert!(
299            !dates.iter().any(|d| d == "2026-02-31"),
300            "Feb 31 is not a calendar date: {dates:?}"
301        );
302        assert!(
303            !dates.iter().any(|d| d == "2023-02-29"),
304            "2023 is not a leap year: {dates:?}"
305        );
306        assert!(
307            !dates.iter().any(|d| d == "2026-04-31"),
308            "April has 30 days: {dates:?}"
309        );
310        assert!(
311            dates.iter().any(|d| d == "2024-02-29"),
312            "2024-02-29 is a leap day: {dates:?}"
313        );
314        assert!(
315            dates.iter().any(|d| d == "2026-08-15"),
316            "existing valid date must still parse: {dates:?}"
317        );
318    }
319}