Skip to main content

qualia_core_db/nlp/
substrate.rs

1//! Full symbolic pipeline — orchestrates the NLP modules end-to-end.
2//!
3//! Pipeline: tokenize → gazetteer → normalize → relation extract →
4//! frame extract → coref. The `Substrate` owns the borrowed tokens (which
5//! borrow from the caller's `text`), plus owned hits/norms/relations/frames/
6//! coref chains.
7//!
8//! Mention generation is unchanged for NLP-001 (every word becomes a mention).
9//! Coreference failures are returned; they are never reported as a successful
10//! substrate. Detector quality belongs to NLP-500.
11
12use super::coref::{
13    resolve_coreferences_counted, CancellationToken, CorefChain, CorefError, CorefLimits,
14    CorefMention, MentionKind,
15};
16use super::frame::FrameInstance;
17use super::gazetteer::{Gazetteer, Hit};
18use super::normalize::Normalized;
19use super::relation::ExtractedRelation;
20use super::tokenize::{tokenize, Token, TokenKind};
21
22/// The full symbolic substrate extracted from a document.
23#[derive(Debug, Clone)]
24pub struct Substrate<'a> {
25    pub tokens: Vec<Token<'a>>,
26    pub hits: Vec<Hit>,
27    pub norms: Vec<Normalized>,
28    pub relations: Vec<ExtractedRelation>,
29    pub frames: Vec<FrameInstance>,
30    pub coref_chains: Vec<CorefChain>,
31}
32
33/// Extract the full substrate from `text`.
34pub fn extract_substrate(text: &str) -> Result<Substrate<'_>, CorefError> {
35    extract_substrate_with_limits(text, &CorefLimits::DEFAULT)
36}
37
38/// Extract the substrate using explicit coreference limits.
39pub fn extract_substrate_with_limits<'a>(
40    text: &'a str,
41    limits: &CorefLimits,
42) -> Result<Substrate<'a>, CorefError> {
43    if text.len() > limits.max_source_bytes {
44        return Err(limits.source_too_large(text.len()));
45    }
46    let tokens = tokenize(text);
47    let hits = Gazetteer::default().find(text);
48    let norms = super::normalize::normalize_dates_and_numbers(text);
49    let relations = super::relation::extract_relations(text);
50    let frames = super::frame::extract_frames(text);
51
52    let mentions = build_mentions(text, &tokens, limits)?;
53    let (coref_chains, _) =
54        resolve_coreferences_counted(text, &mentions, limits, &CancellationToken::new())?;
55
56    Ok(Substrate {
57        tokens,
58        hits,
59        norms,
60        relations,
61        frames,
62        coref_chains,
63    })
64}
65
66/// Derive coref mentions from tokens. A word is `Proper` when it is
67/// capitalised and not sentence-initial; pronouns are matched against a small
68/// closed set; every other word is `Common`. Over-broad by design until NLP-500.
69fn build_mentions(
70    _text: &str,
71    tokens: &[Token<'_>],
72    limits: &CorefLimits,
73) -> Result<Vec<CorefMention>, CorefError> {
74    let mut out = Vec::new();
75    let mut at_sentence_start = true;
76    for tok in tokens {
77        if tok.kind == TokenKind::Punct && matches!(tok.text, "." | "!" | "?") {
78            at_sentence_start = true;
79            continue;
80        }
81        if tok.kind != TokenKind::Word {
82            continue;
83        }
84        if out.len() >= limits.max_mentions {
85            return Err(CorefError::TooManyMentions {
86                count: out.len().saturating_add(1),
87                max: limits.max_mentions,
88            });
89        }
90        let lower = tok.text.to_ascii_lowercase();
91        let kind = if is_pronoun(&lower) {
92            MentionKind::Pronoun
93        } else {
94            let is_capitalised = tok
95                .text
96                .chars()
97                .next()
98                .map(|c| c.is_uppercase())
99                .unwrap_or(false);
100            if is_capitalised && !at_sentence_start {
101                MentionKind::Proper
102            } else {
103                MentionKind::Common
104            }
105        };
106        out.push(CorefMention {
107            span: tok.span,
108            text: tok.text.to_string(),
109            kind,
110        });
111        at_sentence_start = false;
112    }
113    Ok(out)
114}
115
116fn is_pronoun(s: &str) -> bool {
117    matches!(
118        s,
119        "he" | "him"
120            | "his"
121            | "himself"
122            | "she"
123            | "her"
124            | "hers"
125            | "herself"
126            | "it"
127            | "its"
128            | "itself"
129            | "they"
130            | "them"
131            | "their"
132            | "theirs"
133            | "themselves"
134    )
135}
136
137#[cfg(test)]
138mod tests {
139    use super::*;
140
141    #[test]
142    fn end_to_end_paragraph() {
143        let src = "John bought a book from Mary. She gave it to John.";
144        let sub = extract_substrate(src).expect("substrate");
145        assert!(!sub.tokens.is_empty());
146        assert_eq!(sub.frames.len(), 2);
147        assert!(sub.frames.iter().any(|f| f.frame_type == "BUY"));
148        assert!(sub.frames.iter().any(|f| f.frame_type == "TRANSFER"));
149        assert!(!sub.coref_chains.is_empty());
150        let mary_she_chain = sub.coref_chains.iter().any(|c| {
151            c.mentions.iter().any(|m| m.text == "Mary")
152                && c.mentions.iter().any(|m| m.text == "She")
153        });
154        assert!(mary_she_chain, "Mary and She should corefer");
155    }
156
157    #[test]
158    fn tokens_borrow_source() {
159        let src = "Hello world";
160        let sub = extract_substrate(src).expect("substrate");
161        assert!(sub.tokens.iter().any(|t| t.text == "Hello"));
162        assert!(sub.tokens.iter().any(|t| t.text == "world"));
163    }
164
165    #[test]
166    fn relations_extracted() {
167        let src = "Socrates is a philosopher";
168        let sub = extract_substrate(src).expect("substrate");
169        assert_eq!(sub.relations.len(), 1);
170        assert_eq!(sub.relations[0].predicate, "rdf:type");
171    }
172
173    #[test]
174    fn empty_input() {
175        let sub = extract_substrate("").expect("empty substrate");
176        assert!(sub.tokens.is_empty());
177        assert!(sub.frames.is_empty());
178        assert!(sub.relations.is_empty());
179        assert!(sub.coref_chains.is_empty());
180    }
181
182    #[test]
183    fn coref_failure_is_not_a_successful_substrate() {
184        let mut limits = CorefLimits::DEFAULT;
185        limits.max_mentions = 2;
186        let src = "alpha beta gamma";
187        assert!(matches!(
188            extract_substrate_with_limits(src, &limits),
189            Err(CorefError::TooManyMentions { .. })
190        ));
191    }
192}