qualia_core_db/nlp/
substrate.rs1use super::coref::{
13 resolve_coreferences_counted, CancellationToken, CorefChain, CorefError, CorefLimits,
14 CorefMention, MentionKind,
15};
16use super::frame::FrameInstance;
17use super::gazetteer::{Gazetteer, Hit};
18use super::normalize::Normalized;
19use super::relation::ExtractedRelation;
20use super::tokenize::{tokenize, Token, TokenKind};
21
22#[derive(Debug, Clone)]
24pub struct Substrate<'a> {
25 pub tokens: Vec<Token<'a>>,
26 pub hits: Vec<Hit>,
27 pub norms: Vec<Normalized>,
28 pub relations: Vec<ExtractedRelation>,
29 pub frames: Vec<FrameInstance>,
30 pub coref_chains: Vec<CorefChain>,
31}
32
33pub fn extract_substrate(text: &str) -> Result<Substrate<'_>, CorefError> {
35 extract_substrate_with_limits(text, &CorefLimits::DEFAULT)
36}
37
38pub fn extract_substrate_with_limits<'a>(
40 text: &'a str,
41 limits: &CorefLimits,
42) -> Result<Substrate<'a>, CorefError> {
43 if text.len() > limits.max_source_bytes {
44 return Err(limits.source_too_large(text.len()));
45 }
46 let tokens = tokenize(text);
47 let hits = Gazetteer::default().find(text);
48 let norms = super::normalize::normalize_dates_and_numbers(text);
49 let relations = super::relation::extract_relations(text);
50 let frames = super::frame::extract_frames(text);
51
52 let mentions = build_mentions(text, &tokens, limits)?;
53 let (coref_chains, _) =
54 resolve_coreferences_counted(text, &mentions, limits, &CancellationToken::new())?;
55
56 Ok(Substrate {
57 tokens,
58 hits,
59 norms,
60 relations,
61 frames,
62 coref_chains,
63 })
64}
65
66fn build_mentions(
70 _text: &str,
71 tokens: &[Token<'_>],
72 limits: &CorefLimits,
73) -> Result<Vec<CorefMention>, CorefError> {
74 let mut out = Vec::new();
75 let mut at_sentence_start = true;
76 for tok in tokens {
77 if tok.kind == TokenKind::Punct && matches!(tok.text, "." | "!" | "?") {
78 at_sentence_start = true;
79 continue;
80 }
81 if tok.kind != TokenKind::Word {
82 continue;
83 }
84 if out.len() >= limits.max_mentions {
85 return Err(CorefError::TooManyMentions {
86 count: out.len().saturating_add(1),
87 max: limits.max_mentions,
88 });
89 }
90 let lower = tok.text.to_ascii_lowercase();
91 let kind = if is_pronoun(&lower) {
92 MentionKind::Pronoun
93 } else {
94 let is_capitalised = tok
95 .text
96 .chars()
97 .next()
98 .map(|c| c.is_uppercase())
99 .unwrap_or(false);
100 if is_capitalised && !at_sentence_start {
101 MentionKind::Proper
102 } else {
103 MentionKind::Common
104 }
105 };
106 out.push(CorefMention {
107 span: tok.span,
108 text: tok.text.to_string(),
109 kind,
110 });
111 at_sentence_start = false;
112 }
113 Ok(out)
114}
115
116fn is_pronoun(s: &str) -> bool {
117 matches!(
118 s,
119 "he" | "him"
120 | "his"
121 | "himself"
122 | "she"
123 | "her"
124 | "hers"
125 | "herself"
126 | "it"
127 | "its"
128 | "itself"
129 | "they"
130 | "them"
131 | "their"
132 | "theirs"
133 | "themselves"
134 )
135}
136
137#[cfg(test)]
138mod tests {
139 use super::*;
140
141 #[test]
142 fn end_to_end_paragraph() {
143 let src = "John bought a book from Mary. She gave it to John.";
144 let sub = extract_substrate(src).expect("substrate");
145 assert!(!sub.tokens.is_empty());
146 assert_eq!(sub.frames.len(), 2);
147 assert!(sub.frames.iter().any(|f| f.frame_type == "BUY"));
148 assert!(sub.frames.iter().any(|f| f.frame_type == "TRANSFER"));
149 assert!(!sub.coref_chains.is_empty());
150 let mary_she_chain = sub.coref_chains.iter().any(|c| {
151 c.mentions.iter().any(|m| m.text == "Mary")
152 && c.mentions.iter().any(|m| m.text == "She")
153 });
154 assert!(mary_she_chain, "Mary and She should corefer");
155 }
156
157 #[test]
158 fn tokens_borrow_source() {
159 let src = "Hello world";
160 let sub = extract_substrate(src).expect("substrate");
161 assert!(sub.tokens.iter().any(|t| t.text == "Hello"));
162 assert!(sub.tokens.iter().any(|t| t.text == "world"));
163 }
164
165 #[test]
166 fn relations_extracted() {
167 let src = "Socrates is a philosopher";
168 let sub = extract_substrate(src).expect("substrate");
169 assert_eq!(sub.relations.len(), 1);
170 assert_eq!(sub.relations[0].predicate, "rdf:type");
171 }
172
173 #[test]
174 fn empty_input() {
175 let sub = extract_substrate("").expect("empty substrate");
176 assert!(sub.tokens.is_empty());
177 assert!(sub.frames.is_empty());
178 assert!(sub.relations.is_empty());
179 assert!(sub.coref_chains.is_empty());
180 }
181
182 #[test]
183 fn coref_failure_is_not_a_successful_substrate() {
184 let mut limits = CorefLimits::DEFAULT;
185 limits.max_mentions = 2;
186 let src = "alpha beta gamma";
187 assert!(matches!(
188 extract_substrate_with_limits(src, &limits),
189 Err(CorefError::TooManyMentions { .. })
190 ));
191 }
192}