1use super::error::{AnalysisState, BufferChannel, NlpContractError};
4use super::views::{
5 slice_source, DocumentSummary, DocumentViewBuffers, HitView, NormView, PlanView, SentenceView,
6 TokenView, NORM_INDEX_NONE, SENTENCE_ID_NONE,
7};
8use super::ANNOTATION_CONTRACT_VERSION;
9use crate::nlp::budget::{reject_source, NlpBudgetError};
10use crate::nlp::emit::AnnotationPlan;
11use crate::nlp::gazetteer::{Gazetteer, Hit};
12use crate::nlp::hash::hash60;
13use crate::nlp::link;
14use crate::nlp::normalize::{normalize_dates_and_numbers, Normalized};
15use crate::nlp::span::DocSpan;
16use crate::nlp::tokenize::{split_sentences, tokenize, Sentence, Token};
17use crate::nlp::DocumentAnalysis;
18
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub struct DocumentCapacities {
28 pub tokens: usize,
29 pub sentences: usize,
30 pub hits: usize,
31 pub norms: usize,
32 pub plans: usize,
33}
34
35pub fn analyze_document_into(
41 source: &str,
42 buffers: &mut DocumentViewBuffers<'_>,
43) -> Result<DocumentSummary, NlpContractError> {
44 reject_source(source.len()).map_err(source_error)?;
45 write_views(source, &year_one_layers(source), buffers)
46}
47
48pub fn required_capacities(source: &str) -> Result<DocumentCapacities, NlpContractError> {
50 reject_source(source.len()).map_err(source_error)?;
51 Ok(capacities_from_layers(&year_one_layers(source)))
52}
53
54pub fn analyze_document(source: &str) -> DocumentAnalysis {
62 let layers = year_one_layers(source);
63 let caps = capacities_from_layers(&layers);
64 let mut tokens = vec![TokenView::EMPTY; caps.tokens];
65 let mut sentences = vec![SentenceView::EMPTY; caps.sentences];
66 let mut hits = vec![HitView::EMPTY; caps.hits];
67 let mut norms = vec![NormView::EMPTY; caps.norms];
68 let mut plans = vec![PlanView::EMPTY; caps.plans];
69 let mut buffers = DocumentViewBuffers {
70 tokens: &mut tokens,
71 sentences: &mut sentences,
72 hits: &mut hits,
73 norms: &mut norms,
74 plans: &mut plans,
75 };
76 match write_views(source, &layers, &mut buffers) {
77 Ok(summary) => document_analysis_from_views(source, &hits, &norms, &plans, summary),
78 Err(NlpContractError::SpanInvalid { .. } | NlpContractError::OutputBufferFull { .. }) => {
81 document_analysis_from_layers(source, &layers)
82 }
83 Err(NlpContractError::SourceTooLarge { .. }) => {
84 document_analysis_from_layers(source, &layers)
85 }
86 }
87}
88
89struct YearOneLayers<'a> {
90 tokens: Vec<Token<'a>>,
91 sentences: Vec<Sentence>,
92 hits: Vec<Hit>,
93 norms: Vec<Normalized>,
94}
95
96fn year_one_layers(source: &str) -> YearOneLayers<'_> {
97 YearOneLayers {
98 tokens: tokenize(source),
99 sentences: split_sentences(source),
100 hits: link::filter_known(Gazetteer::default().find(source)),
101 norms: normalize_dates_and_numbers(source),
102 }
103}
104
105fn capacities_from_layers(layers: &YearOneLayers<'_>) -> DocumentCapacities {
106 DocumentCapacities {
107 tokens: layers.tokens.len(),
108 sentences: layers.sentences.len(),
109 hits: layers.hits.len(),
110 norms: layers.norms.len(),
111 plans: layers
112 .hits
113 .len()
114 .checked_add(layers.norms.len())
115 .unwrap_or(0),
116 }
117}
118
119fn source_error(err: NlpBudgetError) -> NlpContractError {
120 match err {
121 NlpBudgetError::SourceTooLarge { bytes, max } => {
122 NlpContractError::SourceTooLarge { bytes, max }
123 }
124 other => unreachable!("reject_source only yields SourceTooLarge, got {other:?}"),
125 }
126}
127
128fn require_cap(
129 needed: usize,
130 capacity: usize,
131 channel: BufferChannel,
132) -> Result<(), NlpContractError> {
133 if needed > capacity {
134 Err(NlpContractError::OutputBufferFull {
135 needed,
136 capacity,
137 channel,
138 })
139 } else {
140 Ok(())
141 }
142}
143
144fn write_views(
145 source: &str,
146 layers: &YearOneLayers<'_>,
147 buffers: &mut DocumentViewBuffers<'_>,
148) -> Result<DocumentSummary, NlpContractError> {
149 let plan_needed = layers.hits.len().checked_add(layers.norms.len()).ok_or(
150 NlpContractError::OutputBufferFull {
151 needed: buffers.plans.len().saturating_add(1),
152 capacity: buffers.plans.len(),
153 channel: BufferChannel::Plans,
154 },
155 )?;
156 require_cap(
157 layers.tokens.len(),
158 buffers.tokens.len(),
159 BufferChannel::Tokens,
160 )?;
161 require_cap(
162 layers.sentences.len(),
163 buffers.sentences.len(),
164 BufferChannel::Sentences,
165 )?;
166 require_cap(layers.hits.len(), buffers.hits.len(), BufferChannel::Hits)?;
167 require_cap(
168 layers.norms.len(),
169 buffers.norms.len(),
170 BufferChannel::Norms,
171 )?;
172 require_cap(plan_needed, buffers.plans.len(), BufferChannel::Plans)?;
173
174 for token in &layers.tokens {
175 let slice = slice_source(token.span, source)?;
176 if slice != token.text {
177 return Err(NlpContractError::SpanInvalid {
178 start: token.span.start_utf8,
179 end: token.span.end_utf8,
180 });
181 }
182 }
183 for sentence in &layers.sentences {
184 slice_source(sentence.span, source)?;
185 }
186 for hit in &layers.hits {
187 slice_source(hit.span, source)?;
188 }
189 for norm in &layers.norms {
190 slice_source(norm_span(norm), source)?;
191 }
192
193 let source_hash = hash60(source.as_bytes());
194 for (i, token) in layers.tokens.iter().enumerate() {
195 buffers.tokens[i] = TokenView {
196 span: token.span,
197 kind: token.kind,
198 sentence_id: sentence_id_for(token.span, &layers.sentences),
199 norm_index: norm_index_for(token.span, &layers.norms),
200 };
201 }
202 for (i, sentence) in layers.sentences.iter().enumerate() {
203 buffers.sentences[i] = SentenceView {
204 span: sentence.span,
205 };
206 }
207 for (i, hit) in layers.hits.iter().enumerate() {
208 buffers.hits[i] = HitView {
209 span: hit.span,
210 iri: hit.iri,
211 surface: hit.surface,
212 };
213 }
214 for (i, norm) in layers.norms.iter().enumerate() {
215 buffers.norms[i] = norm_view(norm);
216 }
217 let mut plan_i = 0usize;
218 for hit in &layers.hits {
219 buffers.plans[plan_i] = plan_from_hit(source, hit, source_hash)?;
220 plan_i += 1;
221 }
222 for norm in &layers.norms {
223 buffers.plans[plan_i] = plan_from_norm(source, norm, source_hash)?;
224 plan_i += 1;
225 }
226
227 Ok(DocumentSummary {
228 contract_version: ANNOTATION_CONTRACT_VERSION,
229 source_hash,
230 token_count: layers.tokens.len(),
231 sentence_count: layers.sentences.len(),
232 hit_count: layers.hits.len(),
233 norm_count: layers.norms.len(),
234 plan_count: plan_i,
235 state: AnalysisState::Complete,
236 })
237}
238
239fn sentence_id_for(span: DocSpan, sentences: &[Sentence]) -> u32 {
240 for (i, sentence) in sentences.iter().enumerate() {
241 if span.start_utf8 >= sentence.span.start_utf8 && span.end_utf8 <= sentence.span.end_utf8 {
242 return i as u32;
243 }
244 }
245 SENTENCE_ID_NONE
246}
247
248fn norm_index_for(span: DocSpan, norms: &[Normalized]) -> u32 {
249 for (i, norm) in norms.iter().enumerate() {
250 let nspan = norm_span(norm);
251 if span.start_utf8 < nspan.end_utf8 && nspan.start_utf8 < span.end_utf8 {
252 return i as u32;
253 }
254 }
255 NORM_INDEX_NONE
256}
257
258fn norm_span(norm: &Normalized) -> DocSpan {
259 match *norm {
260 Normalized::DateIso { span, .. } | Normalized::Number { span, .. } => span,
261 }
262}
263
264fn norm_view(norm: &Normalized) -> NormView {
265 match *norm {
266 Normalized::DateIso { span, yyyy_mm_dd } => NormView::DateIso { span, yyyy_mm_dd },
267 Normalized::Number { span, value, unit } => NormView::Number { span, value, unit },
268 }
269}
270
271fn plan_from_hit(source: &str, hit: &Hit, source_hash: u64) -> Result<PlanView, NlpContractError> {
272 let slice = slice_source(hit.span, source)?;
273 Ok(PlanView {
274 span: hit.span,
275 content_hash: hash60(slice.as_bytes()),
276 source_hash,
277 term_iri: hit.iri,
278 kind: "gazetteer",
279 })
280}
281
282fn plan_from_norm(
283 source: &str,
284 norm: &Normalized,
285 source_hash: u64,
286) -> Result<PlanView, NlpContractError> {
287 let (span, term_iri, kind) = match *norm {
288 Normalized::DateIso { span, .. } => (span, "https://qualiadb.org/datatype/isoDate", "date"),
289 Normalized::Number { span, unit, .. } => (
290 span,
291 if unit.is_some() {
292 "https://qualiadb.org/datatype/quantity"
293 } else {
294 "https://qualiadb.org/datatype/number"
295 },
296 "number",
297 ),
298 };
299 let slice = slice_source(span, source)?;
300 Ok(PlanView {
301 span,
302 content_hash: hash60(slice.as_bytes()),
303 source_hash,
304 term_iri,
305 kind,
306 })
307}
308
309fn document_analysis_from_views(
310 source: &str,
311 hits: &[HitView],
312 norms: &[NormView],
313 plans: &[PlanView],
314 summary: DocumentSummary,
315) -> DocumentAnalysis {
316 DocumentAnalysis {
317 source_hash: summary.source_hash,
318 token_count: summary.token_count,
319 sentence_count: summary.sentence_count,
320 hits: hits[..summary.hit_count]
321 .iter()
322 .map(|hit| Hit {
323 span: hit.span,
324 iri: hit.iri,
325 surface: hit.surface,
326 })
327 .collect(),
328 norms: norms[..summary.norm_count]
329 .iter()
330 .map(normalized_from_view)
331 .collect(),
332 plans: plans[..summary.plan_count]
333 .iter()
334 .map(|plan| plan_to_owned(source, *plan))
335 .collect(),
336 }
337}
338
339fn normalized_from_view(view: &NormView) -> Normalized {
340 match *view {
341 NormView::DateIso { span, yyyy_mm_dd } => Normalized::DateIso { span, yyyy_mm_dd },
342 NormView::Number { span, value, unit } => Normalized::Number { span, value, unit },
343 }
344}
345
346fn plan_to_owned(source: &str, plan: PlanView) -> AnnotationPlan {
347 let surface = plan.span.slice(source).unwrap_or("").to_string();
348 AnnotationPlan {
349 term_iri: plan.term_iri.to_string(),
350 start_utf8: plan.span.start_utf8,
351 end_utf8: plan.span.end_utf8,
352 content_hash: plan.content_hash,
353 source_hash: plan.source_hash,
354 surface,
355 kind: plan.kind,
356 }
357}
358
359fn document_analysis_from_layers(source: &str, layers: &YearOneLayers<'_>) -> DocumentAnalysis {
360 let mut plans = crate::nlp::emit::emit_from_hits(source, &layers.hits);
361 plans.extend(crate::nlp::emit::emit_from_normalized(
362 source,
363 &layers.norms,
364 ));
365 DocumentAnalysis {
366 source_hash: hash60(source.as_bytes()),
367 token_count: layers.tokens.len(),
368 sentence_count: layers.sentences.len(),
369 hits: layers.hits.clone(),
370 norms: layers.norms.clone(),
371 plans,
372 }
373}