Skip to main content

qualia_core_db/nlp/contracts/
analyze.rs

1//! Year-one pipeline behind the caller-buffered contract.
2
3use super::error::{AnalysisState, BufferChannel, NlpContractError};
4use super::views::{
5    slice_source, DocumentSummary, DocumentViewBuffers, HitView, NormView, PlanView, SentenceView,
6    TokenView, NORM_INDEX_NONE, SENTENCE_ID_NONE,
7};
8use super::ANNOTATION_CONTRACT_VERSION;
9use crate::nlp::budget::{reject_source, NlpBudgetError};
10use crate::nlp::emit::AnnotationPlan;
11use crate::nlp::gazetteer::{Gazetteer, Hit};
12use crate::nlp::hash::hash60;
13use crate::nlp::link;
14use crate::nlp::normalize::{normalize_dates_and_numbers, Normalized};
15use crate::nlp::span::DocSpan;
16use crate::nlp::tokenize::{split_sentences, tokenize, Sentence, Token};
17use crate::nlp::DocumentAnalysis;
18
19/// Slot counts a caller must supply for [`analyze_document_into`] on `source`.
20///
21/// This runs the year-one pipeline once. Prefer calling it, allocating those
22/// lengths, then [`analyze_document_into`] — or use [`analyze_document`], which
23/// sizes from one pipeline run. Overflow is a typed error; buffers are not
24/// written when any channel is short. After `OutputBufferFull`, caller slices
25/// retain their prior contents.
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub struct DocumentCapacities {
28    pub tokens: usize,
29    pub sentences: usize,
30    pub hits: usize,
31    pub norms: usize,
32    pub plans: usize,
33}
34
35/// Caller-buffered year-one analysis. Source cap is [`crate::nlp::budget::MAX_SOURCE_BYTES`].
36///
37/// Required capacities: [`required_capacities`]. Retry: allocate at least those
38/// lengths and call again; this function does not resize. On error, `buffers`
39/// are left unchanged (all `require_cap` checks run before any write).
40pub fn analyze_document_into(
41    source: &str,
42    buffers: &mut DocumentViewBuffers<'_>,
43) -> Result<DocumentSummary, NlpContractError> {
44    reject_source(source.len()).map_err(source_error)?;
45    write_views(source, &year_one_layers(source), buffers)
46}
47
48/// Capacities for a successful [`analyze_document_into`] of `source`.
49pub fn required_capacities(source: &str) -> Result<DocumentCapacities, NlpContractError> {
50    reject_source(source.len()).map_err(source_error)?;
51    Ok(capacities_from_layers(&year_one_layers(source)))
52}
53
54/// Cold `Vec` adapter. Same year-one pipeline as [`analyze_document_into`].
55///
56/// Signature stays `-> DocumentAnalysis` for existing hosts. The pipeline runs
57/// **once**; buffers are sized from those layer lengths (no probe-by-error
58/// loop). Hosts and HTTP adapters must still call
59/// [`crate::nlp::budget::reject_source`] (or the 64 KiB Poet payload cap)
60/// before this entry — this adapter does not change the public error type.
61pub fn analyze_document(source: &str) -> DocumentAnalysis {
62    let layers = year_one_layers(source);
63    let caps = capacities_from_layers(&layers);
64    let mut tokens = vec![TokenView::EMPTY; caps.tokens];
65    let mut sentences = vec![SentenceView::EMPTY; caps.sentences];
66    let mut hits = vec![HitView::EMPTY; caps.hits];
67    let mut norms = vec![NormView::EMPTY; caps.norms];
68    let mut plans = vec![PlanView::EMPTY; caps.plans];
69    let mut buffers = DocumentViewBuffers {
70        tokens: &mut tokens,
71        sentences: &mut sentences,
72        hits: &mut hits,
73        norms: &mut norms,
74        plans: &mut plans,
75    };
76    match write_views(source, &layers, &mut buffers) {
77        Ok(summary) => document_analysis_from_views(source, &hits, &norms, &plans, summary),
78        // Cold adapter cannot return Result. Span/capacity failures fall back
79        // to the owned emit path rather than panic.
80        Err(NlpContractError::SpanInvalid { .. } | NlpContractError::OutputBufferFull { .. }) => {
81            document_analysis_from_layers(source, &layers)
82        }
83        Err(NlpContractError::SourceTooLarge { .. }) => {
84            document_analysis_from_layers(source, &layers)
85        }
86    }
87}
88
89struct YearOneLayers<'a> {
90    tokens: Vec<Token<'a>>,
91    sentences: Vec<Sentence>,
92    hits: Vec<Hit>,
93    norms: Vec<Normalized>,
94}
95
96fn year_one_layers(source: &str) -> YearOneLayers<'_> {
97    YearOneLayers {
98        tokens: tokenize(source),
99        sentences: split_sentences(source),
100        hits: link::filter_known(Gazetteer::default().find(source)),
101        norms: normalize_dates_and_numbers(source),
102    }
103}
104
105fn capacities_from_layers(layers: &YearOneLayers<'_>) -> DocumentCapacities {
106    DocumentCapacities {
107        tokens: layers.tokens.len(),
108        sentences: layers.sentences.len(),
109        hits: layers.hits.len(),
110        norms: layers.norms.len(),
111        plans: layers
112            .hits
113            .len()
114            .checked_add(layers.norms.len())
115            .unwrap_or(0),
116    }
117}
118
119fn source_error(err: NlpBudgetError) -> NlpContractError {
120    match err {
121        NlpBudgetError::SourceTooLarge { bytes, max } => {
122            NlpContractError::SourceTooLarge { bytes, max }
123        }
124        other => unreachable!("reject_source only yields SourceTooLarge, got {other:?}"),
125    }
126}
127
128fn require_cap(
129    needed: usize,
130    capacity: usize,
131    channel: BufferChannel,
132) -> Result<(), NlpContractError> {
133    if needed > capacity {
134        Err(NlpContractError::OutputBufferFull {
135            needed,
136            capacity,
137            channel,
138        })
139    } else {
140        Ok(())
141    }
142}
143
144fn write_views(
145    source: &str,
146    layers: &YearOneLayers<'_>,
147    buffers: &mut DocumentViewBuffers<'_>,
148) -> Result<DocumentSummary, NlpContractError> {
149    let plan_needed = layers.hits.len().checked_add(layers.norms.len()).ok_or(
150        NlpContractError::OutputBufferFull {
151            needed: buffers.plans.len().saturating_add(1),
152            capacity: buffers.plans.len(),
153            channel: BufferChannel::Plans,
154        },
155    )?;
156    require_cap(
157        layers.tokens.len(),
158        buffers.tokens.len(),
159        BufferChannel::Tokens,
160    )?;
161    require_cap(
162        layers.sentences.len(),
163        buffers.sentences.len(),
164        BufferChannel::Sentences,
165    )?;
166    require_cap(layers.hits.len(), buffers.hits.len(), BufferChannel::Hits)?;
167    require_cap(
168        layers.norms.len(),
169        buffers.norms.len(),
170        BufferChannel::Norms,
171    )?;
172    require_cap(plan_needed, buffers.plans.len(), BufferChannel::Plans)?;
173
174    for token in &layers.tokens {
175        let slice = slice_source(token.span, source)?;
176        if slice != token.text {
177            return Err(NlpContractError::SpanInvalid {
178                start: token.span.start_utf8,
179                end: token.span.end_utf8,
180            });
181        }
182    }
183    for sentence in &layers.sentences {
184        slice_source(sentence.span, source)?;
185    }
186    for hit in &layers.hits {
187        slice_source(hit.span, source)?;
188    }
189    for norm in &layers.norms {
190        slice_source(norm_span(norm), source)?;
191    }
192
193    let source_hash = hash60(source.as_bytes());
194    for (i, token) in layers.tokens.iter().enumerate() {
195        buffers.tokens[i] = TokenView {
196            span: token.span,
197            kind: token.kind,
198            sentence_id: sentence_id_for(token.span, &layers.sentences),
199            norm_index: norm_index_for(token.span, &layers.norms),
200        };
201    }
202    for (i, sentence) in layers.sentences.iter().enumerate() {
203        buffers.sentences[i] = SentenceView {
204            span: sentence.span,
205        };
206    }
207    for (i, hit) in layers.hits.iter().enumerate() {
208        buffers.hits[i] = HitView {
209            span: hit.span,
210            iri: hit.iri,
211            surface: hit.surface,
212        };
213    }
214    for (i, norm) in layers.norms.iter().enumerate() {
215        buffers.norms[i] = norm_view(norm);
216    }
217    let mut plan_i = 0usize;
218    for hit in &layers.hits {
219        buffers.plans[plan_i] = plan_from_hit(source, hit, source_hash)?;
220        plan_i += 1;
221    }
222    for norm in &layers.norms {
223        buffers.plans[plan_i] = plan_from_norm(source, norm, source_hash)?;
224        plan_i += 1;
225    }
226
227    Ok(DocumentSummary {
228        contract_version: ANNOTATION_CONTRACT_VERSION,
229        source_hash,
230        token_count: layers.tokens.len(),
231        sentence_count: layers.sentences.len(),
232        hit_count: layers.hits.len(),
233        norm_count: layers.norms.len(),
234        plan_count: plan_i,
235        state: AnalysisState::Complete,
236    })
237}
238
239fn sentence_id_for(span: DocSpan, sentences: &[Sentence]) -> u32 {
240    for (i, sentence) in sentences.iter().enumerate() {
241        if span.start_utf8 >= sentence.span.start_utf8 && span.end_utf8 <= sentence.span.end_utf8 {
242            return i as u32;
243        }
244    }
245    SENTENCE_ID_NONE
246}
247
248fn norm_index_for(span: DocSpan, norms: &[Normalized]) -> u32 {
249    for (i, norm) in norms.iter().enumerate() {
250        let nspan = norm_span(norm);
251        if span.start_utf8 < nspan.end_utf8 && nspan.start_utf8 < span.end_utf8 {
252            return i as u32;
253        }
254    }
255    NORM_INDEX_NONE
256}
257
258fn norm_span(norm: &Normalized) -> DocSpan {
259    match *norm {
260        Normalized::DateIso { span, .. } | Normalized::Number { span, .. } => span,
261    }
262}
263
264fn norm_view(norm: &Normalized) -> NormView {
265    match *norm {
266        Normalized::DateIso { span, yyyy_mm_dd } => NormView::DateIso { span, yyyy_mm_dd },
267        Normalized::Number { span, value, unit } => NormView::Number { span, value, unit },
268    }
269}
270
271fn plan_from_hit(source: &str, hit: &Hit, source_hash: u64) -> Result<PlanView, NlpContractError> {
272    let slice = slice_source(hit.span, source)?;
273    Ok(PlanView {
274        span: hit.span,
275        content_hash: hash60(slice.as_bytes()),
276        source_hash,
277        term_iri: hit.iri,
278        kind: "gazetteer",
279    })
280}
281
282fn plan_from_norm(
283    source: &str,
284    norm: &Normalized,
285    source_hash: u64,
286) -> Result<PlanView, NlpContractError> {
287    let (span, term_iri, kind) = match *norm {
288        Normalized::DateIso { span, .. } => (span, "https://qualiadb.org/datatype/isoDate", "date"),
289        Normalized::Number { span, unit, .. } => (
290            span,
291            if unit.is_some() {
292                "https://qualiadb.org/datatype/quantity"
293            } else {
294                "https://qualiadb.org/datatype/number"
295            },
296            "number",
297        ),
298    };
299    let slice = slice_source(span, source)?;
300    Ok(PlanView {
301        span,
302        content_hash: hash60(slice.as_bytes()),
303        source_hash,
304        term_iri,
305        kind,
306    })
307}
308
309fn document_analysis_from_views(
310    source: &str,
311    hits: &[HitView],
312    norms: &[NormView],
313    plans: &[PlanView],
314    summary: DocumentSummary,
315) -> DocumentAnalysis {
316    DocumentAnalysis {
317        source_hash: summary.source_hash,
318        token_count: summary.token_count,
319        sentence_count: summary.sentence_count,
320        hits: hits[..summary.hit_count]
321            .iter()
322            .map(|hit| Hit {
323                span: hit.span,
324                iri: hit.iri,
325                surface: hit.surface,
326            })
327            .collect(),
328        norms: norms[..summary.norm_count]
329            .iter()
330            .map(normalized_from_view)
331            .collect(),
332        plans: plans[..summary.plan_count]
333            .iter()
334            .map(|plan| plan_to_owned(source, *plan))
335            .collect(),
336    }
337}
338
339fn normalized_from_view(view: &NormView) -> Normalized {
340    match *view {
341        NormView::DateIso { span, yyyy_mm_dd } => Normalized::DateIso { span, yyyy_mm_dd },
342        NormView::Number { span, value, unit } => Normalized::Number { span, value, unit },
343    }
344}
345
346fn plan_to_owned(source: &str, plan: PlanView) -> AnnotationPlan {
347    let surface = plan.span.slice(source).unwrap_or("").to_string();
348    AnnotationPlan {
349        term_iri: plan.term_iri.to_string(),
350        start_utf8: plan.span.start_utf8,
351        end_utf8: plan.span.end_utf8,
352        content_hash: plan.content_hash,
353        source_hash: plan.source_hash,
354        surface,
355        kind: plan.kind,
356    }
357}
358
359fn document_analysis_from_layers(source: &str, layers: &YearOneLayers<'_>) -> DocumentAnalysis {
360    let mut plans = crate::nlp::emit::emit_from_hits(source, &layers.hits);
361    plans.extend(crate::nlp::emit::emit_from_normalized(
362        source,
363        &layers.norms,
364    ));
365    DocumentAnalysis {
366        source_hash: hash60(source.as_bytes()),
367        token_count: layers.tokens.len(),
368        sentence_count: layers.sentences.len(),
369        hits: layers.hits.clone(),
370        norms: layers.norms.clone(),
371        plans,
372    }
373}