Skip to main content

qualia_core_db/nlp/
mod.rs

1//! Document NLP in the engine: bytes → tokens → gazetteer → span plans.
2//!
3//! Lives next to `text_span` and `lexicon`. Not VibeScript. Not an MCP-only tool.
4//! Year-one is tokenize + gazetteer + spans. Not FrameNet, RST, OpenIE, NLI, or MT.
5
6pub mod budget;
7pub mod capability;
8pub mod contracts;
9pub mod coref;
10pub mod emit;
11pub mod frame;
12pub mod fst;
13pub mod gazetteer;
14pub mod graphrag;
15pub mod hash;
16pub mod link;
17pub mod normalize;
18pub mod relation;
19pub mod span;
20pub mod substrate;
21pub mod terms;
22pub mod tokenize;
23
24pub use capability::{lookup, NlpCapability, NLP_CAPABILITIES};
25pub use contracts::{
26    analyze_document_into, required_capacities, slice_source, AnalysisState, BufferChannel,
27    DocumentCapacities, DocumentSummary, DocumentViewBuffers, HitView, NlpContractError, NormView,
28    PlanView, SentenceView, TokenView, ANNOTATION_CONTRACT_VERSION, NORM_INDEX_NONE,
29    SENTENCE_ID_NONE,
30};
31
32use emit::AnnotationPlan;
33use gazetteer::Hit;
34use normalize::Normalized;
35
36/// One-shot document analysis. Desktop and Vibe hosts call this; they do not
37/// reimplement tokenisation.
38#[derive(Debug, Clone)]
39pub struct DocumentAnalysis {
40    pub source_hash: u64,
41    pub token_count: usize,
42    pub sentence_count: usize,
43    pub hits: Vec<Hit>,
44    pub norms: Vec<Normalized>,
45    pub plans: Vec<AnnotationPlan>,
46}
47
48/// Analyzes a document into annotations.
49pub fn analyze_document(source: &str) -> DocumentAnalysis {
50    contracts::analyze_document(source)
51}
52
53#[cfg(test)]
54mod tests {
55    use super::*;
56
57    #[test]
58    fn year_one_pipeline_on_catchment_paragraph() {
59        let src = "North Spring is the reference site. Timothy Charles Holborn recorded 12.5 mm of rain on 2026-08-15.";
60        let a = analyze_document(src);
61        assert!(a.token_count >= 10);
62        assert!(a.sentence_count >= 2);
63        assert!(a.plans.len() >= 4);
64        assert_ne!(a.source_hash, 0);
65    }
66}