Skip to main content

qualia_semantic_library/ingest/
extract.rs

1//! Stage 2 — extract: turn the verbatim source into a canonical **structured**
2//! representation: plain text (the cheap full-text surface) and canonical HTML
3//! (the structure-bearing surface that the CML layer annotates).
4//!
5//! The default extractor is pure-Rust `pdf-extract` so the tool always builds
6//! and runs with no system libraries. It produces faithful text but no
7//! per-page markers or MathML; those are the quality knob delivered by heavier
8//! extractors (PyMuPDF / nougat) wired in behind the `extractor` field later.
9//! HTML here is built deterministically from the text by paragraph/heading
10//! heuristics — honest structure, not invented content.
11
12use super::acquire::{Acquired, SourceKind};
13
14/// Result of extraction — what becomes the `derived/` assets.
15pub struct Extracted {
16    /// Plain text (UTF-8). Page markers `[[page N]]` when the extractor supplies
17    /// them; otherwise a single stream.
18    pub text: String,
19    /// Canonical structured HTML.
20    pub html: String,
21    pub page_count: u32,
22    /// Extractor identifier recorded in the manifest pipeline info.
23    pub extractor: String,
24    /// Non-fatal warnings (e.g. "no per-page markers", "scanned/empty text").
25    pub notes: Vec<String>,
26}
27
28/// Extracts.
29pub fn extract(acq: &Acquired) -> Extracted {
30    match acq.kind {
31        SourceKind::Pdf => extract_pdf(acq),
32        SourceKind::Html => {
33            let html = String::from_utf8_lossy(&acq.bytes).to_string();
34            let text = html_to_text(&html);
35            Extracted {
36                text,
37                html,
38                page_count: 0,
39                extractor: "passthrough-html".into(),
40                notes: vec![],
41            }
42        }
43        SourceKind::Text | SourceKind::Markdown | SourceKind::Unknown => {
44            let text = String::from_utf8_lossy(&acq.bytes).to_string();
45            let html = text_to_html(&acq.source.title, &text);
46            Extracted {
47                text,
48                html,
49                page_count: 0,
50                extractor: "passthrough-text".into(),
51                notes: vec![],
52            }
53        }
54        SourceKind::Epub => {
55            let mut notes =
56                vec!["epub extraction not implemented in the pure-Rust baseline".into()];
57            notes.push("install a structured extractor to populate derived assets".into());
58            Extracted {
59                text: String::new(),
60                html: text_to_html(&acq.source.title, ""),
61                page_count: 0,
62                extractor: "none-epub".into(),
63                notes,
64            }
65        }
66    }
67}
68
69#[cfg(feature = "pdf")]
70fn extract_pdf(acq: &Acquired) -> Extracted {
71    let mut notes = Vec::new();
72    let page_count = pdf_page_count(&acq.bytes).unwrap_or(0);
73
74    // `pdf-extract` panics on some malformed/unusual PDFs. A 60 GB sweep must
75    // not die on one bad file, so we isolate the call: a panic becomes an empty
76    // extraction + a note, the verbatim source is still preserved, and a better
77    // extractor can re-process the container later. The panic hook is silenced
78    // around the call to keep batch output clean (ingest is single-threaded).
79    let text = {
80        let prev = std::panic::take_hook();
81        std::panic::set_hook(Box::new(|_| {}));
82        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
83            pdf_extract::extract_text_from_mem(&acq.bytes)
84        }));
85        std::panic::set_hook(prev);
86        match result {
87            Ok(Ok(t)) => t,
88            Ok(Err(e)) => {
89                notes.push(format!("pdf-extract failed: {e}"));
90                String::new()
91            }
92            Err(_) => {
93                notes.push(
94                    "pdf-extract panicked on this document — needs an alternate extractor".into(),
95                );
96                String::new()
97            }
98        }
99    };
100    if text.trim().is_empty() {
101        notes.push(
102            "no extractable text (likely scanned/image PDF) — needs OCR/vision extractor".into(),
103        );
104    }
105    notes.push("baseline extractor: no per-page markers or MathML".into());
106
107    let html = text_to_html(&acq.source.title, &text);
108    Extracted {
109        text,
110        html,
111        page_count,
112        extractor: "pdf-extract".into(),
113        notes,
114    }
115}
116
117#[cfg(not(feature = "pdf"))]
118fn extract_pdf(acq: &Acquired) -> Extracted {
119    Extracted {
120        text: String::new(),
121        html: text_to_html(&acq.source.title, ""),
122        page_count: 0,
123        extractor: "none-pdf-feature-off".into(),
124        notes: vec!["built without the `pdf` feature".into()],
125    }
126}
127
128#[cfg(feature = "pdf")]
129fn pdf_page_count(bytes: &[u8]) -> Option<u32> {
130    let doc = lopdf::Document::load_mem(bytes).ok()?;
131    Some(doc.get_pages().len() as u32)
132}
133
134/// Build canonical HTML from plain text. Blank-line-separated blocks become
135/// paragraphs; short ALL-CAPS / numbered lines are treated as headings. The
136/// document `<body>` carries a `data-doc-title` for provenance.
137fn text_to_html(title: &str, text: &str) -> String {
138    let mut out = String::with_capacity(text.len() + 256);
139    out.push_str("<!DOCTYPE html>\n<html><head><meta charset=\"utf-8\">");
140    out.push_str("<title>");
141    out.push_str(&html_escape::encode_text(title));
142    out.push_str("</title></head>\n<body data-doc-title=\"");
143    out.push_str(&html_escape::encode_double_quoted_attribute(title));
144    out.push_str("\">\n");
145
146    for block in text.split("\n\n") {
147        let trimmed = block.trim();
148        if trimmed.is_empty() {
149            continue;
150        }
151        if looks_like_heading(trimmed) {
152            out.push_str("<h2>");
153            out.push_str(&html_escape::encode_text(trimmed));
154            out.push_str("</h2>\n");
155        } else {
156            out.push_str("<p>");
157            out.push_str(&html_escape::encode_text(trimmed));
158            out.push_str("</p>\n");
159        }
160    }
161    out.push_str("</body></html>\n");
162    out
163}
164
165fn looks_like_heading(s: &str) -> bool {
166    if s.len() > 120 || s.contains('\n') {
167        return false;
168    }
169    // Numbered section ("3.2 Foo") or short Title Case / ALL CAPS line.
170    let first = s.chars().next().unwrap_or(' ');
171    let numbered = first.is_ascii_digit() && s.split_whitespace().count() <= 12;
172    let shouty = s.chars().filter(|c| c.is_alphabetic()).count() > 2
173        && s.chars()
174            .filter(|c| c.is_alphabetic())
175            .all(|c| c.is_uppercase());
176    numbered || shouty
177}
178
179/// Very small HTML→text fallback (strips tags). Adequate for the cheap
180/// full-text surface; the HTML asset remains the structured source of truth.
181fn html_to_text(html: &str) -> String {
182    let mut out = String::with_capacity(html.len());
183    let mut in_tag = false;
184    for c in html.chars() {
185        match c {
186            '<' => in_tag = true,
187            '>' => {
188                in_tag = false;
189                out.push(' ');
190            }
191            _ if !in_tag => out.push(c),
192            _ => {}
193        }
194    }
195    // collapse runs of whitespace
196    out.split_whitespace().collect::<Vec<_>>().join(" ")
197}