qualia_semantic_library/ingest/
extract.rs1use super::acquire::{Acquired, SourceKind};
13
14pub struct Extracted {
16 pub text: String,
19 pub html: String,
21 pub page_count: u32,
22 pub extractor: String,
24 pub notes: Vec<String>,
26}
27
28pub fn extract(acq: &Acquired) -> Extracted {
30 match acq.kind {
31 SourceKind::Pdf => extract_pdf(acq),
32 SourceKind::Html => {
33 let html = String::from_utf8_lossy(&acq.bytes).to_string();
34 let text = html_to_text(&html);
35 Extracted {
36 text,
37 html,
38 page_count: 0,
39 extractor: "passthrough-html".into(),
40 notes: vec![],
41 }
42 }
43 SourceKind::Text | SourceKind::Markdown | SourceKind::Unknown => {
44 let text = String::from_utf8_lossy(&acq.bytes).to_string();
45 let html = text_to_html(&acq.source.title, &text);
46 Extracted {
47 text,
48 html,
49 page_count: 0,
50 extractor: "passthrough-text".into(),
51 notes: vec![],
52 }
53 }
54 SourceKind::Epub => {
55 let mut notes =
56 vec!["epub extraction not implemented in the pure-Rust baseline".into()];
57 notes.push("install a structured extractor to populate derived assets".into());
58 Extracted {
59 text: String::new(),
60 html: text_to_html(&acq.source.title, ""),
61 page_count: 0,
62 extractor: "none-epub".into(),
63 notes,
64 }
65 }
66 }
67}
68
69#[cfg(feature = "pdf")]
70fn extract_pdf(acq: &Acquired) -> Extracted {
71 let mut notes = Vec::new();
72 let page_count = pdf_page_count(&acq.bytes).unwrap_or(0);
73
74 let text = {
80 let prev = std::panic::take_hook();
81 std::panic::set_hook(Box::new(|_| {}));
82 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
83 pdf_extract::extract_text_from_mem(&acq.bytes)
84 }));
85 std::panic::set_hook(prev);
86 match result {
87 Ok(Ok(t)) => t,
88 Ok(Err(e)) => {
89 notes.push(format!("pdf-extract failed: {e}"));
90 String::new()
91 }
92 Err(_) => {
93 notes.push(
94 "pdf-extract panicked on this document — needs an alternate extractor".into(),
95 );
96 String::new()
97 }
98 }
99 };
100 if text.trim().is_empty() {
101 notes.push(
102 "no extractable text (likely scanned/image PDF) — needs OCR/vision extractor".into(),
103 );
104 }
105 notes.push("baseline extractor: no per-page markers or MathML".into());
106
107 let html = text_to_html(&acq.source.title, &text);
108 Extracted {
109 text,
110 html,
111 page_count,
112 extractor: "pdf-extract".into(),
113 notes,
114 }
115}
116
117#[cfg(not(feature = "pdf"))]
118fn extract_pdf(acq: &Acquired) -> Extracted {
119 Extracted {
120 text: String::new(),
121 html: text_to_html(&acq.source.title, ""),
122 page_count: 0,
123 extractor: "none-pdf-feature-off".into(),
124 notes: vec!["built without the `pdf` feature".into()],
125 }
126}
127
128#[cfg(feature = "pdf")]
129fn pdf_page_count(bytes: &[u8]) -> Option<u32> {
130 let doc = lopdf::Document::load_mem(bytes).ok()?;
131 Some(doc.get_pages().len() as u32)
132}
133
134fn text_to_html(title: &str, text: &str) -> String {
138 let mut out = String::with_capacity(text.len() + 256);
139 out.push_str("<!DOCTYPE html>\n<html><head><meta charset=\"utf-8\">");
140 out.push_str("<title>");
141 out.push_str(&html_escape::encode_text(title));
142 out.push_str("</title></head>\n<body data-doc-title=\"");
143 out.push_str(&html_escape::encode_double_quoted_attribute(title));
144 out.push_str("\">\n");
145
146 for block in text.split("\n\n") {
147 let trimmed = block.trim();
148 if trimmed.is_empty() {
149 continue;
150 }
151 if looks_like_heading(trimmed) {
152 out.push_str("<h2>");
153 out.push_str(&html_escape::encode_text(trimmed));
154 out.push_str("</h2>\n");
155 } else {
156 out.push_str("<p>");
157 out.push_str(&html_escape::encode_text(trimmed));
158 out.push_str("</p>\n");
159 }
160 }
161 out.push_str("</body></html>\n");
162 out
163}
164
165fn looks_like_heading(s: &str) -> bool {
166 if s.len() > 120 || s.contains('\n') {
167 return false;
168 }
169 let first = s.chars().next().unwrap_or(' ');
171 let numbered = first.is_ascii_digit() && s.split_whitespace().count() <= 12;
172 let shouty = s.chars().filter(|c| c.is_alphabetic()).count() > 2
173 && s.chars()
174 .filter(|c| c.is_alphabetic())
175 .all(|c| c.is_uppercase());
176 numbered || shouty
177}
178
179fn html_to_text(html: &str) -> String {
182 let mut out = String::with_capacity(html.len());
183 let mut in_tag = false;
184 for c in html.chars() {
185 match c {
186 '<' => in_tag = true,
187 '>' => {
188 in_tag = false;
189 out.push(' ');
190 }
191 _ if !in_tag => out.push(c),
192 _ => {}
193 }
194 }
195 out.split_whitespace().collect::<Vec<_>>().join(" ")
197}