Skip to main content

qualia_semantic_library/ingest/
acquire.rs

1//! Stage 1 — acquire: read a file, establish its identity (BLAKE3), and detect
2//! its source kind. No extraction yet; this stage only produces the verbatim
3//! bytes + a [`SourceInfo`] the rest of the pipeline keys on.
4
5use std::path::Path;
6
7use crate::container::{blake3_hex, SourceInfo};
8
9/// What kind of source we are looking at — drives extractor selection.
10#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11pub enum SourceKind {
12    Pdf,
13    Html,
14    Text,
15    Markdown,
16    Epub,
17    Unknown,
18}
19
20impl SourceKind {
21    /// MIME type.
22    pub fn mime(self) -> &'static str {
23        match self {
24            SourceKind::Pdf => "application/pdf",
25            SourceKind::Html => "text/html",
26            SourceKind::Text => "text/plain",
27            SourceKind::Markdown => "text/markdown",
28            SourceKind::Epub => "application/epub+zip",
29            SourceKind::Unknown => "application/octet-stream",
30        }
31    }
32}
33
34/// The verbatim source plus its derived identity/description.
35pub struct Acquired {
36    pub bytes: Vec<u8>,
37    pub kind: SourceKind,
38    pub source: SourceInfo,
39}
40
41/// Best-effort source-kind detection from magic bytes first, extension second.
42pub fn detect_kind(path: &Path, bytes: &[u8]) -> SourceKind {
43    if bytes.starts_with(b"%PDF-") {
44        return SourceKind::Pdf;
45    }
46    // EPUB / OOXML are zips; only call EPUB if the extension says so.
47    let ext = path
48        .extension()
49        .and_then(|e| e.to_str())
50        .map(|e| e.to_ascii_lowercase())
51        .unwrap_or_default();
52    match ext.as_str() {
53        "pdf" => SourceKind::Pdf,
54        "html" | "htm" | "xhtml" => SourceKind::Html,
55        "md" | "markdown" => SourceKind::Markdown,
56        "txt" | "text" => SourceKind::Text,
57        "epub" => SourceKind::Epub,
58        _ => {
59            // Sniff for an HTML doctype/tag near the start.
60            let head = &bytes[..bytes.len().min(512)];
61            let lower = String::from_utf8_lossy(head).to_ascii_lowercase();
62            if lower.contains("<!doctype html") || lower.contains("<html") {
63                SourceKind::Html
64            } else if bytes.is_empty() {
65                SourceKind::Unknown
66            } else if head
67                .iter()
68                .all(|&b| b == b'\n' || b == b'\r' || b == b'\t' || b >= 0x20)
69            {
70                SourceKind::Text
71            } else {
72                SourceKind::Unknown
73            }
74        }
75    }
76}
77
78/// Acquires.
79pub fn acquire(path: &Path) -> std::io::Result<Acquired> {
80    let bytes = std::fs::read(path)?;
81    let kind = detect_kind(path, &bytes);
82    let filename = path
83        .file_name()
84        .and_then(|n| n.to_str())
85        .unwrap_or("unnamed")
86        .to_string();
87    let title = path
88        .file_stem()
89        .and_then(|n| n.to_str())
90        .unwrap_or("")
91        .to_string();
92    let source = SourceInfo {
93        filename,
94        mime: kind.mime().to_string(),
95        size_bytes: bytes.len() as u64,
96        blake3: blake3_hex(&bytes),
97        title,
98        page_count: 0,
99    };
100    Ok(Acquired {
101        bytes,
102        kind,
103        source,
104    })
105}