Skip to main content

qualia_semantic_library/ingest/
mod.rs

1//! Ingest orchestration: a single file → a `.hmc` hypermedia container.
2//!
3//! This whole module is **deterministic** — no LLM. It acquires the source,
4//! extracts structured text/HTML, chunks it, emits the CML annotation layer, and
5//! packs the original plus every derived asset into one container. Embeddings
6//! and LLM method-extraction are separate enrichment passes ([`crate::llm`]) so
7//! ingest stays reproducible and offline-clean.
8
9pub mod acquire;
10pub mod chunk;
11pub mod cml;
12pub mod extract;
13
14use std::path::{Path, PathBuf};
15
16use walkdir::WalkDir;
17
18use crate::container::{AssetKind, HmcWriter, PipelineInfo};
19
20/// Options controlling an ingest run.
21pub struct IngestOptions {
22    /// Directory the `.hmc` containers are written to.
23    pub out_dir: PathBuf,
24    /// Recurse into sub-directories when ingesting a directory.
25    pub recursive: bool,
26    /// Skip a source whose container already exists in `out_dir`.
27    pub skip_existing: bool,
28}
29
30impl Default for IngestOptions {
31    fn default() -> Self {
32        IngestOptions {
33            out_dir: PathBuf::from("library"),
34            recursive: true,
35            skip_existing: true,
36        }
37    }
38}
39
40/// Outcome of ingesting one file.
41pub struct IngestResult {
42    pub container_path: PathBuf,
43    pub doc_id: String,
44    pub skipped: bool,
45    pub notes: Vec<String>,
46}
47
48/// Ingest a single file into a container. Returns the container path.
49pub fn ingest_file(path: &Path, opts: &IngestOptions) -> anyhow::Result<IngestResult> {
50    let acq = acquire::acquire(path)?;
51    let doc_id = acq.source.blake3.clone();
52    let target = opts
53        .out_dir
54        .join(format!("{doc_id}.{}", crate::container::HMC_EXTENSION));
55
56    if opts.skip_existing && target.exists() {
57        return Ok(IngestResult {
58            container_path: target,
59            doc_id,
60            skipped: true,
61            notes: vec![],
62        });
63    }
64
65    let mut source = acq.source.clone();
66    let extracted = extract::extract(&acq);
67    source.page_count = extracted.page_count;
68
69    let chunks = chunk::chunk_text(&extracted.text);
70
71    let mut w = HmcWriter::new(source, &acq.bytes);
72    // page count from extraction
73    w.manifest_mut().source.page_count = extracted.page_count;
74
75    w.add_derived(
76        AssetKind::Text,
77        "document.txt",
78        "text/plain",
79        extracted.text.into_bytes(),
80    );
81    w.add_derived(
82        AssetKind::Html,
83        "document.html",
84        "text/html",
85        extracted.html.into_bytes(),
86    );
87
88    let jsonl = chunk::chunks_to_jsonl(&chunks);
89    w.add_derived(
90        AssetKind::Chunks,
91        "chunks.jsonl",
92        "application/jsonl",
93        jsonl.into_bytes(),
94    );
95
96    let ttl = cml::build_cml(w.manifest(), &chunks);
97    w.add_derived(
98        AssetKind::Cml,
99        "document.cml.ttl",
100        "text/turtle",
101        ttl.into_bytes(),
102    );
103
104    w.manifest_mut().pipeline = PipelineInfo {
105        tool: format!("qualia-semantic-library {}", env!("CARGO_PKG_VERSION")),
106        extractor: extracted.extractor,
107        embedder: String::new(),
108        embed_dim: 0,
109    };
110    w.manifest_mut().status.extracted = !chunks.is_empty();
111    w.manifest_mut().status.chunked = !chunks.is_empty();
112    w.manifest_mut().status.notes = extracted.notes.clone();
113
114    let container_path = w.write_to_dir(&opts.out_dir)?;
115    Ok(IngestResult {
116        container_path,
117        doc_id,
118        skipped: false,
119        notes: extracted.notes,
120    })
121}
122
123/// Ingest a file or a directory tree. Returns per-file results.
124pub fn ingest_path(path: &Path, opts: &IngestOptions) -> anyhow::Result<Vec<IngestResult>> {
125    let mut results = Vec::new();
126    if path.is_file() {
127        results.push(ingest_file(path, opts)?);
128        return Ok(results);
129    }
130
131    let walker = WalkDir::new(path).max_depth(if opts.recursive { usize::MAX } else { 1 });
132    for entry in walker.into_iter().filter_map(|e| e.ok()) {
133        if !entry.file_type().is_file() {
134            continue;
135        }
136        let p = entry.path();
137        if !is_ingestible(p) {
138            continue;
139        }
140        // Defence in depth: a panic anywhere in extract/chunk/cml on one
141        // document must not abort a 60 GB sweep. Isolate every file.
142        let prev = std::panic::take_hook();
143        std::panic::set_hook(Box::new(|_| {}));
144        let outcome =
145            std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| ingest_file(p, opts)));
146        std::panic::set_hook(prev);
147        match outcome {
148            Ok(Ok(r)) => results.push(r),
149            Ok(Err(e)) => eprintln!("[ingest] FAILED {}: {e}", p.display()),
150            Err(_) => eprintln!("[ingest] PANIC (skipped) {}", p.display()),
151        }
152    }
153    Ok(results)
154}
155
156fn is_ingestible(p: &Path) -> bool {
157    matches!(
158        p.extension()
159            .and_then(|e| e.to_str())
160            .map(|e| e.to_ascii_lowercase())
161            .as_deref(),
162        Some("pdf" | "html" | "htm" | "xhtml" | "txt" | "text" | "md" | "markdown" | "epub")
163    )
164}