qualia_semantic_library/ingest/
mod.rs1pub mod acquire;
10pub mod chunk;
11pub mod cml;
12pub mod extract;
13
14use std::path::{Path, PathBuf};
15
16use walkdir::WalkDir;
17
18use crate::container::{AssetKind, HmcWriter, PipelineInfo};
19
20pub struct IngestOptions {
22 pub out_dir: PathBuf,
24 pub recursive: bool,
26 pub skip_existing: bool,
28}
29
30impl Default for IngestOptions {
31 fn default() -> Self {
32 IngestOptions {
33 out_dir: PathBuf::from("library"),
34 recursive: true,
35 skip_existing: true,
36 }
37 }
38}
39
40pub struct IngestResult {
42 pub container_path: PathBuf,
43 pub doc_id: String,
44 pub skipped: bool,
45 pub notes: Vec<String>,
46}
47
48pub fn ingest_file(path: &Path, opts: &IngestOptions) -> anyhow::Result<IngestResult> {
50 let acq = acquire::acquire(path)?;
51 let doc_id = acq.source.blake3.clone();
52 let target = opts
53 .out_dir
54 .join(format!("{doc_id}.{}", crate::container::HMC_EXTENSION));
55
56 if opts.skip_existing && target.exists() {
57 return Ok(IngestResult {
58 container_path: target,
59 doc_id,
60 skipped: true,
61 notes: vec![],
62 });
63 }
64
65 let mut source = acq.source.clone();
66 let extracted = extract::extract(&acq);
67 source.page_count = extracted.page_count;
68
69 let chunks = chunk::chunk_text(&extracted.text);
70
71 let mut w = HmcWriter::new(source, &acq.bytes);
72 w.manifest_mut().source.page_count = extracted.page_count;
74
75 w.add_derived(
76 AssetKind::Text,
77 "document.txt",
78 "text/plain",
79 extracted.text.into_bytes(),
80 );
81 w.add_derived(
82 AssetKind::Html,
83 "document.html",
84 "text/html",
85 extracted.html.into_bytes(),
86 );
87
88 let jsonl = chunk::chunks_to_jsonl(&chunks);
89 w.add_derived(
90 AssetKind::Chunks,
91 "chunks.jsonl",
92 "application/jsonl",
93 jsonl.into_bytes(),
94 );
95
96 let ttl = cml::build_cml(w.manifest(), &chunks);
97 w.add_derived(
98 AssetKind::Cml,
99 "document.cml.ttl",
100 "text/turtle",
101 ttl.into_bytes(),
102 );
103
104 w.manifest_mut().pipeline = PipelineInfo {
105 tool: format!("qualia-semantic-library {}", env!("CARGO_PKG_VERSION")),
106 extractor: extracted.extractor,
107 embedder: String::new(),
108 embed_dim: 0,
109 };
110 w.manifest_mut().status.extracted = !chunks.is_empty();
111 w.manifest_mut().status.chunked = !chunks.is_empty();
112 w.manifest_mut().status.notes = extracted.notes.clone();
113
114 let container_path = w.write_to_dir(&opts.out_dir)?;
115 Ok(IngestResult {
116 container_path,
117 doc_id,
118 skipped: false,
119 notes: extracted.notes,
120 })
121}
122
123pub fn ingest_path(path: &Path, opts: &IngestOptions) -> anyhow::Result<Vec<IngestResult>> {
125 let mut results = Vec::new();
126 if path.is_file() {
127 results.push(ingest_file(path, opts)?);
128 return Ok(results);
129 }
130
131 let walker = WalkDir::new(path).max_depth(if opts.recursive { usize::MAX } else { 1 });
132 for entry in walker.into_iter().filter_map(|e| e.ok()) {
133 if !entry.file_type().is_file() {
134 continue;
135 }
136 let p = entry.path();
137 if !is_ingestible(p) {
138 continue;
139 }
140 let prev = std::panic::take_hook();
143 std::panic::set_hook(Box::new(|_| {}));
144 let outcome =
145 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| ingest_file(p, opts)));
146 std::panic::set_hook(prev);
147 match outcome {
148 Ok(Ok(r)) => results.push(r),
149 Ok(Err(e)) => eprintln!("[ingest] FAILED {}: {e}", p.display()),
150 Err(_) => eprintln!("[ingest] PANIC (skipped) {}", p.display()),
151 }
152 }
153 Ok(results)
154}
155
156fn is_ingestible(p: &Path) -> bool {
157 matches!(
158 p.extension()
159 .and_then(|e| e.to_str())
160 .map(|e| e.to_ascii_lowercase())
161 .as_deref(),
162 Some("pdf" | "html" | "htm" | "xhtml" | "txt" | "text" | "md" | "markdown" | "epub")
163 )
164}