Skip to main content

qualia_semantic_library/container/
mod.rs

1//! The `.hmc` **hypermedia container**: a single, fast-compression (ZIP) file
2//! that bundles an original source document together with all of its derived
3//! assets (canonical HTML, CML/RDF, plain text, structural chunks, embeddings,
4//! extraction metadata) behind one self-describing [`HmcManifest`].
5//!
6//! Why a container per document:
7//!   * the original and everything derived from it travel together — provenance
8//!     never gets separated from the source;
9//!   * ZIP gives random access, partial reads, and is readable by every tool;
10//!   * re-running the pipeline updates derived assets in place without touching
11//!     the verbatim source.
12//!
13//! Layout inside the zip:
14//! ```text
15//! manifest.json              # the index (HmcManifest)
16//! source/<original>          # verbatim source bytes
17//! derived/document.html      # canonical structured HTML
18//! derived/document.txt       # plain text with [[page N]] markers
19//! derived/document.cml.ttl   # CML / RDF annotations (optional)
20//! derived/chunks.jsonl       # structural chunks (optional)
21//! embeddings/vectors.f32     # row-major f32 embedding matrix (optional)
22//! meta/extraction.json       # tool versions / timings / confidence (optional)
23//! ```
24
25mod manifest;
26pub use manifest::*;
27
28use std::io::{Cursor, Read, Write};
29use std::path::Path;
30
31use thiserror::Error;
32use zip::write::SimpleFileOptions;
33
34/// Conventional file extension for a hypermedia container.
35pub const HMC_EXTENSION: &str = "hmc";
36const MANIFEST_NAME: &str = "manifest.json";
37
38#[derive(Debug, Error)]
39pub enum HmcError {
40    #[error("io: {0}")]
41    Io(#[from] std::io::Error),
42    #[error("zip: {0}")]
43    Zip(#[from] zip::result::ZipError),
44    #[error("json: {0}")]
45    Json(#[from] serde_json::Error),
46    #[error("container has no {MANIFEST_NAME}")]
47    MissingManifest,
48    #[error("asset not found: {0}")]
49    AssetNotFound(String),
50    #[error("integrity: asset {path} blake3 mismatch (manifest {want}, actual {got})")]
51    Integrity {
52        path: String,
53        want: String,
54        got: String,
55    },
56    #[error("invalid container: {0}")]
57    Invalid(String),
58}
59
60pub type Result<T> = std::result::Result<T, HmcError>;
61
62/// BLAKE3 hex of a byte slice — the hashing used throughout the container.
63pub fn blake3_hex(bytes: &[u8]) -> String {
64    blake3::hash(bytes).to_hex().to_string()
65}
66
67/// Builder that accumulates assets in memory and writes a `.hmc` zip in one shot.
68///
69/// Kept in-memory because individual documents are small relative to RAM and a
70/// single atomic write avoids half-written containers on crash/interrupt.
71pub struct HmcWriter {
72    manifest: HmcManifest,
73    /// (path inside zip, bytes)
74    members: Vec<(String, Vec<u8>)>,
75}
76
77impl HmcWriter {
78    /// Start a container from the (already-hashed) source description and bytes.
79    pub fn new(mut source: SourceInfo, source_bytes: &[u8]) -> Self {
80        // Ensure the source hash is authoritative.
81        source.blake3 = blake3_hex(source_bytes);
82        source.size_bytes = source_bytes.len() as u64;
83        let manifest = HmcManifest::new(source.clone());
84
85        let mut w = HmcWriter {
86            manifest,
87            members: Vec::new(),
88        };
89        let path = format!("{}/{}", AssetKind::Source.dir(), sanitize(&source.filename));
90        w.push_asset(
91            AssetKind::Source,
92            &path,
93            &source.mime.clone(),
94            source_bytes.to_vec(),
95        );
96        w
97    }
98
99    fn push_asset(&mut self, kind: AssetKind, path: &str, mime: &str, bytes: Vec<u8>) {
100        let entry = AssetEntry {
101            path: path.to_string(),
102            kind,
103            mime: mime.to_string(),
104            blake3: blake3_hex(&bytes),
105            bytes: bytes.len() as u64,
106        };
107        // Replace an existing asset at the same path (idempotent re-runs).
108        if let Some(i) = self.manifest.assets.iter().position(|a| a.path == path) {
109            self.manifest.assets[i] = entry;
110            if let Some(m) = self.members.iter_mut().find(|(p, _)| p == path) {
111                m.1 = bytes;
112                return;
113            }
114        } else {
115            self.manifest.assets.push(entry);
116        }
117        self.members.push((path.to_string(), bytes));
118    }
119
120    /// Add a derived asset under its conventional directory using `name`
121    /// (e.g. `document.html`). Returns the in-zip path.
122    pub fn add_derived(
123        &mut self,
124        kind: AssetKind,
125        name: &str,
126        mime: &str,
127        bytes: Vec<u8>,
128    ) -> String {
129        let path = format!("{}/{}", kind.dir(), sanitize(name));
130        self.push_asset(kind, &path, mime, bytes);
131        path
132    }
133
134    /// Mutable access to the manifest to set pipeline/status/tags before finalize.
135    pub fn manifest_mut(&mut self) -> &mut HmcManifest {
136        &mut self.manifest
137    }
138
139    /// Serialize the manifest + all members into ZIP bytes.
140    pub fn to_bytes(&self) -> Result<Vec<u8>> {
141        let mut buf = Cursor::new(Vec::new());
142        {
143            let mut zw = zip::ZipWriter::new(&mut buf);
144            let stored =
145                SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
146            let deflated =
147                SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated);
148
149            // Manifest first, uncompressed, for instant header reads.
150            let manifest_json = serde_json::to_vec_pretty(&self.manifest)?;
151            zw.start_file(MANIFEST_NAME, stored)?;
152            zw.write_all(&manifest_json)?;
153
154            for (path, bytes) in &self.members {
155                // Don't waste CPU re-compressing the source if it's already a
156                // compressed format (PDF); deflate the text-like derived assets.
157                let already_compressed =
158                    path.starts_with("source/") && is_precompressed_mime(self.member_mime(path));
159                let opts = if already_compressed { stored } else { deflated };
160                zw.start_file(path.as_str(), opts)?;
161                zw.write_all(bytes)?;
162            }
163            zw.finish()?;
164        }
165        Ok(buf.into_inner())
166    }
167
168    fn member_mime(&self, path: &str) -> &str {
169        self.manifest
170            .assets
171            .iter()
172            .find(|a| a.path == path)
173            .map(|a| a.mime.as_str())
174            .unwrap_or("application/octet-stream")
175    }
176
177    /// Write the container to `dir/<doc_id>.hmc` and return the path.
178    pub fn write_to_dir(&self, dir: &Path) -> Result<std::path::PathBuf> {
179        std::fs::create_dir_all(dir)?;
180        let out = dir.join(format!("{}.{}", self.manifest.doc_id, HMC_EXTENSION));
181        std::fs::write(&out, self.to_bytes()?)?;
182        Ok(out)
183    }
184
185    /// Manifest.
186    pub fn manifest(&self) -> &HmcManifest {
187        &self.manifest
188    }
189
190    /// Load an existing container back into a writer so an enrichment pass
191    /// (embeddings, analysis tags, CML) can add/replace derived assets and
192    /// re-emit the container in place. The verbatim source is preserved.
193    pub fn reopen(path: &Path) -> Result<Self> {
194        let mut c = HmcContainer::open(path)?;
195        let manifest = c.manifest().clone();
196        let mut members = Vec::with_capacity(manifest.assets.len());
197        for a in &manifest.assets {
198            members.push((a.path.clone(), c.read_asset_bytes(&a.path)?));
199        }
200        Ok(HmcWriter { manifest, members })
201    }
202}
203
204/// Read side: opens a `.hmc`, exposes the manifest, and reads assets on demand.
205pub struct HmcContainer {
206    manifest: HmcManifest,
207    archive: zip::ZipArchive<Cursor<Vec<u8>>>,
208}
209
210impl HmcContainer {
211    /// Opens.
212    pub fn open(path: &Path) -> Result<Self> {
213        Self::from_bytes(std::fs::read(path)?)
214    }
215
216    /// Parses bytes.
217    pub fn from_bytes(bytes: Vec<u8>) -> Result<Self> {
218        let mut archive = zip::ZipArchive::new(Cursor::new(bytes))?;
219        let manifest: HmcManifest = {
220            let mut f = archive
221                .by_name(MANIFEST_NAME)
222                .map_err(|_| HmcError::MissingManifest)?;
223            let mut s = String::new();
224            f.read_to_string(&mut s)?;
225            serde_json::from_str(&s)?
226        };
227        if manifest.format != "hmc" {
228            return Err(HmcError::Invalid(format!(
229                "unexpected format `{}`",
230                manifest.format
231            )));
232        }
233        Ok(HmcContainer { manifest, archive })
234    }
235
236    /// Manifest.
237    pub fn manifest(&self) -> &HmcManifest {
238        &self.manifest
239    }
240
241    /// Raw bytes of a stored member by its in-zip path.
242    pub fn read_asset_bytes(&mut self, path: &str) -> Result<Vec<u8>> {
243        let mut f = self
244            .archive
245            .by_name(path)
246            .map_err(|_| HmcError::AssetNotFound(path.to_string()))?;
247        let mut out = Vec::with_capacity(f.size() as usize);
248        f.read_to_end(&mut out)?;
249        Ok(out)
250    }
251
252    /// Bytes of the first asset of a given kind.
253    pub fn read_kind(&mut self, kind: AssetKind) -> Result<Vec<u8>> {
254        let path = self
255            .manifest
256            .asset_of(kind)
257            .map(|a| a.path.clone())
258            .ok_or_else(|| HmcError::AssetNotFound(format!("kind {:?}", kind)))?;
259        self.read_asset_bytes(&path)
260    }
261
262    /// Verify every asset's stored bytes hash to the value recorded in the
263    /// manifest. Returns `Ok(())` on a clean container.
264    pub fn verify(&mut self) -> Result<()> {
265        let entries: Vec<(String, String)> = self
266            .manifest
267            .assets
268            .iter()
269            .map(|a| (a.path.clone(), a.blake3.clone()))
270            .collect();
271        for (path, want) in entries {
272            let got = blake3_hex(&self.read_asset_bytes(&path)?);
273            if got != want {
274                return Err(HmcError::Integrity { path, want, got });
275            }
276        }
277        Ok(())
278    }
279}
280
281/// MIME types whose bytes are already compressed — re-deflating wastes CPU.
282fn is_precompressed_mime(mime: &str) -> bool {
283    matches!(
284        mime,
285        "application/pdf"
286            | "image/png"
287            | "image/jpeg"
288            | "image/webp"
289            | "application/zip"
290            | "application/epub+zip"
291    )
292}
293
294/// Make a filename safe as a zip member path component (no traversal, no
295/// backslashes, no leading slash).
296fn sanitize(name: &str) -> String {
297    let base = name
298        .rsplit(['/', '\\'])
299        .next()
300        .unwrap_or(name)
301        .trim()
302        .replace('\u{0}', "");
303    if base.is_empty() {
304        "unnamed".to_string()
305    } else {
306        base
307    }
308}
309
310#[cfg(test)]
311mod tests;