Skip to main content

qualia_semantic_library/container/
manifest.rs

1//! The `manifest.json` that sits at the root of every `.hmc` container.
2//!
3//! The manifest is the self-describing index of a hypermedia container: it names
4//! the original source, every derived asset, content hashes, and the pipeline
5//! that produced them. A reader can understand a container fully from the
6//! manifest alone, without unpacking the heavy assets.
7
8use serde::{Deserialize, Serialize};
9
10/// Current on-disk format version of the `.hmc` container.
11pub const HMC_FORMAT_VERSION: u32 = 1;
12
13/// Logical role of an asset inside a container. The path in [`AssetEntry`] is
14/// authoritative; `kind` is the typed lens used for retrieval/UX.
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(rename_all = "snake_case")]
17pub enum AssetKind {
18    /// The original, verbatim source file (e.g. the PDF). Never modified.
19    Source,
20    /// Canonical structured HTML (headings, MathML, provenance `data-*` attrs).
21    Html,
22    /// Plain text with `[[page N]]` markers (compat + cheap full-text).
23    Text,
24    /// CML / RDF (Turtle) semantic annotation layer over the HTML.
25    Cml,
26    /// Structural chunks as JSON-Lines (one object per line).
27    Chunks,
28    /// Binary embedding matrix for the chunks (`f32`, row-major).
29    Embeddings,
30    /// Extraction/analysis metadata (tool versions, timings, confidence).
31    Meta,
32    /// Anything else (figures, tables, page images…).
33    Other,
34}
35
36impl AssetKind {
37    /// Conventional directory prefix for this kind inside the zip.
38    pub fn dir(self) -> &'static str {
39        match self {
40            AssetKind::Source => "source",
41            AssetKind::Html | AssetKind::Text | AssetKind::Cml | AssetKind::Chunks => "derived",
42            AssetKind::Embeddings => "embeddings",
43            AssetKind::Meta => "meta",
44            AssetKind::Other => "assets",
45        }
46    }
47}
48
49/// One stored member of the container.
50#[derive(Debug, Clone, Serialize, Deserialize)]
51pub struct AssetEntry {
52    /// Path of the member inside the zip (e.g. `derived/document.html`).
53    pub path: String,
54    /// Logical role.
55    pub kind: AssetKind,
56    /// MIME type, best-effort (e.g. `text/html`, `application/pdf`).
57    pub mime: String,
58    /// BLAKE3 hex of the asset bytes (integrity + dedup).
59    pub blake3: String,
60    /// Uncompressed size in bytes.
61    pub bytes: u64,
62}
63
64/// Description of the original source document.
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct SourceInfo {
67    /// Original filename as found on disk.
68    pub filename: String,
69    pub mime: String,
70    pub size_bytes: u64,
71    /// BLAKE3 hex of the original bytes — the document identity.
72    pub blake3: String,
73    /// Title (from document metadata, else derived from filename).
74    #[serde(default)]
75    pub title: String,
76    /// Page count when known (PDFs); 0 otherwise.
77    #[serde(default)]
78    pub page_count: u32,
79}
80
81/// Which tools produced this container — for reproducibility and re-runs.
82#[derive(Debug, Clone, Default, Serialize, Deserialize)]
83pub struct PipelineInfo {
84    /// Producing tool + version (e.g. `qualia-semantic-library 0.0.20`).
85    pub tool: String,
86    /// Text/structure extractor used (e.g. `pdf-extract`, `pymupdf`, `nougat`).
87    #[serde(default)]
88    pub extractor: String,
89    /// Embedding model id, if embeddings were computed.
90    #[serde(default)]
91    pub embedder: String,
92    /// Embedding dimensionality (0 if none).
93    #[serde(default)]
94    pub embed_dim: u32,
95}
96
97/// Coarse processing status carried in the manifest so a library scan can route
98/// work (what still needs extraction / embedding / analysis) without unpacking.
99#[derive(Debug, Clone, Default, Serialize, Deserialize)]
100pub struct StatusFlags {
101    pub extracted: bool,
102    pub chunked: bool,
103    pub embedded: bool,
104    pub analyzed: bool,
105    /// Free-form notes / non-fatal extraction warnings.
106    #[serde(default)]
107    pub notes: Vec<String>,
108}
109
110/// The root manifest of an `.hmc` container.
111#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct HmcManifest {
113    /// Always `"hmc"`.
114    pub format: String,
115    pub format_version: u32,
116    /// Document identity = `blake3` of the source bytes (hex).
117    pub doc_id: String,
118    pub source: SourceInfo,
119    /// RFC3339 creation timestamp.
120    pub created: String,
121    #[serde(default)]
122    pub pipeline: PipelineInfo,
123    #[serde(default)]
124    pub status: StatusFlags,
125    /// Topical tags (human or analysis assigned). Used by reorganise/route.
126    #[serde(default)]
127    pub tags: Vec<String>,
128    /// All stored members (excluding the manifest itself).
129    #[serde(default)]
130    pub assets: Vec<AssetEntry>,
131}
132
133impl HmcManifest {
134    /// Creates a new instance.
135    pub fn new(source: SourceInfo) -> Self {
136        let doc_id = source.blake3.clone();
137        HmcManifest {
138            format: "hmc".to_string(),
139            format_version: HMC_FORMAT_VERSION,
140            doc_id,
141            source,
142            created: chrono::Utc::now().to_rfc3339(),
143            pipeline: PipelineInfo::default(),
144            status: StatusFlags::default(),
145            tags: Vec::new(),
146            assets: Vec::new(),
147        }
148    }
149
150    /// First asset of a given kind, if present.
151    pub fn asset_of(&self, kind: AssetKind) -> Option<&AssetEntry> {
152        self.assets.iter().find(|a| a.kind == kind)
153    }
154}