qualia_semantic_library/container/manifest.rs
1//! The `manifest.json` that sits at the root of every `.hmc` container.
2//!
3//! The manifest is the self-describing index of a hypermedia container: it names
4//! the original source, every derived asset, content hashes, and the pipeline
5//! that produced them. A reader can understand a container fully from the
6//! manifest alone, without unpacking the heavy assets.
7
8use serde::{Deserialize, Serialize};
9
10/// Current on-disk format version of the `.hmc` container.
11pub const HMC_FORMAT_VERSION: u32 = 1;
12
13/// Logical role of an asset inside a container. The path in [`AssetEntry`] is
14/// authoritative; `kind` is the typed lens used for retrieval/UX.
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(rename_all = "snake_case")]
17pub enum AssetKind {
18 /// The original, verbatim source file (e.g. the PDF). Never modified.
19 Source,
20 /// Canonical structured HTML (headings, MathML, provenance `data-*` attrs).
21 Html,
22 /// Plain text with `[[page N]]` markers (compat + cheap full-text).
23 Text,
24 /// CML / RDF (Turtle) semantic annotation layer over the HTML.
25 Cml,
26 /// Structural chunks as JSON-Lines (one object per line).
27 Chunks,
28 /// Binary embedding matrix for the chunks (`f32`, row-major).
29 Embeddings,
30 /// Extraction/analysis metadata (tool versions, timings, confidence).
31 Meta,
32 /// Anything else (figures, tables, page images…).
33 Other,
34}
35
36impl AssetKind {
37 /// Conventional directory prefix for this kind inside the zip.
38 pub fn dir(self) -> &'static str {
39 match self {
40 AssetKind::Source => "source",
41 AssetKind::Html | AssetKind::Text | AssetKind::Cml | AssetKind::Chunks => "derived",
42 AssetKind::Embeddings => "embeddings",
43 AssetKind::Meta => "meta",
44 AssetKind::Other => "assets",
45 }
46 }
47}
48
49/// One stored member of the container.
50#[derive(Debug, Clone, Serialize, Deserialize)]
51pub struct AssetEntry {
52 /// Path of the member inside the zip (e.g. `derived/document.html`).
53 pub path: String,
54 /// Logical role.
55 pub kind: AssetKind,
56 /// MIME type, best-effort (e.g. `text/html`, `application/pdf`).
57 pub mime: String,
58 /// BLAKE3 hex of the asset bytes (integrity + dedup).
59 pub blake3: String,
60 /// Uncompressed size in bytes.
61 pub bytes: u64,
62}
63
64/// Description of the original source document.
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct SourceInfo {
67 /// Original filename as found on disk.
68 pub filename: String,
69 pub mime: String,
70 pub size_bytes: u64,
71 /// BLAKE3 hex of the original bytes — the document identity.
72 pub blake3: String,
73 /// Title (from document metadata, else derived from filename).
74 #[serde(default)]
75 pub title: String,
76 /// Page count when known (PDFs); 0 otherwise.
77 #[serde(default)]
78 pub page_count: u32,
79}
80
81/// Which tools produced this container — for reproducibility and re-runs.
82#[derive(Debug, Clone, Default, Serialize, Deserialize)]
83pub struct PipelineInfo {
84 /// Producing tool + version (e.g. `qualia-semantic-library 0.0.20`).
85 pub tool: String,
86 /// Text/structure extractor used (e.g. `pdf-extract`, `pymupdf`, `nougat`).
87 #[serde(default)]
88 pub extractor: String,
89 /// Embedding model id, if embeddings were computed.
90 #[serde(default)]
91 pub embedder: String,
92 /// Embedding dimensionality (0 if none).
93 #[serde(default)]
94 pub embed_dim: u32,
95}
96
97/// Coarse processing status carried in the manifest so a library scan can route
98/// work (what still needs extraction / embedding / analysis) without unpacking.
99#[derive(Debug, Clone, Default, Serialize, Deserialize)]
100pub struct StatusFlags {
101 pub extracted: bool,
102 pub chunked: bool,
103 pub embedded: bool,
104 pub analyzed: bool,
105 /// Free-form notes / non-fatal extraction warnings.
106 #[serde(default)]
107 pub notes: Vec<String>,
108}
109
110/// The root manifest of an `.hmc` container.
111#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct HmcManifest {
113 /// Always `"hmc"`.
114 pub format: String,
115 pub format_version: u32,
116 /// Document identity = `blake3` of the source bytes (hex).
117 pub doc_id: String,
118 pub source: SourceInfo,
119 /// RFC3339 creation timestamp.
120 pub created: String,
121 #[serde(default)]
122 pub pipeline: PipelineInfo,
123 #[serde(default)]
124 pub status: StatusFlags,
125 /// Topical tags (human or analysis assigned). Used by reorganise/route.
126 #[serde(default)]
127 pub tags: Vec<String>,
128 /// All stored members (excluding the manifest itself).
129 #[serde(default)]
130 pub assets: Vec<AssetEntry>,
131}
132
133impl HmcManifest {
134 /// Creates a new instance.
135 pub fn new(source: SourceInfo) -> Self {
136 let doc_id = source.blake3.clone();
137 HmcManifest {
138 format: "hmc".to_string(),
139 format_version: HMC_FORMAT_VERSION,
140 doc_id,
141 source,
142 created: chrono::Utc::now().to_rfc3339(),
143 pipeline: PipelineInfo::default(),
144 status: StatusFlags::default(),
145 tags: Vec::new(),
146 assets: Vec::new(),
147 }
148 }
149
150 /// First asset of a given kind, if present.
151 pub fn asset_of(&self, kind: AssetKind) -> Option<&AssetEntry> {
152 self.assets.iter().find(|a| a.kind == kind)
153 }
154}