qualia_core_db/bundle/format.rs
1//! On-disk layout, index record, constants, and errors for the `.hmc` bundle.
2//!
3//! A `.hmc` bundle is a **transparent container of files**. It concatenates
4//! each embedded file *byte-for-byte*, page-aligned, at an absolute offset
5//! recorded in a CBOR index, and **never touches the interior of an entry**.
6//! That transparency is the whole point: a consumer can `mmap` the bundle and
7//! hand an entry's `[offset .. offset+length]` slice straight to the existing
8//! `.q42` / `.10d` / `.p64` reader, and that reader's *interior* segment offsets
9//! resolve unchanged — because the slice is a bit-identical standalone file.
10//! Nothing is compressed, re-chunked, or reframed. HTTP range-fetching one
11//! interior segment of one embedded file therefore works directly, too
12//! (`entry.offset + seg.offset`, `seg.len`).
13//!
14//! ```text
15//! offset size field
16//! 0 4 magic = "QBDL"
17//! 4 2 version (u16 LE)
18//! 6 2 flags (u16 LE, reserved = 0)
19//! 8 4 entry_count (u32 LE)
20//! 12 8 index_offset (u64 LE) — absolute offset of the CBOR index
21//! 20 8 index_length (u64 LE)
22//! 28 8 total_length (u64 LE) — == file length
23//! 36 4 crc32c (u32 LE) — whole-file CRC-32C, computed with
24//! these 4 bytes treated as zero
25//! 40 24 reserved (zero)
26//! 64 … page-aligned intact file payloads …
27//! … … CBOR index footer: Vec<BundleEntry>
28//! ```
29//!
30//! The index lives at the end (referenced from the fixed front header) so entry
31//! offsets are known before the index is written, and so a consumer can fetch
32//! just the header + index to enumerate the bundle without downloading payloads.
33
34use serde::{Deserialize, Serialize};
35
36/// Magic bytes at the start of every `.hmc` (hyper-media-container) bundle:
37/// `QBDL`. The 4-byte on-disk format tag is retained across the `.qualia`→`.hmc`
38/// rename (extension ≠ magic, as with most formats), so bundles already built
39/// keep parsing; only the file extension and human-facing name changed.
40pub const BUNDLE_MAGIC: [u8; 4] = *b"QBDL";
41
42/// Current bundle format version.
43pub const BUNDLE_VERSION: u16 = 1;
44
45/// Fixed header size in bytes. Payloads begin at (or after) this offset.
46pub const BUNDLE_HEADER_SIZE: usize = 64;
47
48/// Alignment (bytes) applied to every entry payload **and** the index footer.
49/// Page alignment (64) preserves the interior page-alignment of embedded
50/// `.q42` / `.p64` / `.10d` files, so their own segment offsets stay aligned and
51/// `mmap` / zero-copy segment access into an entry is unaffected. This matches
52/// the `Page` alignment tier of the `container_10d` writer and `p64_weight`.
53pub const BUNDLE_ENTRY_ALIGN: usize = 64;
54
55/// OS page alignment (4096 bytes) for zero-copy direct mmap paging of embedded assets.
56pub const BUNDLE_PAGE_ALIGN_4096: usize = 4096;
57
58/// Header flag indicating every entry and index footer is aligned to 4096 bytes.
59pub const BUNDLE_FLAG_PAGE_ALIGN_4096: u16 = 0x0001;
60
61// --- header field byte offsets ---
62pub const OFF_MAGIC: usize = 0;
63pub const OFF_VERSION: usize = 4;
64pub const OFF_FLAGS: usize = 6;
65pub const OFF_ENTRY_COUNT: usize = 8;
66pub const OFF_INDEX_OFFSET: usize = 12;
67pub const OFF_INDEX_LENGTH: usize = 20;
68pub const OFF_TOTAL_LENGTH: usize = 28;
69pub const OFF_CRC32C: usize = 36;
70// [40 .. 64) reserved, must be zero.
71
72/// Length of a SHA-256 digest in bytes.
73pub const SHA256_LEN: usize = 32;
74
75/// One row of the bundle index (the CBOR footer). Describes one intact embedded
76/// file: its logical key, its format `kind`, where it lives (absolute, aligned),
77/// its length, a SHA-256 of the exact bytes, and opaque per-entry `meta` (a
78/// nested CBOR blob whose schema is agreed by the producer/consumer of a given
79/// bundle `kind` — e.g. an anatomy pack stores each organ's body-system,
80/// position and default colour there). The bundle format itself stays domain-
81/// agnostic: it moves intact files; `meta` carries any domain semantics.
82#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
83pub struct BundleEntry {
84 /// Logical key, unique within the bundle (e.g. `"3d-vh-f-kidney-l.glb.10d"`).
85 pub key: String,
86 /// The embedded file's format, e.g. `"10d"`, `"q42"`, `"p64"`, `"manifest"`.
87 pub kind: String,
88 /// Absolute byte offset from the start of the bundle. `BUNDLE_ENTRY_ALIGN`-aligned.
89 pub offset: u64,
90 /// Length of the intact embedded file in bytes.
91 pub length: u64,
92 /// SHA-256 of exactly `[offset .. offset+length]` — per-entry integrity,
93 /// identical to hashing the standalone file.
94 pub sha256: Vec<u8>,
95 /// Opaque per-entry CBOR metadata (domain-specific), or `None`.
96 #[serde(default, skip_serializing_if = "Option::is_none")]
97 pub meta: Option<Vec<u8>>,
98}
99
100/// Round `n` up to the next multiple of `align` (a power of two).
101#[inline]
102pub fn align_up(n: usize, align: usize) -> usize {
103 debug_assert!(align.is_power_of_two());
104 (n + align - 1) & !(align - 1)
105}
106
107/// An error building or reading a `.hmc` bundle.
108#[derive(Debug)]
109pub enum BundleError {
110 /// A key was empty.
111 EmptyKey,
112 /// Two entries shared a key.
113 DuplicateKey(String),
114 /// The byte slice is shorter than the fixed header.
115 TooShort,
116 /// The magic bytes were not `QBDL`.
117 BadMagic,
118 /// The version is not one this build understands.
119 UnsupportedVersion(u16),
120 /// The header's `total_length` did not match the actual byte length.
121 LengthMismatch { header: u64, actual: usize },
122 /// The whole-file CRC did not match (corruption or tampering).
123 CrcMismatch { expected: u32, got: u32 },
124 /// The index offset/length pointed outside the file.
125 BadIndexPointer {
126 offset: u64,
127 length: u64,
128 total: usize,
129 },
130 /// An entry's `[offset, length)` fell outside the payload region.
131 EntryOutOfBounds {
132 key: String,
133 offset: u64,
134 length: u64,
135 },
136 /// An entry was not aligned to the expected byte boundary.
137 UnpageAligned {
138 key: String,
139 offset: u64,
140 alignment: usize,
141 },
142 /// The CBOR index could not be encoded/decoded.
143 Cbor(String),
144 /// An I/O error (mmap/open), native only.
145 Io(String),
146}
147
148impl std::fmt::Display for BundleError {
149 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
150 match self {
151 BundleError::EmptyKey => write!(f, "bundle: empty entry key"),
152 BundleError::DuplicateKey(k) => write!(f, "bundle: duplicate entry key {k:?}"),
153 BundleError::TooShort => write!(f, "bundle: input shorter than header"),
154 BundleError::BadMagic => write!(f, "bundle: bad magic (not a .hmc bundle)"),
155 BundleError::UnsupportedVersion(v) => write!(f, "bundle: unsupported version {v}"),
156 BundleError::LengthMismatch { header, actual } => {
157 write!(
158 f,
159 "bundle: length mismatch (header {header}, actual {actual})"
160 )
161 }
162 BundleError::CrcMismatch { expected, got } => {
163 write!(
164 f,
165 "bundle: CRC mismatch (expected {expected:#010x}, got {got:#010x})"
166 )
167 }
168 BundleError::BadIndexPointer {
169 offset,
170 length,
171 total,
172 } => {
173 write!(
174 f,
175 "bundle: bad index pointer (offset {offset}, length {length}, total {total})"
176 )
177 }
178 BundleError::EntryOutOfBounds {
179 key,
180 offset,
181 length,
182 } => {
183 write!(
184 f,
185 "bundle: entry {key:?} out of bounds (offset {offset}, length {length})"
186 )
187 }
188 BundleError::UnpageAligned {
189 key,
190 offset,
191 alignment,
192 } => {
193 write!(
194 f,
195 "bundle: entry {key:?} at offset {offset} is not aligned to {alignment} bytes"
196 )
197 }
198 BundleError::Cbor(e) => write!(f, "bundle: CBOR error: {e}"),
199 BundleError::Io(e) => write!(f, "bundle: I/O error: {e}"),
200 }
201 }
202}
203
204impl std::error::Error for BundleError {}