qualia_core_db/bundle/format.rs
1//! On-disk layout, index record, constants, and errors for the `.hmc` bundle.
2//!
3//! A `.hmc` bundle is a **transparent container of files**. It concatenates
4//! each embedded file *byte-for-byte*, page-aligned, at an absolute offset
5//! recorded in a CBOR index, and **never touches the interior of an entry**.
6//! That transparency is the whole point: a consumer can `mmap` the bundle and
7//! hand an entry's `[offset .. offset+length]` slice straight to the existing
8//! `.q42` / `.10d` / `.p64` reader, and that reader's *interior* segment offsets
9//! resolve unchanged — because the slice is a bit-identical standalone file.
10//! Nothing is compressed, re-chunked, or reframed. HTTP range-fetching one
11//! interior segment of one embedded file therefore works directly, too
12//! (`entry.offset + seg.offset`, `seg.len`).
13//!
14//! ```text
15//! offset size field
16//! 0 4 magic = "QBDL"
17//! 4 2 version (u16 LE)
18//! 6 2 flags (u16 LE, reserved = 0)
19//! 8 4 entry_count (u32 LE)
20//! 12 8 index_offset (u64 LE) — absolute offset of the CBOR index
21//! 20 8 index_length (u64 LE)
22//! 28 8 total_length (u64 LE) — == file length
23//! 36 4 crc32c (u32 LE) — whole-file CRC-32C, computed with
24//! these 4 bytes treated as zero
25//! 40 24 reserved (zero)
26//! 64 … page-aligned intact file payloads …
27//! … … CBOR index footer: Vec<BundleEntry>
28//! ```
29//!
30//! The index lives at the end (referenced from the fixed front header) so entry
31//! offsets are known before the index is written, and so a consumer can fetch
32//! just the header + index to enumerate the bundle without downloading payloads.
33
34use serde::{Deserialize, Serialize};
35
36/// Magic bytes at the start of every `.hmc` (hyper-media-container) bundle:
37/// `QBDL`. The 4-byte on-disk format tag is retained across the `.qualia`→`.hmc`
38/// rename (extension ≠ magic, as with most formats), so bundles already built
39/// keep parsing; only the file extension and human-facing name changed.
40pub const BUNDLE_MAGIC: [u8; 4] = *b"QBDL";
41
42/// Current bundle format version.
43pub const BUNDLE_VERSION: u16 = 1;
44
45/// Fixed header size in bytes. Payloads begin at (or after) this offset.
46pub const BUNDLE_HEADER_SIZE: usize = 64;
47
48/// Alignment (bytes) applied to every entry payload **and** the index footer.
49/// Page alignment (64) preserves the interior page-alignment of embedded
50/// `.q42` / `.p64` / `.10d` files, so their own segment offsets stay aligned and
51/// `mmap` / zero-copy segment access into an entry is unaffected. This matches
52/// the `Page` alignment tier of the `container_10d` writer and `p64_weight`.
53pub const BUNDLE_ENTRY_ALIGN: usize = 64;
54
55// --- header field byte offsets ---
56pub(crate) const OFF_MAGIC: usize = 0;
57pub(crate) const OFF_VERSION: usize = 4;
58pub(crate) const OFF_FLAGS: usize = 6;
59pub(crate) const OFF_ENTRY_COUNT: usize = 8;
60pub(crate) const OFF_INDEX_OFFSET: usize = 12;
61pub(crate) const OFF_INDEX_LENGTH: usize = 20;
62pub(crate) const OFF_TOTAL_LENGTH: usize = 28;
63pub(crate) const OFF_CRC32C: usize = 36;
64// [40 .. 64) reserved, must be zero.
65
66/// Length of a SHA-256 digest in bytes.
67pub const SHA256_LEN: usize = 32;
68
69/// One row of the bundle index (the CBOR footer). Describes one intact embedded
70/// file: its logical key, its format `kind`, where it lives (absolute, aligned),
71/// its length, a SHA-256 of the exact bytes, and opaque per-entry `meta` (a
72/// nested CBOR blob whose schema is agreed by the producer/consumer of a given
73/// bundle `kind` — e.g. an anatomy pack stores each organ's body-system,
74/// position and default colour there). The bundle format itself stays domain-
75/// agnostic: it moves intact files; `meta` carries any domain semantics.
76#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
77pub struct BundleEntry {
78 /// Logical key, unique within the bundle (e.g. `"3d-vh-f-kidney-l.glb.10d"`).
79 pub key: String,
80 /// The embedded file's format, e.g. `"10d"`, `"q42"`, `"p64"`, `"manifest"`.
81 pub kind: String,
82 /// Absolute byte offset from the start of the bundle. `BUNDLE_ENTRY_ALIGN`-aligned.
83 pub offset: u64,
84 /// Length of the intact embedded file in bytes.
85 pub length: u64,
86 /// SHA-256 of exactly `[offset .. offset+length]` — per-entry integrity,
87 /// identical to hashing the standalone file.
88 pub sha256: Vec<u8>,
89 /// Opaque per-entry CBOR metadata (domain-specific), or `None`.
90 #[serde(default, skip_serializing_if = "Option::is_none")]
91 pub meta: Option<Vec<u8>>,
92}
93
94/// Round `n` up to the next multiple of `align` (a power of two).
95#[inline]
96pub(crate) fn align_up(n: usize, align: usize) -> usize {
97 debug_assert!(align.is_power_of_two());
98 (n + align - 1) & !(align - 1)
99}
100
101/// An error building or reading a `.hmc` bundle.
102#[derive(Debug)]
103pub enum BundleError {
104 /// A key was empty.
105 EmptyKey,
106 /// Two entries shared a key.
107 DuplicateKey(String),
108 /// The byte slice is shorter than the fixed header.
109 TooShort,
110 /// The magic bytes were not `QBDL`.
111 BadMagic,
112 /// The version is not one this build understands.
113 UnsupportedVersion(u16),
114 /// The header's `total_length` did not match the actual byte length.
115 LengthMismatch { header: u64, actual: usize },
116 /// The whole-file CRC did not match (corruption or tampering).
117 CrcMismatch { expected: u32, got: u32 },
118 /// The index offset/length pointed outside the file.
119 BadIndexPointer {
120 offset: u64,
121 length: u64,
122 total: usize,
123 },
124 /// An entry's `[offset, length)` fell outside the payload region.
125 EntryOutOfBounds {
126 key: String,
127 offset: u64,
128 length: u64,
129 },
130 /// The CBOR index could not be encoded/decoded.
131 Cbor(String),
132 /// An I/O error (mmap/open), native only.
133 Io(String),
134}
135
136impl std::fmt::Display for BundleError {
137 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
138 match self {
139 BundleError::EmptyKey => write!(f, "bundle: empty entry key"),
140 BundleError::DuplicateKey(k) => write!(f, "bundle: duplicate entry key {k:?}"),
141 BundleError::TooShort => write!(f, "bundle: input shorter than header"),
142 BundleError::BadMagic => write!(f, "bundle: bad magic (not a .hmc bundle)"),
143 BundleError::UnsupportedVersion(v) => write!(f, "bundle: unsupported version {v}"),
144 BundleError::LengthMismatch { header, actual } => {
145 write!(
146 f,
147 "bundle: length mismatch (header {header}, actual {actual})"
148 )
149 }
150 BundleError::CrcMismatch { expected, got } => {
151 write!(
152 f,
153 "bundle: CRC mismatch (expected {expected:#010x}, got {got:#010x})"
154 )
155 }
156 BundleError::BadIndexPointer {
157 offset,
158 length,
159 total,
160 } => {
161 write!(
162 f,
163 "bundle: bad index pointer (offset {offset}, length {length}, total {total})"
164 )
165 }
166 BundleError::EntryOutOfBounds {
167 key,
168 offset,
169 length,
170 } => {
171 write!(
172 f,
173 "bundle: entry {key:?} out of bounds (offset {offset}, length {length})"
174 )
175 }
176 BundleError::Cbor(e) => write!(f, "bundle: CBOR error: {e}"),
177 BundleError::Io(e) => write!(f, "bundle: I/O error: {e}"),
178 }
179 }
180}
181
182impl std::error::Error for BundleError {}