Skip to main content

qualia_core_db/bundle/
format.rs

1//! On-disk layout, index record, constants, and errors for the `.hmc` bundle.
2//!
3//! A `.hmc` bundle is a **transparent container of files**. It concatenates
4//! each embedded file *byte-for-byte*, page-aligned, at an absolute offset
5//! recorded in a CBOR index, and **never touches the interior of an entry**.
6//! That transparency is the whole point: a consumer can `mmap` the bundle and
7//! hand an entry's `[offset .. offset+length]` slice straight to the existing
8//! `.q42` / `.10d` / `.p64` reader, and that reader's *interior* segment offsets
9//! resolve unchanged — because the slice is a bit-identical standalone file.
10//! Nothing is compressed, re-chunked, or reframed. HTTP range-fetching one
11//! interior segment of one embedded file therefore works directly, too
12//! (`entry.offset + seg.offset`, `seg.len`).
13//!
14//! ```text
15//! offset  size  field
16//! 0       4     magic = "QBDL"
17//! 4       2     version (u16 LE)
18//! 6       2     flags   (u16 LE, reserved = 0)
19//! 8       4     entry_count (u32 LE)
20//! 12      8     index_offset (u64 LE)   — absolute offset of the CBOR index
21//! 20      8     index_length (u64 LE)
22//! 28      8     total_length (u64 LE)   — == file length
23//! 36      4     crc32c (u32 LE)         — whole-file CRC-32C, computed with
24//!                                          these 4 bytes treated as zero
25//! 40      24    reserved (zero)
26//! 64      …     page-aligned intact file payloads …
27//! …       …     CBOR index footer: Vec<BundleEntry>
28//! ```
29//!
30//! The index lives at the end (referenced from the fixed front header) so entry
31//! offsets are known before the index is written, and so a consumer can fetch
32//! just the header + index to enumerate the bundle without downloading payloads.
33
34use serde::{Deserialize, Serialize};
35
36/// Magic bytes at the start of every `.hmc` (hyper-media-container) bundle:
37/// `QBDL`. The 4-byte on-disk format tag is retained across the `.qualia`→`.hmc`
38/// rename (extension ≠ magic, as with most formats), so bundles already built
39/// keep parsing; only the file extension and human-facing name changed.
40pub const BUNDLE_MAGIC: [u8; 4] = *b"QBDL";
41
42/// Current bundle format version.
43pub const BUNDLE_VERSION: u16 = 1;
44
45/// Fixed header size in bytes. Payloads begin at (or after) this offset.
46pub const BUNDLE_HEADER_SIZE: usize = 64;
47
48/// Alignment (bytes) applied to every entry payload **and** the index footer.
49/// Page alignment (64) preserves the interior page-alignment of embedded
50/// `.q42` / `.p64` / `.10d` files, so their own segment offsets stay aligned and
51/// `mmap` / zero-copy segment access into an entry is unaffected. This matches
52/// the `Page` alignment tier of the `container_10d` writer and `p64_weight`.
53pub const BUNDLE_ENTRY_ALIGN: usize = 64;
54
55/// OS page alignment (4096 bytes) for zero-copy direct mmap paging of embedded assets.
56pub const BUNDLE_PAGE_ALIGN_4096: usize = 4096;
57
58/// Header flag indicating every entry and index footer is aligned to 4096 bytes.
59pub const BUNDLE_FLAG_PAGE_ALIGN_4096: u16 = 0x0001;
60
61// --- header field byte offsets ---
62pub const OFF_MAGIC: usize = 0;
63pub const OFF_VERSION: usize = 4;
64pub const OFF_FLAGS: usize = 6;
65pub const OFF_ENTRY_COUNT: usize = 8;
66pub const OFF_INDEX_OFFSET: usize = 12;
67pub const OFF_INDEX_LENGTH: usize = 20;
68pub const OFF_TOTAL_LENGTH: usize = 28;
69pub const OFF_CRC32C: usize = 36;
70// [40 .. 64) reserved, must be zero.
71
72/// Length of a SHA-256 digest in bytes.
73pub const SHA256_LEN: usize = 32;
74
75/// One row of the bundle index (the CBOR footer). Describes one intact embedded
76/// file: its logical key, its format `kind`, where it lives (absolute, aligned),
77/// its length, a SHA-256 of the exact bytes, and opaque per-entry `meta` (a
78/// nested CBOR blob whose schema is agreed by the producer/consumer of a given
79/// bundle `kind` — e.g. an anatomy pack stores each organ's body-system,
80/// position and default colour there). The bundle format itself stays domain-
81/// agnostic: it moves intact files; `meta` carries any domain semantics.
82#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
83pub struct BundleEntry {
84    /// Logical key, unique within the bundle (e.g. `"3d-vh-f-kidney-l.glb.10d"`).
85    pub key: String,
86    /// The embedded file's format, e.g. `"10d"`, `"q42"`, `"p64"`, `"manifest"`.
87    pub kind: String,
88    /// Absolute byte offset from the start of the bundle. `BUNDLE_ENTRY_ALIGN`-aligned.
89    pub offset: u64,
90    /// Length of the intact embedded file in bytes.
91    pub length: u64,
92    /// SHA-256 of exactly `[offset .. offset+length]` — per-entry integrity,
93    /// identical to hashing the standalone file.
94    pub sha256: Vec<u8>,
95    /// Opaque per-entry CBOR metadata (domain-specific), or `None`.
96    #[serde(default, skip_serializing_if = "Option::is_none")]
97    pub meta: Option<Vec<u8>>,
98}
99
100/// Round `n` up to the next multiple of `align` (a power of two).
101#[inline]
102pub fn align_up(n: usize, align: usize) -> usize {
103    debug_assert!(align.is_power_of_two());
104    (n + align - 1) & !(align - 1)
105}
106
107/// An error building or reading a `.hmc` bundle.
108#[derive(Debug)]
109pub enum BundleError {
110    /// A key was empty.
111    EmptyKey,
112    /// Two entries shared a key.
113    DuplicateKey(String),
114    /// The byte slice is shorter than the fixed header.
115    TooShort,
116    /// The magic bytes were not `QBDL`.
117    BadMagic,
118    /// The version is not one this build understands.
119    UnsupportedVersion(u16),
120    /// The header's `total_length` did not match the actual byte length.
121    LengthMismatch { header: u64, actual: usize },
122    /// The whole-file CRC did not match (corruption or tampering).
123    CrcMismatch { expected: u32, got: u32 },
124    /// The index offset/length pointed outside the file.
125    BadIndexPointer {
126        offset: u64,
127        length: u64,
128        total: usize,
129    },
130    /// An entry's `[offset, length)` fell outside the payload region.
131    EntryOutOfBounds {
132        key: String,
133        offset: u64,
134        length: u64,
135    },
136    /// An entry was not aligned to the expected byte boundary.
137    UnpageAligned {
138        key: String,
139        offset: u64,
140        alignment: usize,
141    },
142    /// The CBOR index could not be encoded/decoded.
143    Cbor(String),
144    /// An I/O error (mmap/open), native only.
145    Io(String),
146}
147
148impl std::fmt::Display for BundleError {
149    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
150        match self {
151            BundleError::EmptyKey => write!(f, "bundle: empty entry key"),
152            BundleError::DuplicateKey(k) => write!(f, "bundle: duplicate entry key {k:?}"),
153            BundleError::TooShort => write!(f, "bundle: input shorter than header"),
154            BundleError::BadMagic => write!(f, "bundle: bad magic (not a .hmc bundle)"),
155            BundleError::UnsupportedVersion(v) => write!(f, "bundle: unsupported version {v}"),
156            BundleError::LengthMismatch { header, actual } => {
157                write!(
158                    f,
159                    "bundle: length mismatch (header {header}, actual {actual})"
160                )
161            }
162            BundleError::CrcMismatch { expected, got } => {
163                write!(
164                    f,
165                    "bundle: CRC mismatch (expected {expected:#010x}, got {got:#010x})"
166                )
167            }
168            BundleError::BadIndexPointer {
169                offset,
170                length,
171                total,
172            } => {
173                write!(
174                    f,
175                    "bundle: bad index pointer (offset {offset}, length {length}, total {total})"
176                )
177            }
178            BundleError::EntryOutOfBounds {
179                key,
180                offset,
181                length,
182            } => {
183                write!(
184                    f,
185                    "bundle: entry {key:?} out of bounds (offset {offset}, length {length})"
186                )
187            }
188            BundleError::UnpageAligned {
189                key,
190                offset,
191                alignment,
192            } => {
193                write!(
194                    f,
195                    "bundle: entry {key:?} at offset {offset} is not aligned to {alignment} bytes"
196                )
197            }
198            BundleError::Cbor(e) => write!(f, "bundle: CBOR error: {e}"),
199            BundleError::Io(e) => write!(f, "bundle: I/O error: {e}"),
200        }
201    }
202}
203
204impl std::error::Error for BundleError {}