qualia_core_db/bundle/
reader.rs

1//! Read a `.hmc` bundle: verify integrity, enumerate entries, and hand back
2//! **zero-copy slices** of intact embedded files (and of interior segments).
3
4use crate::container_10d::crc32c::crc32c_update;
5
6use super::format::{
7    BundleEntry, BundleError, BUNDLE_HEADER_SIZE, BUNDLE_MAGIC, BUNDLE_VERSION, OFF_CRC32C,
8    OFF_FLAGS, OFF_INDEX_LENGTH, OFF_INDEX_OFFSET, OFF_MAGIC, OFF_TOTAL_LENGTH, OFF_VERSION,
9};
10
11#[inline]
12fn read_u32(b: &[u8], off: usize) -> u32 {
13    u32::from_le_bytes([b[off], b[off + 1], b[off + 2], b[off + 3]])
14}
15
16#[inline]
17fn read_u64(b: &[u8], off: usize) -> u64 {
18    let mut a = [0u8; 8];
19    a.copy_from_slice(&b[off..off + 8]);
20    u64::from_le_bytes(a)
21}
22
23/// Whole-file CRC-32C computed with the 4 CRC-field bytes treated as zero — no
24/// copy of the payload (streams the three ranges through the incremental CRC),
25/// so it is cheap even over a memory-mapped multi-hundred-MB bundle.
26fn whole_file_crc(bytes: &[u8]) -> u32 {
27    let mut crc = 0xFFFF_FFFFu32;
28    crc = crc32c_update(crc, &bytes[..OFF_CRC32C]);
29    crc = crc32c_update(crc, &[0u8; 4]);
30    crc = crc32c_update(crc, &bytes[OFF_CRC32C + 4..]);
31    !crc
32}
33
34/// A parsed, integrity-checked view over a `.hmc` bundle's bytes. Borrows the
35/// input; `get`/`segment` return slices into it (zero copy).
36pub struct BundleReader<'a> {
37    bytes: &'a [u8],
38    entries: Vec<BundleEntry>,
39    flags: u16,
40}
41
42impl<'a> BundleReader<'a> {
43    /// Parse and fully validate a bundle: magic, version, declared length,
44    /// whole-file CRC, and every entry's bounds. On success the payloads are
45    /// known-good and in-bounds, so `get`/`segment` are total on valid keys.
46    pub fn parse(bytes: &'a [u8]) -> Result<Self, BundleError> {
47        if bytes.len() < BUNDLE_HEADER_SIZE {
48            return Err(BundleError::TooShort);
49        }
50        if bytes[OFF_MAGIC..OFF_MAGIC + 4] != BUNDLE_MAGIC {
51            return Err(BundleError::BadMagic);
52        }
53        let version = u16::from_le_bytes([bytes[OFF_VERSION], bytes[OFF_VERSION + 1]]);
54        if version != BUNDLE_VERSION {
55            return Err(BundleError::UnsupportedVersion(version));
56        }
57        let flags = u16::from_le_bytes([bytes[OFF_FLAGS], bytes[OFF_FLAGS + 1]]);
58
59        let total_length = read_u64(bytes, OFF_TOTAL_LENGTH);
60        if total_length as usize != bytes.len() {
61            return Err(BundleError::LengthMismatch {
62                header: total_length,
63                actual: bytes.len(),
64            });
65        }
66
67        // Integrity before trusting any offset.
68        let stored_crc = read_u32(bytes, OFF_CRC32C);
69        let got = whole_file_crc(bytes);
70        if got != stored_crc {
71            return Err(BundleError::CrcMismatch {
72                expected: stored_crc,
73                got,
74            });
75        }
76
77        let index_offset = read_u64(bytes, OFF_INDEX_OFFSET);
78        let index_length = read_u64(bytes, OFF_INDEX_LENGTH);
79        let end = index_offset
80            .checked_add(index_length)
81            .ok_or(BundleError::BadIndexPointer {
82                offset: index_offset,
83                length: index_length,
84                total: bytes.len(),
85            })?;
86        if index_offset < BUNDLE_HEADER_SIZE as u64 || end as usize > bytes.len() {
87            return Err(BundleError::BadIndexPointer {
88                offset: index_offset,
89                length: index_length,
90                total: bytes.len(),
91            });
92        }
93
94        let index_slice = &bytes[index_offset as usize..end as usize];
95        let entries: Vec<BundleEntry> =
96            ciborium::from_reader(index_slice).map_err(|e| BundleError::Cbor(e.to_string()))?;
97
98        // Every entry must lie within the payload region (before the index).
99        for e in &entries {
100            let e_end =
101                e.offset
102                    .checked_add(e.length)
103                    .ok_or_else(|| BundleError::EntryOutOfBounds {
104                        key: e.key.clone(),
105                        offset: e.offset,
106                        length: e.length,
107                    })?;
108            if e.offset < BUNDLE_HEADER_SIZE as u64 || e_end > index_offset {
109                return Err(BundleError::EntryOutOfBounds {
110                    key: e.key.clone(),
111                    offset: e.offset,
112                    length: e.length,
113                });
114            }
115        }
116
117        Ok(Self {
118            bytes,
119            entries,
120            flags,
121        })
122    }
123
124    /// The bundle's entries (index order).
125    pub fn entries(&self) -> &[BundleEntry] {
126        &self.entries
127    }
128
129    /// Reserved format flags from the header (0 in v1; reserved for future use
130    /// such as a signed/compressed-entry variant).
131    pub fn flags(&self) -> u16 {
132        self.flags
133    }
134
135    /// The index record for `key`, if present.
136    pub fn entry(&self, key: &str) -> Option<&BundleEntry> {
137        self.entries.iter().find(|e| e.key == key)
138    }
139
140    /// A zero-copy slice of the intact embedded file for `key`. This slice **is**
141    /// a byte-identical standalone `.q42` / `.10d` / `.p64` — hand it straight to
142    /// that format's reader.
143    pub fn get(&self, key: &str) -> Option<&'a [u8]> {
144        let e = self.entry(key)?;
145        Some(&self.bytes[e.offset as usize..(e.offset + e.length) as usize])
146    }
147
148    /// A zero-copy slice of an **interior segment** of an entry
149    /// (`[seg_offset .. seg_offset+seg_len)` within the embedded file). Returns
150    /// `None` if the key is unknown or the segment runs past the entry — proving
151    /// the bundle does not interfere with segment-level access into an entry.
152    pub fn segment(&self, key: &str, seg_offset: u64, seg_len: u64) -> Option<&'a [u8]> {
153        let e = self.entry(key)?;
154        let seg_end = seg_offset.checked_add(seg_len)?;
155        if seg_end > e.length {
156            return None;
157        }
158        let start = (e.offset + seg_offset) as usize;
159        Some(&self.bytes[start..start + seg_len as usize])
160    }
161
162    /// Verify one entry's payload against its recorded SHA-256.
163    pub fn verify_entry(&self, key: &str) -> bool {
164        use sha2::{Digest, Sha256};
165        let Some(e) = self.entry(key) else {
166            return false;
167        };
168        let Some(payload) = self.get(key) else {
169            return false;
170        };
171        let mut hasher = Sha256::new();
172        hasher.update(payload);
173        hasher.finalize().as_slice() == e.sha256.as_slice()
174    }
175}
176
177/// Open a `.hmc` bundle from disk via `mmap` for zero-copy access (native).
178///
179/// The mapping backs the slices returned by [`BundleReader`], so an embedded
180/// file (or one of its interior segments) can be read without copying the
181/// bundle into the heap — the intended path for shipping a large asset pack.
182#[cfg(not(target_arch = "wasm32"))]
183pub struct BundleMmap {
184    map: memmap2::Mmap,
185}
186
187#[cfg(not(target_arch = "wasm32"))]
188impl BundleMmap {
189    /// Memory-map a bundle file.
190    pub fn open(path: impl AsRef<std::path::Path>) -> Result<Self, BundleError> {
191        let file = std::fs::File::open(path).map_err(|e| BundleError::Io(e.to_string()))?;
192        // SAFETY: the map is read-only and lives as long as `self`; callers get
193        // slices bounded by `self`.
194        let map =
195            unsafe { memmap2::Mmap::map(&file) }.map_err(|e| BundleError::Io(e.to_string()))?;
196        Ok(Self { map })
197    }
198
199    /// The raw mapped bytes.
200    pub fn as_bytes(&self) -> &[u8] {
201        &self.map
202    }
203
204    /// A validated reader borrowing the mapping.
205    pub fn reader(&self) -> Result<BundleReader<'_>, BundleError> {
206        BundleReader::parse(&self.map)
207    }
208}
209
210#[cfg(test)]
211mod tests {
212    use super::super::writer::BundleWriter;
213    use super::*;
214
215    fn sample_bundle() -> Vec<u8> {
216        let mut w = BundleWriter::new();
217        w.add_file(
218            "liver.10d",
219            "10d",
220            (0..200u32).map(|i| i as u8).collect(),
221            Some(vec![0xAA, 0xBB]),
222        )
223        .unwrap();
224        w.add_file(
225            "graph.q42",
226            "q42",
227            b"hello q42 segment world".to_vec(),
228            None,
229        )
230        .unwrap();
231        w.build().unwrap()
232    }
233
234    #[test]
235    fn roundtrip_is_byte_identical() {
236        let liver: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
237        let bytes = sample_bundle();
238        let r = BundleReader::parse(&bytes).unwrap();
239        assert_eq!(r.entries().len(), 2);
240        // The embedded file comes back byte-for-byte — the transparency guarantee.
241        assert_eq!(r.get("liver.10d").unwrap(), liver.as_slice());
242        assert_eq!(r.get("graph.q42").unwrap(), b"hello q42 segment world");
243        assert_eq!(r.entry("liver.10d").unwrap().kind, "10d");
244        assert_eq!(
245            r.entry("liver.10d").unwrap().meta.as_deref(),
246            Some(&[0xAA, 0xBB][..])
247        );
248        assert!(r.get("missing").is_none());
249    }
250
251    #[test]
252    fn interior_segment_access_is_not_interfered_with() {
253        let bytes = sample_bundle();
254        let r = BundleReader::parse(&bytes).unwrap();
255        // Pull the interior segment "q42 segment" (offset 6, len 11) directly.
256        assert_eq!(r.segment("graph.q42", 6, 11).unwrap(), b"q42 segment");
257        // A segment past the end is refused, not read out of the entry.
258        assert!(r.segment("graph.q42", 20, 100).is_none());
259    }
260
261    #[test]
262    fn per_entry_sha256_verifies_and_detects_tamper() {
263        let mut bytes = sample_bundle();
264        let r = BundleReader::parse(&bytes).unwrap();
265        assert!(r.verify_entry("liver.10d"));
266        assert!(r.verify_entry("graph.q42"));
267        drop(r);
268        // Flip a payload byte inside an entry: whole-file CRC now fails on parse.
269        let off = {
270            let r = BundleReader::parse(&bytes).unwrap();
271            r.entry("liver.10d").unwrap().offset as usize
272        };
273        bytes[off] ^= 0xFF;
274        assert!(matches!(
275            BundleReader::parse(&bytes),
276            Err(BundleError::CrcMismatch { .. })
277        ));
278    }
279
280    #[test]
281    fn rejects_bad_magic_and_short_input() {
282        assert!(matches!(
283            BundleReader::parse(&[0u8; 10]),
284            Err(BundleError::TooShort)
285        ));
286        let mut bytes = sample_bundle();
287        bytes[0] = b'X';
288        assert!(matches!(
289            BundleReader::parse(&bytes),
290            Err(BundleError::BadMagic)
291        ));
292    }
293
294    #[cfg(not(target_arch = "wasm32"))]
295    #[test]
296    fn mmap_roundtrip() {
297        let bytes = sample_bundle();
298        let dir = tempfile::tempdir().unwrap();
299        let path = dir.path().join("pack.hmc");
300        std::fs::write(&path, &bytes).unwrap();
301        let m = BundleMmap::open(&path).unwrap();
302        let r = m.reader().unwrap();
303        assert_eq!(r.get("graph.q42").unwrap(), b"hello q42 segment world");
304        assert!(r.verify_entry("liver.10d"));
305    }
306}
qualia_core_db/bundle/reader.rs

qualia_core_db/bundle/
reader.rs
