Skip to main content

qualia_core_db/bundle/
reader.rs

1//! Read a `.hmc` bundle: verify integrity, enumerate entries, and hand back
2//! **zero-copy slices** of intact embedded files (and of interior segments).
3
4use crate::container_10d::crc32c::crc32c_update;
5
6#[path = "reader_limits.rs"]
7mod limits;
8pub use limits::BundleReadLimits;
9#[cfg(test)]
10#[path = "reader_limits_tests.rs"]
11mod limits_tests;
12
13use super::format::{
14    BUNDLE_FLAG_PAGE_ALIGN_4096, BUNDLE_HEADER_SIZE, BUNDLE_MAGIC, BUNDLE_PAGE_ALIGN_4096,
15    BUNDLE_VERSION, BundleEntry, BundleError, OFF_CRC32C, OFF_FLAGS, OFF_INDEX_LENGTH,
16    OFF_INDEX_OFFSET, OFF_MAGIC, OFF_TOTAL_LENGTH, OFF_VERSION,
17};
18
19#[inline]
20fn read_u32(b: &[u8], off: usize) -> u32 {
21    u32::from_le_bytes([b[off], b[off + 1], b[off + 2], b[off + 3]])
22}
23
24#[inline]
25fn read_u64(b: &[u8], off: usize) -> u64 {
26    let mut a = [0u8; 8];
27    a.copy_from_slice(&b[off..off + 8]);
28    u64::from_le_bytes(a)
29}
30
31/// Whole-file CRC-32C computed with the 4 CRC-field bytes treated as zero — no
32/// copy of the payload (streams the three ranges through the incremental CRC),
33/// so it is cheap even over a memory-mapped multi-hundred-MB bundle.
34fn whole_file_crc(bytes: &[u8]) -> u32 {
35    let mut crc = 0xFFFF_FFFFu32;
36    crc = crc32c_update(crc, &bytes[..OFF_CRC32C]);
37    crc = crc32c_update(crc, &[0u8; 4]);
38    crc = crc32c_update(crc, &bytes[OFF_CRC32C + 4..]);
39    !crc
40}
41
42/// A parsed, integrity-checked view over a `.hmc` bundle's bytes. Borrows the
43/// input; `get`/`segment` return slices into it (zero copy).
44pub struct BundleReader<'a> {
45    bytes: &'a [u8],
46    entries: Vec<BundleEntry>,
47    flags: u16,
48}
49
50impl<'a> BundleReader<'a> {
51    /// Bounded cold admission for the current definite-length index encoding.
52    /// Shared preflight borrows CBOR fields before cold materialization can
53    /// allocate. Unknown fields, indefinite encodings and trailing index data
54    /// require an explicit compatibility profile instead of implicit skipping.
55    /// This verifies bytes and structure, not source rights or asset acceptance.
56    pub fn parse_with_limits(
57        bytes: &'a [u8],
58        limits: BundleReadLimits,
59    ) -> Result<Self, BundleError> {
60        use super::stream::index::{ExtentLimit, IndexError, admit_header, decode_owned_index};
61        if bytes.len() < BUNDLE_HEADER_SIZE {
62            return Err(BundleError::TooShort);
63        }
64        if bytes.len() > limits.file_bytes {
65            return Err(IndexError::ResidentLimit.into_bundle_error());
66        }
67        let extent =
68            u64::try_from(bytes.len()).map_err(|_| IndexError::Overflow.into_bundle_error())?;
69        let max_logical_bytes = u64::try_from(limits.file_bytes)
70            .map_err(|_| IndexError::Overflow.into_bundle_error())?;
71        let header = admit_header(
72            &bytes[..BUNDLE_HEADER_SIZE],
73            extent,
74            ExtentLimit { max_logical_bytes },
75            limits.into(),
76        )
77        .map_err(IndexError::into_bundle_error)?;
78        let start = usize::try_from(header.index_offset())
79            .map_err(|_| IndexError::Overflow.into_bundle_error())?;
80        let len = usize::try_from(header.index_length())
81            .map_err(|_| IndexError::Overflow.into_bundle_error())?;
82        let end = start
83            .checked_add(len)
84            .ok_or_else(|| IndexError::Overflow.into_bundle_error())?;
85        let index = bytes
86            .get(start..end)
87            .ok_or_else(|| IndexError::Footer.into_bundle_error())?;
88        let got = whole_file_crc(bytes);
89        if got != header.crc32c() {
90            return Err(BundleError::CrcMismatch {
91                expected: header.crc32c(),
92                got,
93            });
94        }
95        let entries = decode_owned_index(&header, index, limits.into())
96            .map_err(IndexError::into_bundle_error)?;
97        // Shared admission proves disjointness before any payload hashing.
98        use sha2::{Digest, Sha256};
99        for entry in &entries {
100            let start = usize::try_from(entry.offset)
101                .map_err(|_| IndexError::Overflow.into_bundle_error())?;
102            let end = entry
103                .offset
104                .checked_add(entry.length)
105                .ok_or_else(|| IndexError::Overflow.into_bundle_error())?;
106            let end = usize::try_from(end).map_err(|_| IndexError::Overflow.into_bundle_error())?;
107            let payload = bytes
108                .get(start..end)
109                .ok_or_else(|| IndexError::EntryBounds.into_bundle_error())?;
110            if Sha256::digest(payload).as_slice() != entry.sha256.as_slice() {
111                return Err(BundleError::Cbor(
112                    "Bundle entry payload hash differs".into(),
113                ));
114            }
115        }
116        Ok(Self {
117            bytes,
118            entries,
119            flags: header.probe().flags,
120        })
121    }
122
123    /// Parse and fully validate a bundle: magic, version, declared length,
124    /// whole-file CRC, and every entry's bounds. On success the payloads are
125    /// known-good and in-bounds, so `get`/`segment` are total on valid keys.
126    pub fn parse(bytes: &'a [u8]) -> Result<Self, BundleError> {
127        if bytes.len() < BUNDLE_HEADER_SIZE {
128            return Err(BundleError::TooShort);
129        }
130        if bytes[OFF_MAGIC..OFF_MAGIC + 4] != BUNDLE_MAGIC {
131            return Err(BundleError::BadMagic);
132        }
133        let version = u16::from_le_bytes([bytes[OFF_VERSION], bytes[OFF_VERSION + 1]]);
134        if version != BUNDLE_VERSION {
135            return Err(BundleError::UnsupportedVersion(version));
136        }
137        let flags = u16::from_le_bytes([bytes[OFF_FLAGS], bytes[OFF_FLAGS + 1]]);
138
139        let total_length = read_u64(bytes, OFF_TOTAL_LENGTH);
140        if total_length as usize != bytes.len() {
141            return Err(BundleError::LengthMismatch {
142                header: total_length,
143                actual: bytes.len(),
144            });
145        }
146
147        // Integrity before trusting any offset.
148        let stored_crc = read_u32(bytes, OFF_CRC32C);
149        let got = whole_file_crc(bytes);
150        if got != stored_crc {
151            return Err(BundleError::CrcMismatch {
152                expected: stored_crc,
153                got,
154            });
155        }
156
157        let index_offset = read_u64(bytes, OFF_INDEX_OFFSET);
158        let index_length = read_u64(bytes, OFF_INDEX_LENGTH);
159        let end = index_offset
160            .checked_add(index_length)
161            .ok_or(BundleError::BadIndexPointer {
162                offset: index_offset,
163                length: index_length,
164                total: bytes.len(),
165            })?;
166        if index_offset < BUNDLE_HEADER_SIZE as u64 || end as usize > bytes.len() {
167            return Err(BundleError::BadIndexPointer {
168                offset: index_offset,
169                length: index_length,
170                total: bytes.len(),
171            });
172        }
173
174        let index_slice = &bytes[index_offset as usize..end as usize];
175        let entries: Vec<BundleEntry> =
176            ciborium::from_reader(index_slice).map_err(|e| BundleError::Cbor(e.to_string()))?;
177
178        // Every entry must lie within the payload region (before the index).
179        let check_4096 = (flags & BUNDLE_FLAG_PAGE_ALIGN_4096) != 0;
180        for e in &entries {
181            let e_end =
182                e.offset
183                    .checked_add(e.length)
184                    .ok_or_else(|| BundleError::EntryOutOfBounds {
185                        key: e.key.clone(),
186                        offset: e.offset,
187                        length: e.length,
188                    })?;
189            if e.offset < BUNDLE_HEADER_SIZE as u64 || e_end > index_offset {
190                return Err(BundleError::EntryOutOfBounds {
191                    key: e.key.clone(),
192                    offset: e.offset,
193                    length: e.length,
194                });
195            }
196            if check_4096 && (e.offset % BUNDLE_PAGE_ALIGN_4096 as u64 != 0) {
197                return Err(BundleError::UnpageAligned {
198                    key: e.key.clone(),
199                    offset: e.offset,
200                    alignment: BUNDLE_PAGE_ALIGN_4096,
201                });
202            }
203        }
204
205        Ok(Self {
206            bytes,
207            entries,
208            flags,
209        })
210    }
211
212    /// The bundle's entries (index order).
213    pub fn entries(&self) -> &[BundleEntry] {
214        &self.entries
215    }
216
217    /// Reserved format flags from the header (0 in v1; reserved for future use
218    /// such as a signed/compressed-entry variant).
219    pub fn flags(&self) -> u16 {
220        self.flags
221    }
222
223    /// True when the bundle header declares 4096-byte OS page alignment.
224    pub fn is_page_aligned_4096(&self) -> bool {
225        (self.flags & BUNDLE_FLAG_PAGE_ALIGN_4096) != 0
226    }
227
228    /// The index record for `key`, if present.
229    pub fn entry(&self, key: &str) -> Option<&BundleEntry> {
230        self.entries.iter().find(|e| e.key == key)
231    }
232
233    /// A zero-copy slice of the intact embedded file for `key`. This slice **is**
234    /// a byte-identical standalone `.q42` / `.10d` / `.p64` — hand it straight to
235    /// that format's reader.
236    pub fn get(&self, key: &str) -> Option<&'a [u8]> {
237        let e = self.entry(key)?;
238        Some(&self.bytes[e.offset as usize..(e.offset + e.length) as usize])
239    }
240
241    /// A zero-copy slice of an **interior segment** of an entry
242    /// (`[seg_offset .. seg_offset+seg_len)` within the embedded file). Returns
243    /// `None` if the key is unknown or the segment runs past the entry — proving
244    /// the bundle does not interfere with segment-level access into an entry.
245    pub fn segment(&self, key: &str, seg_offset: u64, seg_len: u64) -> Option<&'a [u8]> {
246        let e = self.entry(key)?;
247        let seg_end = seg_offset.checked_add(seg_len)?;
248        if seg_end > e.length {
249            return None;
250        }
251        let start = (e.offset + seg_offset) as usize;
252        Some(&self.bytes[start..start + seg_len as usize])
253    }
254
255    /// Verify one entry's payload against its recorded SHA-256.
256    pub fn verify_entry(&self, key: &str) -> bool {
257        use sha2::{Digest, Sha256};
258        let Some(e) = self.entry(key) else {
259            return false;
260        };
261        let Some(payload) = self.get(key) else {
262            return false;
263        };
264        let mut hasher = Sha256::new();
265        hasher.update(payload);
266        hasher.finalize().as_slice() == e.sha256.as_slice()
267    }
268
269    /// Retrieve and decode the embedded digital twin family manifest, if present.
270    pub fn get_family_manifest(
271        &self,
272    ) -> Result<Option<super::variant_matrix::TwinFamilyManifest>, BundleError> {
273        match self.get(super::variant_matrix::FAMILY_MANIFEST_ENTRY_KEY) {
274            Some(bytes) => {
275                let manifest = super::variant_matrix::TwinFamilyManifest::decode_from_cbor(bytes)?;
276                Ok(Some(manifest))
277            }
278            None => Ok(None),
279        }
280    }
281
282    /// Retrieve and decode the embedded holistic species metadata, if present.
283    pub fn get_species_meta(
284        &self,
285    ) -> Result<Option<super::species_meta::HolisticSpeciesMeta>, BundleError> {
286        match self.get(super::species_meta::SPECIES_META_ENTRY_KEY) {
287            Some(bytes) => {
288                let meta = super::species_meta::HolisticSpeciesMeta::decode_from_cbor(bytes)?;
289                Ok(Some(meta))
290            }
291            None => Ok(None),
292        }
293    }
294}
295
296/// Open a `.hmc` bundle from disk via `mmap` for zero-copy access (native).
297///
298/// The mapping backs the slices returned by [`BundleReader`], so an embedded
299/// file (or one of its interior segments) can be read without copying the
300/// bundle into the heap — the intended path for shipping a large asset pack.
301#[cfg(not(target_arch = "wasm32"))]
302pub struct BundleMmap {
303    map: memmap2::Mmap,
304}
305
306#[cfg(not(target_arch = "wasm32"))]
307impl BundleMmap {
308    /// Memory-map a bundle file.
309    pub fn open(path: impl AsRef<std::path::Path>) -> Result<Self, BundleError> {
310        let file = std::fs::File::open(path).map_err(|e| BundleError::Io(e.to_string()))?;
311        // SAFETY: the map is read-only and lives as long as `self`; callers get
312        // slices bounded by `self`.
313        let map =
314            unsafe { memmap2::Mmap::map(&file) }.map_err(|e| BundleError::Io(e.to_string()))?;
315        Ok(Self { map })
316    }
317
318    /// The raw mapped bytes.
319    pub fn as_bytes(&self) -> &[u8] {
320        &self.map
321    }
322
323    /// A validated reader borrowing the mapping.
324    pub fn reader(&self) -> Result<BundleReader<'_>, BundleError> {
325        BundleReader::parse(&self.map)
326    }
327}
328
329#[cfg(test)]
330mod tests {
331    use super::super::writer::BundleWriter;
332    use super::*;
333
334    fn sample_bundle() -> Vec<u8> {
335        let mut w = BundleWriter::new();
336        w.add_file(
337            "liver.10d",
338            "10d",
339            (0..200u32).map(|i| i as u8).collect(),
340            Some(vec![0xAA, 0xBB]),
341        )
342        .unwrap();
343        w.add_file(
344            "graph.q42",
345            "q42",
346            b"hello q42 segment world".to_vec(),
347            None,
348        )
349        .unwrap();
350        w.build().unwrap()
351    }
352
353    #[test]
354    fn roundtrip_is_byte_identical() {
355        let liver: Vec<u8> = (0..200u32).map(|i| i as u8).collect();
356        let bytes = sample_bundle();
357        let r = BundleReader::parse(&bytes).unwrap();
358        assert_eq!(r.entries().len(), 2);
359        // The embedded file comes back byte-for-byte — the transparency guarantee.
360        assert_eq!(r.get("liver.10d").unwrap(), liver.as_slice());
361        assert_eq!(r.get("graph.q42").unwrap(), b"hello q42 segment world");
362        assert_eq!(r.entry("liver.10d").unwrap().kind, "10d");
363        assert_eq!(
364            r.entry("liver.10d").unwrap().meta.as_deref(),
365            Some(&[0xAA, 0xBB][..])
366        );
367        assert!(r.get("missing").is_none());
368    }
369
370    #[test]
371    fn interior_segment_access_is_not_interfered_with() {
372        let bytes = sample_bundle();
373        let r = BundleReader::parse(&bytes).unwrap();
374        // Pull the interior segment "q42 segment" (offset 6, len 11) directly.
375        assert_eq!(r.segment("graph.q42", 6, 11).unwrap(), b"q42 segment");
376        // A segment past the end is refused, not read out of the entry.
377        assert!(r.segment("graph.q42", 20, 100).is_none());
378    }
379
380    #[test]
381    fn per_entry_sha256_verifies_and_detects_tamper() {
382        let mut bytes = sample_bundle();
383        let r = BundleReader::parse(&bytes).unwrap();
384        assert!(r.verify_entry("liver.10d"));
385        assert!(r.verify_entry("graph.q42"));
386        drop(r);
387        // Flip a payload byte inside an entry: whole-file CRC now fails on parse.
388        let off = {
389            let r = BundleReader::parse(&bytes).unwrap();
390            r.entry("liver.10d").unwrap().offset as usize
391        };
392        bytes[off] ^= 0xFF;
393        assert!(matches!(
394            BundleReader::parse(&bytes),
395            Err(BundleError::CrcMismatch { .. })
396        ));
397    }
398
399    #[test]
400    fn rejects_bad_magic_and_short_input() {
401        assert!(matches!(
402            BundleReader::parse(&[0u8; 10]),
403            Err(BundleError::TooShort)
404        ));
405        let mut bytes = sample_bundle();
406        bytes[0] = b'X';
407        assert!(matches!(
408            BundleReader::parse(&bytes),
409            Err(BundleError::BadMagic)
410        ));
411    }
412
413    #[cfg(not(target_arch = "wasm32"))]
414    #[test]
415    fn mmap_roundtrip() {
416        let bytes = sample_bundle();
417        let dir = tempfile::tempdir().unwrap();
418        let path = dir.path().join("pack.hmc");
419        std::fs::write(&path, &bytes).unwrap();
420        let m = BundleMmap::open(&path).unwrap();
421        let r = m.reader().unwrap();
422        assert_eq!(r.get("graph.q42").unwrap(), b"hello q42 segment world");
423        assert!(r.verify_entry("liver.10d"));
424    }
425}