Skip to main content

qualia_core_db/bundle/
streaming_loader.rs

1//! Progressive HTTP Range / OPFS slice streaming loader for `.hmc` bundles.
2//!
3//! Fetches the 64-byte `.hmc` header and CBOR index footer first, then schedules
4//! prioritized byte-range requests across three tiers:
5//! 1. [`StreamPriorityTier::Tier0SemanticAndProcedural`] — `.q42` semantic slices,
6//!    manifests, and `.10d` `EONT1` / `PROC1` / `SKL1` / `MRP1` / `IMPO` / `SURF` sections.
7//! 2. [`StreamPriorityTier::Tier1MidLodMeshes`] — Mid-LOD (`LOD2` / `LOD1`) mesh sections.
8//! 3. [`StreamPriorityTier::Tier2HighPolyAndMip0`] — High-poly `LOD0` geometry and `KTX2` mip0.
9//!
10//! Every range arrival is verified against either the entry's SHA-256 digest or the
11//! `.10d` section's hardware-accelerated CRC-32C checksum before admission.
12
13use sha2::{Digest, Sha256};
14
15use crate::container_10d::crc32c::crc32c;
16use crate::container_10d::{
17    Container10dHeader, HEADER_BYTE_SIZE, MAGIC_10D, SECTION_DESCRIPTOR_SIZE, SectionDescriptor,
18    SectionType,
19};
20
21use super::format::{BundleEntry, BundleError, SHA256_LEN};
22#[cfg(test)]
23use super::format::BUNDLE_HEADER_SIZE;
24use super::stream::index::{
25    CheckedHeader, ExtentLimit, IndexError, IndexReadLimits, admit_header, decode_owned_index,
26};
27
28/// Priority tier for progressive `.hmc` range hydration.
29#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
30#[repr(u8)]
31pub enum StreamPriorityTier {
32    /// Immediate semantic + procedural + impostor/surfel readiness (`< 50 KB`).
33    Tier0SemanticAndProcedural = 0,
34    /// Mid-LOD (`LOD2` / `LOD1`) geometry and materials.
35    Tier1MidLodMeshes = 1,
36    /// High-poly `LOD0` geometry, `KTX2` mip0 textures, and bulk weights.
37    Tier2HighPolyAndMip0 = 2,
38}
39
40/// A single byte-range request within a `.hmc` bundle (`[offset .. offset + length)`).
41#[derive(Debug, Clone, PartialEq, Eq)]
42pub struct HmcRangeRequest {
43    pub entry_key: String,
44    pub entry_kind: String,
45    pub tier: StreamPriorityTier,
46    pub offset: u64,
47    pub length: u64,
48    /// Expected SHA-256 when the range covers an entire `BundleEntry`.
49    pub expected_sha256: Option<[u8; SHA256_LEN]>,
50    /// Expected CRC-32C when the range covers an interior `.10d` section.
51    pub expected_crc32c: Option<u32>,
52    /// Optional `.10d` section type tag (`SectionType as u16`).
53    pub section_type_raw: Option<u16>,
54}
55
56pub use super::stream::index::HmcHeaderProbe;
57
58/// Progressive range planner constructed from the 64B `.hmc` header and CBOR index footer.
59#[derive(Debug, Clone)]
60pub struct HmcProgressiveRangePlanner {
61    header: HmcHeaderProbe,
62    entries: Vec<BundleEntry>,
63}
64
65impl HmcProgressiveRangePlanner {
66    /// Construct a planner from the 64-byte header probe and the fetched CBOR index footer bytes.
67    pub fn from_header_and_index(
68        header: HmcHeaderProbe,
69        index_footer_bytes: &[u8],
70    ) -> Result<Self, BundleError> {
71        Self::from_header_and_index_with_limits(
72            header,
73            index_footer_bytes,
74            header.total_length,
75            ExtentLimit::default(),
76            IndexReadLimits::default(),
77        )
78    }
79
80    /// Bounded structural admission. The public probe cannot prove original reserved bytes.
81    /// expected_extent must be pinned independently by the caller for a remote object.
82    pub fn from_header_and_index_with_limits(
83        header: HmcHeaderProbe,
84        index_footer_bytes: &[u8],
85        expected_extent: u64,
86        extent_limit: ExtentLimit,
87        limits: IndexReadLimits,
88    ) -> Result<Self, BundleError> {
89        let checked = CheckedHeader::from_probe(header, expected_extent, extent_limit, limits)
90            .map_err(IndexError::into_bundle_error)?;
91        Self::from_checked_index(checked, index_footer_bytes, limits)
92    }
93
94    /// Preferred remote facade: admit exact original header bytes and an independent extent.
95    /// Success validates directory structure; payload CRC/hash/proofs remain incomplete.
96    pub fn from_raw_header_and_index_with_limits(
97        header_bytes: &[u8],
98        index_footer_bytes: &[u8],
99        expected_extent: u64,
100        extent_limit: ExtentLimit,
101        limits: IndexReadLimits,
102    ) -> Result<Self, BundleError> {
103        let checked = admit_header(header_bytes, expected_extent, extent_limit, limits)
104            .map_err(IndexError::into_bundle_error)?;
105        Self::from_checked_index(checked, index_footer_bytes, limits)
106    }
107
108    fn from_checked_index(
109        checked: CheckedHeader,
110        bytes: &[u8],
111        limits: IndexReadLimits,
112    ) -> Result<Self, BundleError> {
113        let entries =
114            decode_owned_index(&checked, bytes, limits).map_err(IndexError::into_bundle_error)?;
115        Ok(Self {
116            header: checked.probe(),
117            entries,
118        })
119    }
120
121    /// Validated header probe.
122    #[inline]
123    pub fn header(&self) -> &HmcHeaderProbe {
124        &self.header
125    }
126
127    /// Decoded index entries.
128    #[inline]
129    pub fn entries(&self) -> &[BundleEntry] {
130        &self.entries
131    }
132
133    /// Classify an entry into its default [`StreamPriorityTier`] and produce whole-entry
134    /// range requests sorted by `(tier, offset)`.
135    pub fn plan_entry_ranges(&self) -> Vec<HmcRangeRequest> {
136        let mut reqs = Vec::with_capacity(self.entries.len());
137        for entry in &self.entries {
138            let tier = classify_entry_tier(&entry.key, &entry.kind);
139            let expected_sha256 = entry.sha256.as_slice().try_into().ok();
140            reqs.push(HmcRangeRequest {
141                entry_key: entry.key.clone(),
142                entry_kind: entry.kind.clone(),
143                tier,
144                offset: entry.offset,
145                length: entry.length,
146                expected_sha256,
147                expected_crc32c: None,
148                section_type_raw: None,
149            });
150        }
151        reqs.sort_by_key(|r| (r.tier, r.offset));
152        reqs
153    }
154
155    /// Given the front prefix of a `.10d` entry inside the bundle (`>= 64 + section_count * 24` bytes),
156    /// plan sub-entry section range requests ordered by [`StreamPriorityTier`].
157    pub fn plan_10d_interior_section_ranges(
158        &self,
159        entry_key: &str,
160        entry_prefix_bytes: &[u8],
161    ) -> Result<Vec<HmcRangeRequest>, BundleError> {
162        let entry = self
163            .entries
164            .iter()
165            .find(|e| e.key == entry_key)
166            .ok_or(BundleError::EmptyKey)?;
167        if entry_prefix_bytes.len() < HEADER_BYTE_SIZE {
168            return Err(BundleError::TooShort);
169        }
170        // Header validation also checks the table pointer against the supplied
171        // slice. Include the available entry prefix, but never bytes belonging
172        // to a following bundle entry.
173        let prefix_len = entry_prefix_bytes
174            .len()
175            .min(usize::try_from(entry.length).unwrap_or(usize::MAX));
176        let hdr = Container10dHeader::parse(&entry_prefix_bytes[..prefix_len])
177            .map_err(|_| BundleError::BadMagic)?;
178        if hdr.magic != MAGIC_10D {
179            return Err(BundleError::BadMagic);
180        }
181        let sec_count = hdr.section_count as usize;
182        let toc_start = hdr.section_table_offset as usize;
183        let toc_end = sec_count
184            .checked_mul(SECTION_DESCRIPTOR_SIZE)
185            .and_then(|size| toc_start.checked_add(size))
186            .ok_or_else(|| BundleError::EntryOutOfBounds {
187                key: entry.key.clone(),
188                offset: entry.offset,
189                length: entry.length,
190            })?;
191        if toc_end as u64 > entry.length || entry_prefix_bytes.len() < toc_end {
192            return Err(BundleError::TooShort);
193        }
194        let is_low_lod_key = entry.key.contains("lod1") || entry.key.contains("lod2");
195        let mut out = Vec::with_capacity(sec_count);
196        for i in 0..sec_count {
197            let off = toc_start + i * SECTION_DESCRIPTOR_SIZE;
198            let sec: SectionDescriptor = bytemuck::pod_read_unaligned(
199                &entry_prefix_bytes[off..off + SECTION_DESCRIPTOR_SIZE],
200            );
201            let section_offset = u64::from(sec.byte_offset);
202            let section_length = u64::from(sec.byte_length);
203            if section_offset
204                .checked_add(section_length)
205                .is_none_or(|end| end > entry.length)
206            {
207                return Err(BundleError::EntryOutOfBounds {
208                    key: entry.key.clone(),
209                    offset: section_offset,
210                    length: section_length,
211                });
212            }
213            let absolute_offset = entry.offset.checked_add(section_offset).ok_or_else(|| {
214                BundleError::EntryOutOfBounds {
215                    key: entry.key.clone(),
216                    offset: section_offset,
217                    length: section_length,
218                }
219            })?;
220            let tier = classify_10d_section_tier(u16::from(sec.section_type), is_low_lod_key);
221            out.push(HmcRangeRequest {
222                entry_key: entry.key.clone(),
223                entry_kind: entry.kind.clone(),
224                tier,
225                offset: absolute_offset,
226                length: section_length,
227                expected_sha256: None,
228                expected_crc32c: Some(sec.crc32c),
229                section_type_raw: Some(u16::from(sec.section_type)),
230            });
231        }
232        out.sort_by_key(|r| (r.tier, r.offset));
233        Ok(out)
234    }
235}
236
237/// Classify a bundle entry into a progressive priority tier.
238pub fn classify_entry_tier(key: &str, kind: &str) -> StreamPriorityTier {
239    let lower_key = key.to_ascii_lowercase();
240    let lower_kind = kind.to_ascii_lowercase();
241    if lower_kind == "q42"
242        || lower_kind == "manifest"
243        || lower_kind.contains("cbor")
244        || lower_key.ends_with(".q42")
245        || lower_key.contains("manifest")
246    {
247        return StreamPriorityTier::Tier0SemanticAndProcedural;
248    }
249    if lower_key.contains("lod1") || lower_key.contains("lod2") {
250        return StreamPriorityTier::Tier1MidLodMeshes;
251    }
252    if lower_kind == "ktx2" || lower_kind == "p64" || lower_key.contains("lod0") {
253        return StreamPriorityTier::Tier2HighPolyAndMip0;
254    }
255    StreamPriorityTier::Tier1MidLodMeshes
256}
257
258/// Classify a `.10d` interior section into a progressive priority tier.
259pub fn classify_10d_section_tier(
260    section_type_raw: u16,
261    is_mid_lod_entry: bool,
262) -> StreamPriorityTier {
263    if section_type_raw == SectionType::MORPH_TARGETS_DISCRIMINANT {
264        return StreamPriorityTier::Tier0SemanticAndProcedural;
265    }
266    let Ok(raw_u8) = u8::try_from(section_type_raw) else {
267        return StreamPriorityTier::Tier1MidLodMeshes;
268    };
269    match SectionType::from_u8(raw_u8) {
270        Some(
271            SectionType::EntityOntology
272            | SectionType::ProceduralSynthesis
273            | SectionType::SkeletalAnimation
274            | SectionType::ImpostorAtlas
275            | SectionType::HierarchicalSurfels
276            | SectionType::Tensor10DNodes
277            | SectionType::ManifoldIdentityV2
278            | SectionType::ProvenanceSidecar,
279        ) => StreamPriorityTier::Tier0SemanticAndProcedural,
280        Some(
281            SectionType::QuantizedMesh
282            | SectionType::Materials
283            | SectionType::TextureCoordinates
284            | SectionType::FieldSidecar
285            | SectionType::HyperspectralCube
286            | SectionType::VoxelLattice,
287        ) => {
288            if is_mid_lod_entry {
289                StreamPriorityTier::Tier1MidLodMeshes
290            } else {
291                StreamPriorityTier::Tier2HighPolyAndMip0
292            }
293        }
294        _ => StreamPriorityTier::Tier1MidLodMeshes,
295    }
296}
297
298/// Verify an arrived byte range against its `HmcRangeRequest` integrity digest (SHA-256 and/or CRC-32C).
299pub fn verify_arrived_range(
300    req: &HmcRangeRequest,
301    arrived_bytes: &[u8],
302) -> Result<(), BundleError> {
303    if req.expected_sha256.is_none() && req.expected_crc32c.is_none() {
304        return Err(BundleError::Cbor(
305            "Range arrival requires an integrity digest".into(),
306        ));
307    }
308    if arrived_bytes.len() as u64 != req.length {
309        return Err(BundleError::LengthMismatch {
310            header: req.length,
311            actual: arrived_bytes.len(),
312        });
313    }
314    if let Some(expected_crc) = req.expected_crc32c {
315        let got_crc = crc32c(arrived_bytes);
316        if got_crc != expected_crc {
317            return Err(BundleError::CrcMismatch {
318                expected: expected_crc,
319                got: got_crc,
320            });
321        }
322    }
323    if let Some(expected_sha) = req.expected_sha256 {
324        let mut hasher = Sha256::new();
325        hasher.update(arrived_bytes);
326        let digest = hasher.finalize();
327        if digest.as_slice() != expected_sha.as_slice() {
328            return Err(BundleError::CrcMismatch {
329                expected: u32::from_le_bytes([
330                    expected_sha[0],
331                    expected_sha[1],
332                    expected_sha[2],
333                    expected_sha[3],
334                ]),
335                got: u32::from_le_bytes([digest[0], digest[1], digest[2], digest[3]]),
336            });
337        }
338    }
339    Ok(())
340}
341
342#[cfg(test)]
343#[path = "streaming_loader/tests.rs"]
344mod tests;