Skip to main content

qualia_core_db/q42/
selective_streamer.rs

1//! Installed-HMC Q42 Query-to-Byte-Range Streamer (`BS-30`, `AR-04`, `ENG-07`).
2//!
3//! Provides Tier-1 zero-heap selective byte-range resolution and verification
4//! directly from installed `.hmc` containers into caller-allocated buffers.
5
6use core::fmt;
7use crate::container_10d::crc32c::crc32c;
8
9pub const CONTAINER_KIND_Q42: u8 = 1 << 0; // 0x01: .q42 semantic volume / graph
10pub const CONTAINER_KIND_M64: u8 = 1 << 1; // 0x02: .m64 raster / spatial chunk
11pub const CONTAINER_KIND_10D: u8 = 1 << 2; // 0x04: .10d volumetric mesh / manifold
12pub const CONTAINER_KIND_P64: u8 = 1 << 3; // 0x08: .p64 neural weight shard
13pub const CONTAINER_KIND_ALL: u8 = 0x0F;
14
15/// Query selector specifying filtering criteria for selective HMC streaming.
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
17pub struct SelectiveStreamQuery {
18    pub spatial_cell_hash: Option<u64>,
19    pub scale_band_id: Option<u8>,
20    pub organ_id: Option<u64>,
21    pub lod_tier: Option<u8>,
22    pub container_kind_mask: u8,
23}
24
25impl SelectiveStreamQuery {
26    #[inline]
27    pub const fn new() -> Self {
28        Self {
29            spatial_cell_hash: None,
30            scale_band_id: None,
31            organ_id: None,
32            lod_tier: None,
33            container_kind_mask: CONTAINER_KIND_ALL,
34        }
35    }
36
37    #[inline]
38    pub const fn with_organ(mut self, organ_id: u64) -> Self {
39        self.organ_id = Some(organ_id);
40        self
41    }
42
43    #[inline]
44    pub const fn with_lod(mut self, lod_tier: u8) -> Self {
45        self.lod_tier = Some(lod_tier);
46        self
47    }
48
49    #[inline]
50    pub const fn with_spatial_cell(mut self, cell_hash: u64) -> Self {
51        self.spatial_cell_hash = Some(cell_hash);
52        self
53    }
54
55    #[inline]
56    pub const fn with_scale_band(mut self, scale_band_id: u8) -> Self {
57        self.scale_band_id = Some(scale_band_id);
58        self
59    }
60
61    #[inline]
62    pub const fn with_container_mask(mut self, mask: u8) -> Self {
63        self.container_kind_mask = mask;
64        self
65    }
66
67    #[inline]
68    pub fn matches(&self, entry: &HmcIndexSliceEntry) -> bool {
69        if self.container_kind_mask != 0 && (entry.container_kind & self.container_kind_mask) == 0 {
70            return false;
71        }
72        if let Some(spatial) = self.spatial_cell_hash {
73            if entry.spatial_cell_hash != spatial { return false; }
74        }
75        if let Some(band) = self.scale_band_id {
76            if entry.scale_band_id != band { return false; }
77        }
78        if let Some(organ) = self.organ_id {
79            if entry.organ_id != organ { return false; }
80        }
81        if let Some(lod) = self.lod_tier {
82            if entry.lod_tier != lod { return false; }
83        }
84        true
85    }
86}
87
88/// Metadata entry in the HMC index for a single sub-resource slice or chunk.
89#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
90pub struct HmcIndexSliceEntry {
91    pub entry_id: u32,
92    pub container_kind: u8,
93    pub spatial_cell_hash: u64,
94    pub scale_band_id: u8,
95    pub organ_id: u64,
96    pub lod_tier: u8,
97    pub byte_offset: u64,
98    pub byte_length: u32,
99    pub expected_crc32c: u32,
100}
101
102/// A resolved byte range ready for targeted streaming and verification.
103#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
104pub struct ResolvedByteRange {
105    pub entry_id: u32,
106    pub container_kind: u8,
107    pub byte_offset: u64,
108    pub byte_length: u32,
109    pub expected_crc32c: u32,
110}
111
112/// Read statistics oracle proving selective access and untouched entries.
113#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
114pub struct StreamReadOracle {
115    pub total_bytes_requested: u64,
116    pub ranges_touched: usize,
117    pub untouched_entries_skipped: usize,
118}
119
120/// Typed error returned during selective range streaming or integrity verification.
121#[derive(Debug, Clone, Copy, PartialEq, Eq)]
122pub enum SelectiveStreamError {
123    Cancelled,
124    ReadBudgetExceeded { required: u64, budget: u64 },
125    DigestMismatch { entry_id: u32, expected: u32, actual: u32 },
126    OutputBufferTooSmall { required: usize, provided: usize },
127    InvalidSliceBounds { offset: u64, length: u32, source_len: usize },
128}
129
130impl fmt::Display for SelectiveStreamError {
131    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
132        match self {
133            Self::Cancelled => write!(f, "streaming operation cancelled"),
134            Self::ReadBudgetExceeded { required, budget } => {
135                write!(f, "read budget exceeded: {required} > {budget}")
136            }
137            Self::DigestMismatch { entry_id, expected, actual } => {
138                write!(f, "CRC-32C mismatch on entry {entry_id}: expected {expected:#010x}, got {actual:#010x}")
139            }
140            Self::OutputBufferTooSmall { required, provided } => {
141                write!(f, "output buffer too small: required {required}, provided {provided}")
142            }
143            Self::InvalidSliceBounds { offset, length, source_len } => {
144                write!(f, "invalid bounds [{offset}..+{length}] for source length {source_len}")
145            }
146        }
147    }
148}
149
150impl std::error::Error for SelectiveStreamError {}
151
152/// Tier-1 zero-heap selective HMC byte-range resolver.
153pub fn resolve_selective_hmc_ranges_zero_heap(
154    index: &[HmcIndexSliceEntry],
155    query: &SelectiveStreamQuery,
156    max_total_bytes: u64,
157    cancelled: bool,
158    out_ranges: &mut [ResolvedByteRange],
159) -> Result<(usize, StreamReadOracle), SelectiveStreamError> {
160    if cancelled {
161        return Err(SelectiveStreamError::Cancelled);
162    }
163    let mut total_bytes_requested: u64 = 0;
164    let mut ranges_touched: usize = 0;
165    let mut untouched_entries_skipped: usize = 0;
166
167    for entry in index {
168        if query.matches(entry) {
169            let next_total = total_bytes_requested.saturating_add(entry.byte_length as u64);
170            if next_total > max_total_bytes {
171                return Err(SelectiveStreamError::ReadBudgetExceeded {
172                    required: next_total,
173                    budget: max_total_bytes,
174                });
175            }
176            total_bytes_requested = next_total;
177
178            if ranges_touched >= out_ranges.len() {
179                return Err(SelectiveStreamError::OutputBufferTooSmall {
180                    required: ranges_touched + 1,
181                    provided: out_ranges.len(),
182                });
183            }
184
185            out_ranges[ranges_touched] = ResolvedByteRange {
186                entry_id: entry.entry_id,
187                container_kind: entry.container_kind,
188                byte_offset: entry.byte_offset,
189                byte_length: entry.byte_length,
190                expected_crc32c: entry.expected_crc32c,
191            };
192            ranges_touched += 1;
193        } else {
194            untouched_entries_skipped += 1;
195        }
196    }
197
198    Ok((
199        ranges_touched,
200        StreamReadOracle {
201            total_bytes_requested,
202            ranges_touched,
203            untouched_entries_skipped,
204        },
205    ))
206}
207
208/// Streams and verifies a specific sub-range from an in-memory HMC slice into `out_buf`.
209pub fn stream_verified_subrange_into(
210    hmc_slice: &[u8],
211    range: &ResolvedByteRange,
212    cancelled: bool,
213    out_buf: &mut [u8],
214) -> Result<usize, SelectiveStreamError> {
215    if cancelled {
216        return Err(SelectiveStreamError::Cancelled);
217    }
218    let len = range.byte_length as usize;
219    if out_buf.len() < len {
220        return Err(SelectiveStreamError::OutputBufferTooSmall {
221            required: len,
222            provided: out_buf.len(),
223        });
224    }
225
226    let start = usize::try_from(range.byte_offset).map_err(|_| {
227        SelectiveStreamError::InvalidSliceBounds {
228            offset: range.byte_offset,
229            length: range.byte_length,
230            source_len: hmc_slice.len(),
231        }
232    })?;
233
234    let end = start.checked_add(len).ok_or(SelectiveStreamError::InvalidSliceBounds {
235        offset: range.byte_offset,
236        length: range.byte_length,
237        source_len: hmc_slice.len(),
238    })?;
239
240    if end > hmc_slice.len() {
241        return Err(SelectiveStreamError::InvalidSliceBounds {
242            offset: range.byte_offset,
243            length: range.byte_length,
244            source_len: hmc_slice.len(),
245        });
246    }
247
248    let source = &hmc_slice[start..end];
249    let actual_crc = crc32c(source);
250
251    if actual_crc != range.expected_crc32c {
252        out_buf[..len].fill(0);
253        return Err(SelectiveStreamError::DigestMismatch {
254            entry_id: range.entry_id,
255            expected: range.expected_crc32c,
256            actual: actual_crc,
257        });
258    }
259
260    out_buf[..len].copy_from_slice(source);
261    Ok(len)
262}
263
264#[cfg(test)]
265mod tests {
266    use super::*;
267    #[cfg(any(not(target_arch = "wasm32"), feature = "wasm-scientific"))]
268    use crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc;
269
270    fn make_test_fixture() -> ([HmcIndexSliceEntry; 4], [u8; 1024]) {
271        let mut hmc_data = [0u8; 1024];
272        hmc_data[64..192].fill(0x11);
273        let crc0 = crc32c(&hmc_data[64..192]);
274        hmc_data[192..256].fill(0x22);
275        let crc1 = crc32c(&hmc_data[192..256]);
276        hmc_data[256..512].fill(0x33);
277        let crc2 = crc32c(&hmc_data[256..512]);
278        hmc_data[512..640].fill(0x44);
279        let crc3 = crc32c(&hmc_data[512..640]);
280
281        let index = [
282            HmcIndexSliceEntry { entry_id: 1, container_kind: CONTAINER_KIND_10D, spatial_cell_hash: 10, scale_band_id: 1, organ_id: 100, lod_tier: 0, byte_offset: 64, byte_length: 128, expected_crc32c: crc0 },
283            HmcIndexSliceEntry { entry_id: 2, container_kind: CONTAINER_KIND_10D, spatial_cell_hash: 10, scale_band_id: 1, organ_id: 100, lod_tier: 2, byte_offset: 192, byte_length: 64, expected_crc32c: crc1 },
284            HmcIndexSliceEntry { entry_id: 3, container_kind: CONTAINER_KIND_M64, spatial_cell_hash: 20, scale_band_id: 2, organ_id: 200, lod_tier: 0, byte_offset: 256, byte_length: 256, expected_crc32c: crc2 },
285            HmcIndexSliceEntry { entry_id: 4, container_kind: CONTAINER_KIND_Q42, spatial_cell_hash: 20, scale_band_id: 2, organ_id: 200, lod_tier: 1, byte_offset: 512, byte_length: 128, expected_crc32c: crc3 },
286        ];
287        (index, hmc_data)
288    }
289
290    #[test]
291    fn selective_query_skips_unrelated_chunks_and_verifies_payload() {
292        let (index, hmc_data) = make_test_fixture();
293        let query = SelectiveStreamQuery::new().with_organ(100).with_lod(2);
294        let mut out_ranges = [ResolvedByteRange::default(); 8];
295        let (count, oracle) = resolve_selective_hmc_ranges_zero_heap(&index, &query, 1024, false, &mut out_ranges).unwrap();
296
297        assert_eq!(count, 1);
298        assert_eq!(oracle.ranges_touched, 1);
299        assert_eq!(oracle.untouched_entries_skipped, 3);
300        assert_eq!(oracle.total_bytes_requested, 64);
301        assert_eq!(out_ranges[0].entry_id, 2);
302
303        let mut read_buf = [0u8; 128];
304        let bytes_read = stream_verified_subrange_into(&hmc_data, &out_ranges[0], false, &mut read_buf).unwrap();
305        assert_eq!(bytes_read, 64);
306        assert!(read_buf[..64].iter().all(|&b| b == 0x22));
307    }
308
309    #[test]
310    fn budget_overflow_fails_closed_before_read() {
311        let (index, _) = make_test_fixture();
312        let query = SelectiveStreamQuery::new().with_organ(100);
313        let mut out_ranges = [ResolvedByteRange::default(); 8];
314        let err = resolve_selective_hmc_ranges_zero_heap(&index, &query, 100, false, &mut out_ranges).unwrap_err();
315
316        match err {
317            SelectiveStreamError::ReadBudgetExceeded { required, budget } => {
318                assert_eq!(required, 128);
319                assert_eq!(budget, 100);
320            }
321            other => panic!("unexpected error variant: {other:?}"),
322        }
323    }
324
325    #[test]
326    fn cancellation_fails_closed() {
327        let (index, hmc_data) = make_test_fixture();
328        let query = SelectiveStreamQuery::new();
329        let mut out_ranges = [ResolvedByteRange::default(); 8];
330        let err_resolve = resolve_selective_hmc_ranges_zero_heap(&index, &query, 1024, true, &mut out_ranges).unwrap_err();
331        assert_eq!(err_resolve, SelectiveStreamError::Cancelled);
332
333        let range = ResolvedByteRange { entry_id: 1, container_kind: CONTAINER_KIND_10D, byte_offset: 64, byte_length: 128, expected_crc32c: 0x12345678 };
334        let mut buf = [0u8; 128];
335        let err_stream = stream_verified_subrange_into(&hmc_data, &range, true, &mut buf).unwrap_err();
336        assert_eq!(err_stream, SelectiveStreamError::Cancelled);
337    }
338
339    #[test]
340    fn tampered_byte_triggers_digest_mismatch_and_zeroes_buffer() {
341        let (index, mut hmc_data) = make_test_fixture();
342        let query = SelectiveStreamQuery::new().with_organ(100).with_lod(0);
343        let mut out_ranges = [ResolvedByteRange::default(); 4];
344        let (count, _) = resolve_selective_hmc_ranges_zero_heap(&index, &query, 1024, false, &mut out_ranges).unwrap();
345        assert_eq!(count, 1);
346
347        hmc_data[100] ^= 0xA5;
348        let mut out_buf = [0xFFu8; 128];
349        let err = stream_verified_subrange_into(&hmc_data, &out_ranges[0], false, &mut out_buf).unwrap_err();
350
351        match err {
352            SelectiveStreamError::DigestMismatch { entry_id, expected, actual } => {
353                assert_eq!(entry_id, 1);
354                assert_ne!(expected, actual);
355            }
356            other => panic!("unexpected error variant: {other:?}"),
357        }
358        assert!(out_buf.iter().all(|&b| b == 0));
359    }
360
361    #[test]
362    #[cfg(any(not(target_arch = "wasm32"), feature = "wasm-scientific"))]
363    fn zero_alloc_streamer_resolution_and_read() {
364        let (index, hmc_data) = make_test_fixture();
365        let query = SelectiveStreamQuery::new().with_organ(100).with_lod(0);
366        let mut out_ranges = [ResolvedByteRange::default(); 4];
367        let mut out_buf = [0u8; 128];
368        let _ = crc32c(b"warmup");
369
370        assert_zero_alloc("resolve_selective_hmc_ranges_zero_heap", || {
371            let res = resolve_selective_hmc_ranges_zero_heap(&index, &query, 1024, false, &mut out_ranges);
372            assert!(res.is_ok());
373            let read_res = stream_verified_subrange_into(&hmc_data, &out_ranges[0], false, &mut out_buf);
374            assert!(read_res.is_ok());
375        });
376    }
377}