Skip to main content

qualia_core_db/q42/emf_container/
reader.rs

1//! Zero-Copy Structural Reader for EMF-64 Containers.
2//!
3//! Provides zero-copy parsing, CRC-32C and full BLAKE3 verification, and
4//! fast 3D Hilbert key chunk lookup.
5
6use super::chunk_directory::{
7    emf_brick_voxel_coords, emf_brick_voxel_linear_index, EmfChunkDirectoryEntry,
8    CHUNK_DIRECTORY_ENTRY_BYTES, EMF_BRICK_SIDE,
9};
10use super::header::{EmfCellRecord64, EmfContainerHeader, EmfError, MAX_DENSE_CELLS};
11use super::header::{EMF_EXTENDED_VERSION, EMF_CONTAINER_VERSION, EMF_CHUNK_FLAG_BLAKE3_FULL, EMF_FLAG_BAO_VERIFIED};
12use crate::q42::chunk_integrity::{verify_blake3_trailer, BLAKE3_TRAILER_BYTES};
13use crate::q42::m64_matter::brick3d::{decode_hilbert3d_u64, encode_hilbert3d_u64};
14
15/// Explicit EMF admission budgets (file bytes and directory rows).
16#[derive(Clone, Copy, Debug, PartialEq, Eq)]
17pub struct EmfReadLimits {
18    pub max_file_bytes: usize,
19    pub max_chunks: usize,
20}
21
22impl EmfReadLimits {
23    /// Conservative documented defaults used by [`EmfContainerReader::parse`].
24    ///
25    /// 16 MiB matches the produce-path resource budget; 4096 chunks is a
26    /// directory-row cap independent of brick payload size. Untrusted callers
27    /// should pass explicit limits via [`EmfContainerReader::parse_with_limits`].
28    pub const fn conservative() -> Self {
29        Self {
30            max_file_bytes: 16 * 1024 * 1024,
31            max_chunks: 4096,
32        }
33    }
34}
35
36#[inline]
37fn ranges_overlap(a_start: usize, a_end: usize, b_start: usize, b_end: usize) -> bool {
38    a_start < b_end && b_start < a_end
39}
40
41/// Immutable zero-copy view of an EMF-64 container file.
42#[derive(Debug)]
43pub struct EmfContainerReader<'a> {
44    bytes: &'a [u8],
45    header: EmfContainerHeader,
46    directory_offset: usize,
47    chunk_count: usize,
48    directory_sorted: bool,
49    lookup_work_bound: usize,
50    extension: Option<super::extension::EmfExtensionView<'a>>,
51}
52
53impl<'a> EmfContainerReader<'a> {
54    /// Parse under conservative 16 MiB / 4096-chunk budgets.
55    pub fn parse(bytes: &'a [u8]) -> Result<Self, EmfError> {
56        Self::parse_with_limits(bytes, EmfReadLimits::conservative())
57    }
58
59    /// Parse and validate an EMF container under explicit file/chunk budgets.
60    pub fn parse_with_limits(bytes: &'a [u8], limits: EmfReadLimits) -> Result<Self, EmfError> {
61        Self::parse_policy(bytes, limits, false, super::extension::EmfExtensionLimits {max_file_bytes:limits.max_file_bytes,..super::extension::EmfExtensionLimits::conservative()})
62    }
63
64    /// Migration-only admission of v1 files with defective truncated integrity.
65    /// A successful call does not certify a full digest for a legacy file.
66    pub fn parse_legacy_with_limits(bytes: &'a [u8], limits: EmfReadLimits) -> Result<Self, EmfError> {
67        Self::parse_policy(bytes, limits, true, super::extension::EmfExtensionLimits {max_file_bytes:limits.max_file_bytes,..super::extension::EmfExtensionLimits::conservative()})
68    }
69
70    /// Current-format reader with independently bounded metadata/PCM admission.
71    pub fn parse_with_extension_limits(bytes:&'a[u8],limits:EmfReadLimits,extension_limits:super::extension::EmfExtensionLimits)->Result<Self,EmfError>{
72        Self::parse_policy(bytes,limits,false,extension_limits)
73    }
74
75    fn parse_policy(bytes: &'a [u8], limits: EmfReadLimits, allow_legacy: bool, extension_limits:super::extension::EmfExtensionLimits) -> Result<Self, EmfError> {
76        if !cfg!(target_endian = "little") {
77            return Err(EmfError::InvalidMagic);
78        }
79        if bytes.len() < 64 {
80            return Err(EmfError::BufferTooSmall {
81                required: 64,
82                provided: bytes.len(),
83            });
84        }
85        if bytes.len() > limits.max_file_bytes {
86            return Err(EmfError::FileByteLimit);
87        }
88
89        let header: EmfContainerHeader = bytemuck::pod_read_unaligned(&bytes[..64]);
90        header.validate()?;
91        if header.version == super::header::EMF_LEGACY_VERSION && !allow_legacy {
92            return Err(EmfError::LegacyIntegrityRequiresOptIn);
93        }
94        let full_hash = matches!(header.version,EMF_CONTAINER_VERSION|EMF_EXTENDED_VERSION);
95        let extension = if header.version==EMF_EXTENDED_VERSION {
96            if header.flags & !7 != 0 {return Err(EmfError::ExtensionMetadata);}
97            Some(super::extension::parse_extension(bytes,extension_limits)?)
98        } else {None};
99        let field_end=extension.as_ref().map(|e|e.field_end).unwrap_or(bytes.len());
100        if full_hash && header.flags & EMF_FLAG_BAO_VERIFIED != 0 {
101            return Err(EmfError::InvalidIntegrityEncoding { chunk_index: 0 });
102        }
103
104        let directory_offset = usize::try_from(header.directory_offset)
105            .map_err(|_| EmfError::ArithmeticOverflow)?;
106        let chunk_count = usize::try_from(header.chunk_count)
107            .map_err(|_| EmfError::ArithmeticOverflow)?;
108
109        if chunk_count > limits.max_chunks {
110            return Err(EmfError::ChunkLimitExceeded);
111        }
112        if directory_offset < 64 || directory_offset % 64 != 0 {
113            return Err(if directory_offset % 64 != 0 {
114                EmfError::DirectoryAlignment
115            } else {
116                EmfError::DirectoryBounds
117            });
118        }
119        let directory_bytes = chunk_count
120            .checked_mul(CHUNK_DIRECTORY_ENTRY_BYTES)
121            .ok_or(EmfError::ArithmeticOverflow)?;
122        let directory_end = directory_offset
123            .checked_add(directory_bytes)
124            .ok_or(EmfError::ArithmeticOverflow)?;
125
126        if directory_end > field_end {
127            return Err(EmfError::DirectoryBounds);
128        }
129
130        let mut directory_sorted=true;let mut previous_key=None;let mut equal_run=0usize;let mut max_equal_run=0usize;
131        for chunk in 0..chunk_count {
132            let offset = directory_offset + chunk * CHUNK_DIRECTORY_ENTRY_BYTES;
133            let entry: EmfChunkDirectoryEntry =
134                bytemuck::pod_read_unaligned(&bytes[offset..offset + CHUNK_DIRECTORY_ENTRY_BYTES]);
135            entry.validate()?;
136            if previous_key.is_some_and(|key|key>entry.spatial_hilbert_index){directory_sorted=false;}
137            equal_run=if previous_key==Some(entry.spatial_hilbert_index){equal_run+1}else{1};
138            max_equal_run=max_equal_run.max(equal_run);previous_key=Some(entry.spatial_hilbert_index);
139
140            let start = usize::try_from(entry.byte_offset).map_err(|_| EmfError::ArithmeticOverflow)?;
141            let length = usize::try_from(entry.byte_length).map_err(|_| EmfError::ArithmeticOverflow)?;
142            let end = start.checked_add(length).ok_or(EmfError::ArithmeticOverflow)?;
143
144            if start % 64 != 0 {
145                return Err(EmfError::PayloadUnaligned);
146            }
147            if end > field_end {
148                return Err(EmfError::PayloadOutOfBounds);
149            }
150            if start < 64 || ranges_overlap(start, end, directory_offset, directory_end) {
151                return Err(EmfError::PayloadOverlap);
152            }
153            let expected_bytes = usize::try_from(entry.cell_count)
154                .map_err(|_| EmfError::ArithmeticOverflow)?
155                .checked_mul(64)
156                .ok_or(EmfError::ArithmeticOverflow)?;
157            let stored_bytes = expected_bytes.checked_add(if full_hash { BLAKE3_TRAILER_BYTES } else { 0 })
158                .ok_or(EmfError::ArithmeticOverflow)?;
159            if length != stored_bytes {
160                return Err(EmfError::PayloadOutOfBounds);
161            }
162
163            for prev in 0..chunk {
164                let prev_off = directory_offset + prev * CHUNK_DIRECTORY_ENTRY_BYTES;
165                let prev_entry: EmfChunkDirectoryEntry = bytemuck::pod_read_unaligned(
166                    &bytes[prev_off..prev_off + CHUNK_DIRECTORY_ENTRY_BYTES],
167                );
168                let prev_start = usize::try_from(prev_entry.byte_offset)
169                    .map_err(|_| EmfError::ArithmeticOverflow)?;
170                let prev_len = usize::try_from(prev_entry.byte_length)
171                    .map_err(|_| EmfError::ArithmeticOverflow)?;
172                let prev_end = prev_start
173                    .checked_add(prev_len)
174                    .ok_or(EmfError::ArithmeticOverflow)?;
175                if ranges_overlap(start, end, prev_start, prev_end) {
176                    return Err(EmfError::PayloadOverlap);
177                }
178            }
179
180            if full_hash {
181                if entry.flags & EMF_CHUNK_FLAG_BLAKE3_FULL == 0 || entry.flags & 2 != 0
182                    || entry.bao_hash_prefix != [0; 16] || entry.reserved != [0; 12] {
183                    return Err(EmfError::InvalidIntegrityEncoding { chunk_index: chunk });
184                }
185                let records_end = start + expected_bytes;
186                if !verify_blake3_trailer(&bytes[start..records_end], &bytes[records_end..end]) {
187                    return Err(EmfError::FullHashMismatch { chunk_index: chunk });
188                }
189            } else if (entry.flags & 2 != 0) || (entry.bao_hash_prefix != [0u8; 16]) {
190                // Legacy migration only: reproduce the defective comparison, never call it Bao.
191                let payload = &bytes[start..end];
192                let digest = blake3::hash(payload);
193                if digest.as_bytes()[..16] != entry.bao_hash_prefix {
194                    return Err(EmfError::BaoHashMismatch { chunk_index: chunk });
195                }
196            }
197        }
198
199        let lookup_work_bound=if directory_sorted{(usize::BITS-chunk_count.leading_zeros()) as usize+max_equal_run+1}else{chunk_count}.max(1);
200        Ok(Self {
201            bytes,
202            header,
203            directory_offset,
204            chunk_count,
205            directory_sorted,
206            lookup_work_bound,
207            extension,
208        }).and_then(|reader| {
209            if let Some(extension)=&reader.extension {
210                super::extension::validate_field_directory(&reader.header,reader.directory_offset,reader.chunk_count,reader.bytes,extension)?;
211            }
212            Ok(reader)
213        })
214    }
215
216    /// Access the parsed container header.
217    #[inline]
218    pub fn header(&self) -> &EmfContainerHeader {
219        &self.header
220    }
221
222    /// Typed v3 interpretation and verified original PCM; absent for repaired v2.
223    pub fn extension(&self)->Option<&super::extension::EmfExtensionView<'a>>{self.extension.as_ref()}
224
225    /// Total number of chunks / bricks in this container.
226    #[inline]
227    pub fn chunk_count(&self) -> usize {
228        self.chunk_count
229    }
230
231    /// Read a chunk directory entry by index.
232    pub fn chunk_entry(&self, index: usize) -> Option<EmfChunkDirectoryEntry> {
233        if index >= self.chunk_count {
234            return None;
235        }
236        let offset = self.directory_offset + index * CHUNK_DIRECTORY_ENTRY_BYTES;
237        Some(bytemuck::pod_read_unaligned(
238            &self.bytes[offset..offset + CHUNK_DIRECTORY_ENTRY_BYTES],
239        ))
240    }
241
242    /// Borrow the raw byte payload of a chunk.
243    pub fn chunk_payload(&self, index: usize) -> Option<&'a [u8]> {
244        let entry = self.chunk_entry(index)?;
245        let start = entry.byte_offset as usize;
246        let end = start + entry.cell_count as usize * 64;
247        self.bytes.get(start..end)
248    }
249
250    /// Read a specific 64B cell from a chunk.
251    pub fn cell(&self, chunk_index: usize, cell_index: usize) -> Option<EmfCellRecord64> {
252        let payload = self.chunk_payload(chunk_index)?;
253        let start = cell_index.checked_mul(64)?;
254        let raw = payload.get(start..start.checked_add(64)?)?;
255        Some(bytemuck::pod_read_unaligned(raw))
256    }
257
258    /// Legacy spatial-only lookup deterministically selects time index zero.
259    /// Use find_chunk_at_time for another stored snapshot.
260    pub fn find_chunk_by_hilbert(&self,hilbert_key:u64)->Option<usize>{self.find_chunk_at_time(hilbert_key,0)}
261
262    /// Exact spatial/time lookup. V3 canonical directories use a lower-bound
263    /// binary search then scan only equal spatial keys; unsorted v2 is bounded linear.
264    pub fn find_chunk_at_time(&self,hilbert_key:u64,time_slice:u32)->Option<usize>{
265        if !self.directory_sorted {
266            return (0..self.chunk_count).find(|&index|self.chunk_entry(index).is_some_and(|entry|entry.spatial_hilbert_index==hilbert_key&&entry.time_slice_index==time_slice));
267        }
268        let mut low=0usize;let mut high=self.chunk_count;
269        while low<high {let middle=low+(high-low)/2;if self.chunk_entry(middle)?.spatial_hilbert_index<hilbert_key{low=middle+1;}else{high=middle;}}
270        while low<self.chunk_count {
271            let entry=self.chunk_entry(low)?;if entry.spatial_hilbert_index!=hilbert_key{return None;}
272            if entry.time_slice_index==time_slice{return Some(low);}low+=1;
273        }None
274    }
275
276    /// Conservative directory visits per exact lookup, for caller work admission.
277    pub fn lookup_work_bound(&self)->usize{self.lookup_work_bound}
278
279    /// Direct lookup of a voxel cell using brick coordinates `(bx, by, bz)` and local coords `(lx, ly, lz)`.
280    pub fn lookup_brick_cell(&self,bx:u32,by:u32,bz:u32,lx:u8,ly:u8,lz:u8,order:u32)->Option<EmfCellRecord64>{
281        self.lookup_brick_cell_at_time(bx,by,bz,lx,ly,lz,order,0)
282    }
283
284    pub fn lookup_brick_cell_at_time(&self,bx:u32,by:u32,bz:u32,lx:u8,ly:u8,lz:u8,order:u32,time_slice:u32)->Option<EmfCellRecord64>{
285        if lx>=8||ly>=8||lz>=8||order==0||order>21{return None;}
286        if bx>=(1u32<<order)||by>=(1u32<<order)||bz>=(1u32<<order){return None;}
287        let hilbert_key=encode_hilbert3d_u64(bx,by,bz,order);
288        let chunk=self.find_chunk_at_time(hilbert_key,time_slice)?;
289        self.cell(chunk,emf_brick_voxel_linear_index(lx,ly,lz))
290    }
291
292    /// Unpack Hilbert 8³ bricks into a dense `[t][z][y][x]` grid of scientific cells.
293    ///
294    /// This decodes physical fields only. V3 independent PCM and interpretation
295    /// remain available through extension(); they must not be inferred from sigma.
296    /// `.10d` MID2/FIEL names the field; it does not duplicate these cells.
297    pub fn dense_volume(&self) -> Result<DenseEmfVolume, EmfError> {
298        let dims = self.header.grid_dims_bricks;
299        if dims[0] == 0 || dims[1] == 0 || dims[2] == 0 {
300            return Err(EmfError::ChunkLimitExceeded);
301        }
302        let side = EMF_BRICK_SIDE as u32;
303        let nx = dims[0]
304            .checked_mul(side)
305            .ok_or(EmfError::ArithmeticOverflow)?;
306        let ny = dims[1]
307            .checked_mul(side)
308            .ok_or(EmfError::ArithmeticOverflow)?;
309        let nz = dims[2]
310            .checked_mul(side)
311            .ok_or(EmfError::ArithmeticOverflow)?;
312
313        let mut max_t = 0u32;
314        for i in 0..self.chunk_count {
315            let entry = self.chunk_entry(i).ok_or(EmfError::DirectoryBounds)?;
316            max_t = max_t.max(entry.time_slice_index);
317        }
318        let nt = max_t
319            .checked_add(1)
320            .ok_or(EmfError::ArithmeticOverflow)?;
321
322        let cell_count = (nx as usize)
323            .checked_mul(ny as usize)
324            .and_then(|n| n.checked_mul(nz as usize))
325            .and_then(|n| n.checked_mul(nt as usize))
326            .ok_or(EmfError::ArithmeticOverflow)?;
327        if cell_count > MAX_DENSE_CELLS {
328            return Err(EmfError::DenseVolumeCap {
329                cells: cell_count,
330                cap: MAX_DENSE_CELLS,
331            });
332        }
333        let mut cells = vec![EmfCellRecord64::default(); cell_count];
334        let mut sigma_sum = 0.0f64;
335        let mut sigma_n = 0u64;
336        let order = match &self.extension {
337            Some(extension)=>u32::from(extension.metadata().field.hilbert_order),
338            None=>infer_container_hilbert_order(self,dims)?,
339        };
340
341        for i in 0..self.chunk_count {
342            let entry = self.chunk_entry(i).ok_or(EmfError::DirectoryBounds)?;
343            let (bx, by, bz) = decode_hilbert3d_u64(entry.spatial_hilbert_index, order);
344            if bx >= dims[0] || by >= dims[1] || bz >= dims[2] {
345                return Err(EmfError::HilbertOrder);
346            }
347            let t = entry.time_slice_index;
348            let n_cells = usize::try_from(entry.cell_count).map_err(|_| EmfError::ArithmeticOverflow)?;
349            for vi in 0..n_cells {
350                let cell = self.cell(i, vi).ok_or(EmfError::PayloadOutOfBounds)?;
351                let (lx, ly, lz) = emf_brick_voxel_coords(vi);
352                let x = bx
353                    .checked_mul(side)
354                    .and_then(|v| v.checked_add(lx as u32))
355                    .ok_or(EmfError::ArithmeticOverflow)?;
356                let y = by
357                    .checked_mul(side)
358                    .and_then(|v| v.checked_add(ly as u32))
359                    .ok_or(EmfError::ArithmeticOverflow)?;
360                let z = bz
361                    .checked_mul(side)
362                    .and_then(|v| v.checked_add(lz as u32))
363                    .ok_or(EmfError::ArithmeticOverflow)?;
364                if x >= nx || y >= ny || z >= nz {
365                    return Err(EmfError::PayloadOutOfBounds);
366                }
367                let idx = dense_cell_index(x, y, z, t, nx, ny, nz)?;
368                cells[idx] = cell;
369                if cell.e_magnitude() > 0.0
370                    || cell.wavelength_nm > 0.0
371                    || cell.spectral_sigma != 0.0
372                {
373                    sigma_sum += cell.effective_sigma() as f64;
374                    sigma_n += 1;
375                }
376            }
377        }
378
379        let mean_sigma = if sigma_n > 0 {
380            (sigma_sum / sigma_n as f64) as f32
381        } else {
382            0.0
383        };
384
385        Ok(DenseEmfVolume {
386            origin_meters: self.header.origin_meters,
387            voxel_size_meters: self.header.voxel_size_meters,
388            nx,
389            ny,
390            nz,
391            nt,
392            cells,
393            mean_sigma,
394        })
395    }
396}
397
398/// Dense `[t][z][y][x]` unpack of a `Q42EMF64` container.
399#[derive(Clone, Debug)]
400pub struct DenseEmfVolume {
401    pub origin_meters: [f32; 3],
402    pub voxel_size_meters: f32,
403    pub nx: u32,
404    pub ny: u32,
405    pub nz: u32,
406    pub nt: u32,
407    pub cells: Vec<EmfCellRecord64>,
408    pub mean_sigma: f32,
409}
410
411impl DenseEmfVolume {
412    /// Axis-aligned bounds in metres for GPU slice uniforms.
413    pub fn bounds_meters(&self) -> [f32; 6] {
414        let ox = self.origin_meters[0];
415        let oy = self.origin_meters[1];
416        let oz = self.origin_meters[2];
417        let vx = self.voxel_size_meters * self.nx as f32;
418        let vy = self.voxel_size_meters * self.ny as f32;
419        let vz = self.voxel_size_meters * self.nz as f32;
420        [ox, ox + vx, oy, oy + vy, oz, oz + vz]
421    }
422}
423
424fn dense_cell_index(
425    x: u32,
426    y: u32,
427    z: u32,
428    t: u32,
429    nx: u32,
430    ny: u32,
431    nz: u32,
432) -> Result<usize, EmfError> {
433    let plane = (nx as usize)
434        .checked_mul(ny as usize)
435        .ok_or(EmfError::ArithmeticOverflow)?;
436    let volume = plane
437        .checked_mul(nz as usize)
438        .ok_or(EmfError::ArithmeticOverflow)?;
439    let t_off = (t as usize)
440        .checked_mul(volume)
441        .ok_or(EmfError::ArithmeticOverflow)?;
442    let z_off = (z as usize)
443        .checked_mul(plane)
444        .ok_or(EmfError::ArithmeticOverflow)?;
445    let y_off = (y as usize)
446        .checked_mul(nx as usize)
447        .ok_or(EmfError::ArithmeticOverflow)?;
448    t_off
449        .checked_add(z_off)
450        .and_then(|n| n.checked_add(y_off))
451        .and_then(|n| n.checked_add(x as usize))
452        .ok_or(EmfError::ArithmeticOverflow)
453}
454
455fn infer_container_hilbert_order(
456    reader: &EmfContainerReader<'_>,
457    dims: [u32; 3],
458) -> Result<u32, EmfError> {
459    if reader.chunk_count == 0 {
460        return Ok(2);
461    }
462    // Writers: compile_emf uses 2, anatomy seal uses 3. Prefer those before
463    // a lower order that can accidentally round-trip a subset of keys.
464    for order in [2, 3, 1, 4, 5, 6, 7, 8] {
465        let mut seen = std::collections::BTreeSet::new();
466        let mut ok = true;
467        for i in 0..reader.chunk_count {
468            let entry = reader.chunk_entry(i).ok_or(EmfError::DirectoryBounds)?;
469            let (bx, by, bz) = decode_hilbert3d_u64(entry.spatial_hilbert_index, order);
470            if bx >= dims[0]
471                || by >= dims[1]
472                || bz >= dims[2]
473                || encode_hilbert3d_u64(bx, by, bz, order) != entry.spatial_hilbert_index
474                || !seen.insert((bx, by, bz, entry.time_slice_index))
475            {
476                ok = false;
477                break;
478            }
479        }
480        if ok {
481            return Ok(order);
482        }
483    }
484    Err(EmfError::HilbertOrder)
485}