Skip to main content

qualia_core_db/q42/m64_matter/
checked_reader.rs

1//! Bounded structural reader for little-endian M64/E64 containers.
2//!
3//! Header and directory PODs are copied to aligned stack values; payload bytes
4//! remain borrowed zero-copy slices. Supports full BLAKE3 digests; truncated legacy hashes require migration opt-in.
5
6use crate::q42::chunk_integrity::{verify_blake3_trailer, BLAKE3_TRAILER_BYTES};
7use super::M64_CODEC_BLAKE3_FULL;
8
9use super::{
10    M64BioTissueRecord, M64CellMode, M64ChunkDirectoryEntry, M64Error, M64GeoRasterError,
11    M64GeoRasterMetadata, M64Header, M64HoloRecord, M64LocalMetricHeader,
12    M64PhaseJunctureRecord, M64PointRecord, M64RasterSample, M64ShellRecord,
13    M64StrataColumnRecord, LRM3_BLOCK_BYTES, M64_FLAG_3D_BRICK_16X16X16,
14    M64_FLAG_LOCAL_METRIC_LRM3, M64_MAGIC, M64_MODE_BIO_TISSUE, M64_MODE_HOLO,
15    M64_MODE_PHASE_JUNCTURE, M64_MODE_POINT, M64_MODE_SHELL, M64_MODE_STRATA,
16};
17
18#[cfg(test)]
19use super::E64_MAGIC;
20
21/// Explicit admission budgets; there is no implicit unlimited/default policy.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub struct M64ReadLimits {
24    pub max_file_bytes: usize,
25    pub max_chunks: usize,
26    pub max_records: u64,
27}
28
29/// Caller-owned overlap scratch. One element per directory entry is required.
30#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
31pub struct M64PayloadInterval {
32    pub start: usize,
33    pub end: usize,
34}
35
36/// Verification metrics collected during structural validation.
37#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
38pub struct M64Verification {
39    pub chunks: usize,
40    pub total_records: u64,
41    pub record_crc_checked: u64,
42    pub full_hash_chunks: usize,
43    pub legacy_prefix_chunks: usize,
44    pub records_without_checksum: u64,
45}
46
47#[derive(Clone, Copy, Debug, PartialEq, Eq)]
48pub enum M64ReadError {
49    GeoRaster(M64GeoRasterError),
50    UnsupportedEndian,
51    FileByteLimit,
52    TruncatedHeader,
53    Header(M64Error),
54    Lrm3Missing,
55    Lrm3(M64Error),
56    UnsupportedVersion(u32),
57    UnsupportedContainerFlags(u32),
58    FileSizeMismatch,
59    ChunkLimit,
60    RecordLimit,
61    ScratchTooSmall { required: usize, provided: usize },
62    ArithmeticOverflow,
63    DirectoryBounds,
64    DirectoryAlignment,
65    Directory { chunk: usize, error: M64Error },
66    UnsupportedMode { chunk: usize, mode: u8 },
67    UnsupportedCodec { chunk: usize, codec: u8 },
68    PayloadBounds { chunk: usize },
69    PayloadAlignment { chunk: usize },
70    RecordLength { chunk: usize },
71    MetadataOverlap { chunk: usize },
72    PayloadOverlap,
73    Record { chunk: usize, record: usize, error: M64Error },
74    BaoHashMismatch { chunk_index: usize },
75    FullHashMismatch { chunk_index: usize },
76    LegacyIntegrityRequiresOptIn { chunk_index: usize },
77}
78
79impl std::fmt::Display for M64ReadError {
80    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
81        write!(f, "{self:?}")
82    }
83}
84
85impl std::error::Error for M64ReadError {}
86
87fn sift_down(values: &mut [M64PayloadInterval], mut root: usize, end: usize) {
88    while root < end / 2 {
89        let mut child = root * 2 + 1;
90        if child + 1 < end && values[child].start < values[child + 1].start {
91            child += 1;
92        }
93        if values[root].start >= values[child].start {
94            break;
95        }
96        values.swap(root, child);
97        root = child;
98    }
99}
100
101fn has_overlap(values: &mut [M64PayloadInterval]) -> bool {
102    for root in (0..values.len() / 2).rev() {
103        sift_down(values, root, values.len());
104    }
105    for end in (1..values.len()).rev() {
106        values.swap(0, end);
107        sift_down(values, 0, end);
108    }
109    values.windows(2).any(|pair| pair[0].end > pair[1].start)
110}
111
112/// A fully structurally checked immutable file view.
113#[derive(Debug)]
114pub struct M64Reader<'a> {
115    bytes: &'a [u8],
116    header: M64Header,
117    directory_start: usize,
118    verification: M64Verification,
119    georaster: Option<M64GeoRasterMetadata>,
120    lrm3: Option<M64LocalMetricHeader>,
121}
122
123impl<'a> M64Reader<'a> {
124    /// Admit the entire file with bounded structural checks.
125    pub fn parse(
126        bytes: &'a [u8],
127        limits: M64ReadLimits,
128        scratch: &mut [M64PayloadInterval],
129    ) -> Result<Self, M64ReadError> {
130        Self::parse_policy(bytes, limits, scratch, false)
131    }
132
133    /// Migration-only admission of defective legacy truncated hashes.
134    /// `legacy_prefix_chunks` reports these separately from full verification.
135    pub fn parse_legacy_with_limits(
136        bytes: &'a [u8], limits: M64ReadLimits, scratch: &mut [M64PayloadInterval],
137    ) -> Result<Self, M64ReadError> {
138        Self::parse_policy(bytes, limits, scratch, true)
139    }
140
141    fn parse_policy(
142        bytes: &'a [u8], limits: M64ReadLimits, scratch: &mut [M64PayloadInterval],
143        allow_legacy: bool,
144    ) -> Result<Self, M64ReadError> {
145        if !cfg!(target_endian = "little") {
146            return Err(M64ReadError::UnsupportedEndian);
147        }
148        if bytes.len() > limits.max_file_bytes {
149            return Err(M64ReadError::FileByteLimit);
150        }
151        let raw_header = bytes.get(..64).ok_or(M64ReadError::TruncatedHeader)?;
152        let header: M64Header = bytemuck::pod_read_unaligned(raw_header);
153        if !matches!(header.version, 1 | 2 | 3) {
154            return Err(M64ReadError::UnsupportedVersion(header.version));
155        }
156        header.validate().map_err(M64ReadError::Header)?;
157
158        let admitted_flags =
159            (M64_FLAG_3D_BRICK_16X16X16 as u32) | (M64_FLAG_LOCAL_METRIC_LRM3 as u32);
160        if header.container_flags & !admitted_flags != 0 {
161            return Err(M64ReadError::UnsupportedContainerFlags(header.container_flags));
162        }
163        if header.total_file_size != bytes.len() as u64 {
164            return Err(M64ReadError::FileSizeMismatch);
165        }
166
167        let georaster = if header.version == 3 {
168            let meta = M64GeoRasterMetadata::decode(
169                bytes.get(64..192).ok_or(M64ReadError::GeoRaster(M64GeoRasterError::Truncated))?,
170            ).map_err(M64ReadError::GeoRaster)?;
171            let shape = meta.shape().map_err(M64ReadError::GeoRaster)?;
172            if header.magic != M64_MAGIC
173                || header.chunk_table_offset != 192
174                || header.chunk_count != shape.chunks as u64
175                || bytes.len() != shape.file_bytes
176            {
177                return Err(M64ReadError::GeoRaster(M64GeoRasterError::HeaderSummary));
178            }
179            Some(meta)
180        } else {
181            None
182        };
183
184        let count = usize::try_from(header.chunk_count).map_err(|_| M64ReadError::ArithmeticOverflow)?;
185        if count > limits.max_chunks {
186            return Err(M64ReadError::ChunkLimit);
187        }
188        if scratch.len() < count {
189            return Err(M64ReadError::ScratchTooSmall {
190                required: count,
191                provided: scratch.len(),
192            });
193        }
194        let directory_start =
195            usize::try_from(header.chunk_table_offset).map_err(|_| M64ReadError::ArithmeticOverflow)?;
196        if directory_start % 64 != 0 {
197            return Err(M64ReadError::DirectoryAlignment);
198        }
199        let directory_end = directory_start
200            .checked_add(count.checked_mul(64).ok_or(M64ReadError::ArithmeticOverflow)?)
201            .ok_or(M64ReadError::ArithmeticOverflow)?;
202        if directory_start < header.header_size as usize || directory_end > bytes.len() {
203            return Err(M64ReadError::DirectoryBounds);
204        }
205
206        let lrm3 = if header.container_flags & (M64_FLAG_LOCAL_METRIC_LRM3 as u32) != 0 {
207            let header_size = usize::try_from(header.header_size)
208                .map_err(|_| M64ReadError::ArithmeticOverflow)?;
209            let lrm3_end = header_size
210                .checked_add(LRM3_BLOCK_BYTES)
211                .ok_or(M64ReadError::ArithmeticOverflow)?;
212            if lrm3_end > directory_start || lrm3_end > bytes.len() {
213                return Err(M64ReadError::Lrm3Missing);
214            }
215            let block: M64LocalMetricHeader =
216                bytemuck::pod_read_unaligned(&bytes[header_size..lrm3_end]);
217            block.validate().map_err(M64ReadError::Lrm3)?;
218            Some(block)
219        } else {
220            None
221        };
222
223        let mut verification = M64Verification {
224            chunks: count,
225            ..M64Verification::default()
226        };
227
228        for (chunk, interval) in scratch[..count].iter_mut().enumerate() {
229            let offset = directory_start + chunk * 64;
230            let entry: M64ChunkDirectoryEntry =
231                bytemuck::pod_read_unaligned(&bytes[offset..offset + 64]);
232            entry
233                .validate()
234                .map_err(|error| M64ReadError::Directory { chunk, error })?;
235
236            if !matches!(
237                entry.chunk_record_mode,
238                M64_MODE_POINT
239                    | M64_MODE_HOLO
240                    | M64_MODE_STRATA
241                    | M64_MODE_PHASE_JUNCTURE
242                    | M64_MODE_SHELL
243                    | M64_MODE_BIO_TISSUE
244            ) {
245                return Err(M64ReadError::UnsupportedMode {
246                    chunk,
247                    mode: entry.chunk_record_mode,
248                });
249            }
250            // Codec 2 is legacy migration only; codec 3 verifies all 256 digest bits.
251            if !matches!(entry.compression_flags, 0 | 2 | M64_CODEC_BLAKE3_FULL) {
252                return Err(M64ReadError::UnsupportedCodec {
253                    chunk,
254                    codec: entry.compression_flags,
255                });
256            }
257
258            let expected_length = u64::from(entry.record_count)
259                .checked_mul(64)
260                .ok_or(M64ReadError::ArithmeticOverflow)?;
261            let full_hash = entry.compression_flags == M64_CODEC_BLAKE3_FULL;
262            let stored_length = expected_length.checked_add(if full_hash { BLAKE3_TRAILER_BYTES as u64 } else { 0 })
263                .ok_or(M64ReadError::ArithmeticOverflow)?;
264            if entry.record_count == 0 || stored_length != u64::from(entry.byte_length) {
265                return Err(M64ReadError::RecordLength { chunk });
266            }
267            verification.total_records = verification
268                .total_records
269                .checked_add(u64::from(entry.record_count))
270                .ok_or(M64ReadError::ArithmeticOverflow)?;
271            if verification.total_records > limits.max_records {
272                return Err(M64ReadError::RecordLimit);
273            }
274
275            let start = usize::try_from(entry.byte_offset).map_err(|_| M64ReadError::ArithmeticOverflow)?;
276            let end = start
277                .checked_add(entry.byte_length as usize)
278                .ok_or(M64ReadError::ArithmeticOverflow)?;
279            if start % 64 != 0 {
280                return Err(M64ReadError::PayloadAlignment { chunk });
281            }
282            if end > bytes.len() {
283                return Err(M64ReadError::PayloadBounds { chunk });
284            }
285            // LRM3 lives in [header_size, header_size+LRM3_BLOCK_BYTES). CRC covers those
286            // bytes, but a payload alias of the authenticated gap is still overlap.
287            let header_end = header.header_size as usize;
288            let metadata_end = if lrm3.is_some() {
289                header_end + LRM3_BLOCK_BYTES
290            } else {
291                header_end
292            };
293            if start < metadata_end || (start < directory_end && end > directory_start) {
294                return Err(M64ReadError::MetadataOverlap { chunk });
295            }
296
297            if full_hash {
298                let record_end = end - BLAKE3_TRAILER_BYTES;
299                if entry.bao_root_hash_prefix != [0; 12]
300                    || !verify_blake3_trailer(&bytes[start..record_end], &bytes[record_end..end]) {
301                    return Err(M64ReadError::FullHashMismatch { chunk_index: chunk });
302                }
303                verification.full_hash_chunks += 1;
304            } else if entry.compression_flags == 2 || entry.bao_root_hash_prefix != [0u8; 12] {
305                if !allow_legacy {
306                    return Err(M64ReadError::LegacyIntegrityRequiresOptIn { chunk_index: chunk });
307                }
308                verification.legacy_prefix_chunks += 1;
309                // Defective historical comparison retained ONLY for explicit migration.
310                let chunk_payload = &bytes[start..end];
311                let digest = blake3::hash(chunk_payload);
312                if digest.as_bytes()[..12] != entry.bao_root_hash_prefix {
313                    return Err(M64ReadError::BaoHashMismatch { chunk_index: chunk });
314                }
315            }
316
317            *interval = M64PayloadInterval { start, end };
318        }
319
320        if has_overlap(&mut scratch[..count]) {
321            return Err(M64ReadError::PayloadOverlap);
322        }
323
324        for chunk in 0..count {
325            let offset = directory_start + chunk * 64;
326            let entry: M64ChunkDirectoryEntry =
327                bytemuck::pod_read_unaligned(&bytes[offset..offset + 64]);
328            if matches!(entry.chunk_record_mode, M64_MODE_POINT | M64_MODE_HOLO) {
329                verification.records_without_checksum += u64::from(entry.record_count);
330                continue;
331            }
332            let start = entry.byte_offset as usize;
333            let end = start + entry.record_count as usize * 64;
334            for (record, raw) in bytes[start..end].chunks_exact(64).enumerate() {
335                let result = match entry.chunk_record_mode {
336                    M64_MODE_STRATA => {
337                        bytemuck::pod_read_unaligned::<M64StrataColumnRecord>(raw).validate()
338                    }
339                    M64_MODE_SHELL => {
340                        bytemuck::pod_read_unaligned::<M64ShellRecord>(raw).validate()
341                    }
342                    M64_MODE_PHASE_JUNCTURE => {
343                        bytemuck::pod_read_unaligned::<M64PhaseJunctureRecord>(raw).validate()
344                    }
345                    M64_MODE_BIO_TISSUE => {
346                        bytemuck::pod_read_unaligned::<M64BioTissueRecord>(raw).validate()
347                    }
348                    M64_MODE_POINT | M64_MODE_HOLO => continue,
349                    mode => {
350                        return Err(M64ReadError::UnsupportedMode { chunk, mode });
351                    }
352                };
353                result.map_err(|error| M64ReadError::Record { chunk, record, error })?;
354                verification.record_crc_checked += 1;
355            }
356        }
357
358        if let Some(meta) = &georaster {
359            super::georaster::validate_container(bytes, &header, meta)
360                .map_err(M64ReadError::GeoRaster)?;
361        }
362
363        Ok(M64Reader {
364            bytes,
365            header,
366            directory_start,
367            verification,
368            georaster,
369            lrm3,
370        })
371    }
372
373    pub fn header(&self) -> &M64Header {
374        &self.header
375    }
376
377    pub fn georaster(&self) -> Option<&M64GeoRasterMetadata> {
378        self.georaster.as_ref()
379    }
380
381    pub fn lrm3(&self) -> Option<&M64LocalMetricHeader> {
382        self.lrm3.as_ref()
383    }
384
385    pub fn raster_sample(&self, x: usize, y: usize) -> Option<M64RasterSample> {
386        let meta = self.georaster.as_ref()?;
387        let shape = meta.shape().ok()?;
388        if x >= shape.width || y >= shape.height {
389            return None;
390        }
391        let chunk = self.chunk((y / 64) * shape.chunks_x + x / 64)?;
392        let raw = chunk.record_bytes((y % 64) * 64 + x % 64)?;
393        let value = i32::from_le_bytes(raw[..4].try_into().ok()?);
394        Some(if meta.nodata_sentinel_mm == Some(value) {
395            M64RasterSample::Nodata(value)
396        } else {
397            M64RasterSample::ElevationMm(value)
398        })
399    }
400
401    pub fn verification(&self) -> M64Verification {
402        self.verification
403    }
404
405    pub fn chunk(&self, index: usize) -> Option<M64Chunk<'a>> {
406        if index >= self.verification.chunks {
407            return None;
408        }
409        let offset = self.directory_start + index * 64;
410        let entry: M64ChunkDirectoryEntry =
411            bytemuck::pod_read_unaligned(&self.bytes[offset..offset + 64]);
412        let start = entry.byte_offset as usize;
413        let end = start + entry.record_count as usize * 64;
414        Some(M64Chunk {
415            entry,
416            payload: &self.bytes[start..end],
417        })
418    }
419}
420
421/// Typed 64-byte M64 record decoded from a validated chunk.
422#[derive(Clone, Copy, Debug, PartialEq)]
423pub enum M64DecodedRecord {
424    Point(M64PointRecord),
425    Holo(M64HoloRecord),
426    Strata(M64StrataColumnRecord),
427    PhaseJuncture(M64PhaseJunctureRecord),
428    Shell(M64ShellRecord),
429    BioTissue(M64BioTissueRecord),
430}
431
432#[derive(Debug)]
433pub struct M64Chunk<'a> {
434    entry: M64ChunkDirectoryEntry,
435    payload: &'a [u8],
436}
437
438impl<'a> M64Chunk<'a> {
439    pub fn directory_entry(&self) -> &M64ChunkDirectoryEntry {
440        &self.entry
441    }
442
443    pub fn payload(&self) -> &'a [u8] {
444        self.payload
445    }
446
447    pub fn record_bytes(&self, index: usize) -> Option<&'a [u8]> {
448        let start = index.checked_mul(64)?;
449        self.payload.get(start..start.checked_add(64)?)
450    }
451
452    /// Dispatch a chunk record by `M64CellMode` into `M64DecodedRecord`, verifying per-record CRC-32C.
453    pub fn decode_record(&self, index: usize) -> Result<Option<M64DecodedRecord>, M64ReadError> {
454        let Some(raw) = self.record_bytes(index) else {
455            return Ok(None);
456        };
457        let mode = M64CellMode::try_from(self.entry.chunk_record_mode).map_err(|_| {
458            M64ReadError::UnsupportedMode {
459                chunk: 0,
460                mode: self.entry.chunk_record_mode,
461            }
462        })?;
463        let record = match mode {
464            M64CellMode::Point => M64DecodedRecord::Point(bytemuck::pod_read_unaligned(raw)),
465            M64CellMode::Holo => M64DecodedRecord::Holo(bytemuck::pod_read_unaligned(raw)),
466            M64CellMode::Strata => {
467                let rec: M64StrataColumnRecord = bytemuck::pod_read_unaligned(raw);
468                rec.validate().map_err(|error| M64ReadError::Record {
469                    chunk: 0,
470                    record: index,
471                    error,
472                })?;
473                M64DecodedRecord::Strata(rec)
474            }
475            M64CellMode::PhaseJuncture => {
476                let rec: M64PhaseJunctureRecord = bytemuck::pod_read_unaligned(raw);
477                rec.validate().map_err(|error| M64ReadError::Record {
478                    chunk: 0,
479                    record: index,
480                    error,
481                })?;
482                M64DecodedRecord::PhaseJuncture(rec)
483            }
484            M64CellMode::Shell => {
485                let rec: M64ShellRecord = bytemuck::pod_read_unaligned(raw);
486                rec.validate().map_err(|error| M64ReadError::Record {
487                    chunk: 0,
488                    record: index,
489                    error,
490                })?;
491                M64DecodedRecord::Shell(rec)
492            }
493            M64CellMode::BioTissue => {
494                let rec: M64BioTissueRecord = bytemuck::pod_read_unaligned(raw);
495                rec.validate().map_err(|error| M64ReadError::Record {
496                    chunk: 0,
497                    record: index,
498                    error,
499                })?;
500                M64DecodedRecord::BioTissue(rec)
501            }
502        };
503        Ok(Some(record))
504    }
505}
506
507#[cfg(test)]
508#[path = "checked_reader_tests.rs"]
509mod tests;
510
511#[cfg(test)]
512mod bio_tissue_reader_tests {
513    use super::*;
514    use crate::q42::m64_matter::bio_tissue_record::TISSUE_CLASS_MYOCARDIUM;
515
516    fn bio_container(payload: &[u8; 64]) -> Vec<u8> {
517        let mut header = M64Header::default();
518        header.total_file_size = 192;
519        header.chunk_count = 1;
520        header.chunk_table_offset = 64;
521        header.update_crc();
522        let mut entry = M64ChunkDirectoryEntry {
523            chunk_uuid: [0; 16],
524            spatial_hilbert_index: 0,
525            byte_offset: 128,
526            byte_length: 64,
527            record_count: 1,
528            srid_geodetic_code: 0,
529            lut_bank_epoch_id: 0,
530            chunk_record_mode: M64CellMode::BioTissue.as_u8(),
531            compression_flags: 0,
532            bao_root_hash_prefix: [0u8; 12],
533            entry_crc32c: 0,
534        };
535        entry.update_crc();
536        let mut buf = vec![0u8; 192];
537        buf[..64].copy_from_slice(bytemuck::bytes_of(&header));
538        buf[64..128].copy_from_slice(bytemuck::bytes_of(&entry));
539        buf[128..192].copy_from_slice(payload);
540        buf
541    }
542
543    #[test]
544    fn dispatches_mode_0x05_bio_tissue_and_rejects_corruption() {
545        let mut rec = M64BioTissueRecord::default();
546        rec.tissue_class_id = TISSUE_CLASS_MYOCARDIUM;
547        rec.apparent_density_kg_m3 = 1060;
548        rec.set_elastic_moduli_mpa(0.08, 0.04, 0.04, 0.02);
549        rec.update_crc();
550
551        let mut buf = bio_container(bytemuck::cast_ref(&rec));
552        let limits = M64ReadLimits {
553            max_file_bytes: 1024,
554            max_chunks: 4,
555            max_records: 4,
556        };
557        let mut scratch = [M64PayloadInterval::default(); 4];
558        let reader = M64Reader::parse(&buf, limits, &mut scratch).expect("bio tissue parse");
559        assert_eq!(reader.verification().record_crc_checked, 1);
560
561        let chunk = reader.chunk(0).expect("chunk 0");
562        assert_eq!(
563            chunk.decode_record(0).expect("decode"),
564            Some(M64DecodedRecord::BioTissue(rec))
565        );
566        assert_eq!(chunk.decode_record(1).expect("oob"), None);
567
568        buf[128] ^= 0x01;
569        assert!(matches!(
570            M64Reader::parse(&buf, limits, &mut scratch),
571            Err(M64ReadError::Record {
572                chunk: 0,
573                record: 0,
574                error: M64Error::ChecksumMismatch { .. },
575            })
576        ));
577    }
578}