1use super::chunk_directory::{
7 emf_brick_voxel_coords, emf_brick_voxel_linear_index, EmfChunkDirectoryEntry,
8 CHUNK_DIRECTORY_ENTRY_BYTES, EMF_BRICK_SIDE,
9};
10use super::header::{EmfCellRecord64, EmfContainerHeader, EmfError, MAX_DENSE_CELLS};
11use super::header::{EMF_EXTENDED_VERSION, EMF_CONTAINER_VERSION, EMF_CHUNK_FLAG_BLAKE3_FULL, EMF_FLAG_BAO_VERIFIED};
12use crate::q42::chunk_integrity::{verify_blake3_trailer, BLAKE3_TRAILER_BYTES};
13use crate::q42::m64_matter::brick3d::{decode_hilbert3d_u64, encode_hilbert3d_u64};
14
15#[derive(Clone, Copy, Debug, PartialEq, Eq)]
17pub struct EmfReadLimits {
18 pub max_file_bytes: usize,
19 pub max_chunks: usize,
20}
21
22impl EmfReadLimits {
23 pub const fn conservative() -> Self {
29 Self {
30 max_file_bytes: 16 * 1024 * 1024,
31 max_chunks: 4096,
32 }
33 }
34}
35
36#[inline]
37fn ranges_overlap(a_start: usize, a_end: usize, b_start: usize, b_end: usize) -> bool {
38 a_start < b_end && b_start < a_end
39}
40
41#[derive(Debug)]
43pub struct EmfContainerReader<'a> {
44 bytes: &'a [u8],
45 header: EmfContainerHeader,
46 directory_offset: usize,
47 chunk_count: usize,
48 directory_sorted: bool,
49 lookup_work_bound: usize,
50 extension: Option<super::extension::EmfExtensionView<'a>>,
51}
52
53impl<'a> EmfContainerReader<'a> {
54 pub fn parse(bytes: &'a [u8]) -> Result<Self, EmfError> {
56 Self::parse_with_limits(bytes, EmfReadLimits::conservative())
57 }
58
59 pub fn parse_with_limits(bytes: &'a [u8], limits: EmfReadLimits) -> Result<Self, EmfError> {
61 Self::parse_policy(bytes, limits, false, super::extension::EmfExtensionLimits {max_file_bytes:limits.max_file_bytes,..super::extension::EmfExtensionLimits::conservative()})
62 }
63
64 pub fn parse_legacy_with_limits(bytes: &'a [u8], limits: EmfReadLimits) -> Result<Self, EmfError> {
67 Self::parse_policy(bytes, limits, true, super::extension::EmfExtensionLimits {max_file_bytes:limits.max_file_bytes,..super::extension::EmfExtensionLimits::conservative()})
68 }
69
70 pub fn parse_with_extension_limits(bytes:&'a[u8],limits:EmfReadLimits,extension_limits:super::extension::EmfExtensionLimits)->Result<Self,EmfError>{
72 Self::parse_policy(bytes,limits,false,extension_limits)
73 }
74
75 fn parse_policy(bytes: &'a [u8], limits: EmfReadLimits, allow_legacy: bool, extension_limits:super::extension::EmfExtensionLimits) -> Result<Self, EmfError> {
76 if !cfg!(target_endian = "little") {
77 return Err(EmfError::InvalidMagic);
78 }
79 if bytes.len() < 64 {
80 return Err(EmfError::BufferTooSmall {
81 required: 64,
82 provided: bytes.len(),
83 });
84 }
85 if bytes.len() > limits.max_file_bytes {
86 return Err(EmfError::FileByteLimit);
87 }
88
89 let header: EmfContainerHeader = bytemuck::pod_read_unaligned(&bytes[..64]);
90 header.validate()?;
91 if header.version == super::header::EMF_LEGACY_VERSION && !allow_legacy {
92 return Err(EmfError::LegacyIntegrityRequiresOptIn);
93 }
94 let full_hash = matches!(header.version,EMF_CONTAINER_VERSION|EMF_EXTENDED_VERSION);
95 let extension = if header.version==EMF_EXTENDED_VERSION {
96 if header.flags & !7 != 0 {return Err(EmfError::ExtensionMetadata);}
97 Some(super::extension::parse_extension(bytes,extension_limits)?)
98 } else {None};
99 let field_end=extension.as_ref().map(|e|e.field_end).unwrap_or(bytes.len());
100 if full_hash && header.flags & EMF_FLAG_BAO_VERIFIED != 0 {
101 return Err(EmfError::InvalidIntegrityEncoding { chunk_index: 0 });
102 }
103
104 let directory_offset = usize::try_from(header.directory_offset)
105 .map_err(|_| EmfError::ArithmeticOverflow)?;
106 let chunk_count = usize::try_from(header.chunk_count)
107 .map_err(|_| EmfError::ArithmeticOverflow)?;
108
109 if chunk_count > limits.max_chunks {
110 return Err(EmfError::ChunkLimitExceeded);
111 }
112 if directory_offset < 64 || directory_offset % 64 != 0 {
113 return Err(if directory_offset % 64 != 0 {
114 EmfError::DirectoryAlignment
115 } else {
116 EmfError::DirectoryBounds
117 });
118 }
119 let directory_bytes = chunk_count
120 .checked_mul(CHUNK_DIRECTORY_ENTRY_BYTES)
121 .ok_or(EmfError::ArithmeticOverflow)?;
122 let directory_end = directory_offset
123 .checked_add(directory_bytes)
124 .ok_or(EmfError::ArithmeticOverflow)?;
125
126 if directory_end > field_end {
127 return Err(EmfError::DirectoryBounds);
128 }
129
130 let mut directory_sorted=true;let mut previous_key=None;let mut equal_run=0usize;let mut max_equal_run=0usize;
131 for chunk in 0..chunk_count {
132 let offset = directory_offset + chunk * CHUNK_DIRECTORY_ENTRY_BYTES;
133 let entry: EmfChunkDirectoryEntry =
134 bytemuck::pod_read_unaligned(&bytes[offset..offset + CHUNK_DIRECTORY_ENTRY_BYTES]);
135 entry.validate()?;
136 if previous_key.is_some_and(|key|key>entry.spatial_hilbert_index){directory_sorted=false;}
137 equal_run=if previous_key==Some(entry.spatial_hilbert_index){equal_run+1}else{1};
138 max_equal_run=max_equal_run.max(equal_run);previous_key=Some(entry.spatial_hilbert_index);
139
140 let start = usize::try_from(entry.byte_offset).map_err(|_| EmfError::ArithmeticOverflow)?;
141 let length = usize::try_from(entry.byte_length).map_err(|_| EmfError::ArithmeticOverflow)?;
142 let end = start.checked_add(length).ok_or(EmfError::ArithmeticOverflow)?;
143
144 if start % 64 != 0 {
145 return Err(EmfError::PayloadUnaligned);
146 }
147 if end > field_end {
148 return Err(EmfError::PayloadOutOfBounds);
149 }
150 if start < 64 || ranges_overlap(start, end, directory_offset, directory_end) {
151 return Err(EmfError::PayloadOverlap);
152 }
153 let expected_bytes = usize::try_from(entry.cell_count)
154 .map_err(|_| EmfError::ArithmeticOverflow)?
155 .checked_mul(64)
156 .ok_or(EmfError::ArithmeticOverflow)?;
157 let stored_bytes = expected_bytes.checked_add(if full_hash { BLAKE3_TRAILER_BYTES } else { 0 })
158 .ok_or(EmfError::ArithmeticOverflow)?;
159 if length != stored_bytes {
160 return Err(EmfError::PayloadOutOfBounds);
161 }
162
163 for prev in 0..chunk {
164 let prev_off = directory_offset + prev * CHUNK_DIRECTORY_ENTRY_BYTES;
165 let prev_entry: EmfChunkDirectoryEntry = bytemuck::pod_read_unaligned(
166 &bytes[prev_off..prev_off + CHUNK_DIRECTORY_ENTRY_BYTES],
167 );
168 let prev_start = usize::try_from(prev_entry.byte_offset)
169 .map_err(|_| EmfError::ArithmeticOverflow)?;
170 let prev_len = usize::try_from(prev_entry.byte_length)
171 .map_err(|_| EmfError::ArithmeticOverflow)?;
172 let prev_end = prev_start
173 .checked_add(prev_len)
174 .ok_or(EmfError::ArithmeticOverflow)?;
175 if ranges_overlap(start, end, prev_start, prev_end) {
176 return Err(EmfError::PayloadOverlap);
177 }
178 }
179
180 if full_hash {
181 if entry.flags & EMF_CHUNK_FLAG_BLAKE3_FULL == 0 || entry.flags & 2 != 0
182 || entry.bao_hash_prefix != [0; 16] || entry.reserved != [0; 12] {
183 return Err(EmfError::InvalidIntegrityEncoding { chunk_index: chunk });
184 }
185 let records_end = start + expected_bytes;
186 if !verify_blake3_trailer(&bytes[start..records_end], &bytes[records_end..end]) {
187 return Err(EmfError::FullHashMismatch { chunk_index: chunk });
188 }
189 } else if (entry.flags & 2 != 0) || (entry.bao_hash_prefix != [0u8; 16]) {
190 let payload = &bytes[start..end];
192 let digest = blake3::hash(payload);
193 if digest.as_bytes()[..16] != entry.bao_hash_prefix {
194 return Err(EmfError::BaoHashMismatch { chunk_index: chunk });
195 }
196 }
197 }
198
199 let lookup_work_bound=if directory_sorted{(usize::BITS-chunk_count.leading_zeros()) as usize+max_equal_run+1}else{chunk_count}.max(1);
200 Ok(Self {
201 bytes,
202 header,
203 directory_offset,
204 chunk_count,
205 directory_sorted,
206 lookup_work_bound,
207 extension,
208 }).and_then(|reader| {
209 if let Some(extension)=&reader.extension {
210 super::extension::validate_field_directory(&reader.header,reader.directory_offset,reader.chunk_count,reader.bytes,extension)?;
211 }
212 Ok(reader)
213 })
214 }
215
216 #[inline]
218 pub fn header(&self) -> &EmfContainerHeader {
219 &self.header
220 }
221
222 pub fn extension(&self)->Option<&super::extension::EmfExtensionView<'a>>{self.extension.as_ref()}
224
225 #[inline]
227 pub fn chunk_count(&self) -> usize {
228 self.chunk_count
229 }
230
231 pub fn chunk_entry(&self, index: usize) -> Option<EmfChunkDirectoryEntry> {
233 if index >= self.chunk_count {
234 return None;
235 }
236 let offset = self.directory_offset + index * CHUNK_DIRECTORY_ENTRY_BYTES;
237 Some(bytemuck::pod_read_unaligned(
238 &self.bytes[offset..offset + CHUNK_DIRECTORY_ENTRY_BYTES],
239 ))
240 }
241
242 pub fn chunk_payload(&self, index: usize) -> Option<&'a [u8]> {
244 let entry = self.chunk_entry(index)?;
245 let start = entry.byte_offset as usize;
246 let end = start + entry.cell_count as usize * 64;
247 self.bytes.get(start..end)
248 }
249
250 pub fn cell(&self, chunk_index: usize, cell_index: usize) -> Option<EmfCellRecord64> {
252 let payload = self.chunk_payload(chunk_index)?;
253 let start = cell_index.checked_mul(64)?;
254 let raw = payload.get(start..start.checked_add(64)?)?;
255 Some(bytemuck::pod_read_unaligned(raw))
256 }
257
258 pub fn find_chunk_by_hilbert(&self,hilbert_key:u64)->Option<usize>{self.find_chunk_at_time(hilbert_key,0)}
261
262 pub fn find_chunk_at_time(&self,hilbert_key:u64,time_slice:u32)->Option<usize>{
265 if !self.directory_sorted {
266 return (0..self.chunk_count).find(|&index|self.chunk_entry(index).is_some_and(|entry|entry.spatial_hilbert_index==hilbert_key&&entry.time_slice_index==time_slice));
267 }
268 let mut low=0usize;let mut high=self.chunk_count;
269 while low<high {let middle=low+(high-low)/2;if self.chunk_entry(middle)?.spatial_hilbert_index<hilbert_key{low=middle+1;}else{high=middle;}}
270 while low<self.chunk_count {
271 let entry=self.chunk_entry(low)?;if entry.spatial_hilbert_index!=hilbert_key{return None;}
272 if entry.time_slice_index==time_slice{return Some(low);}low+=1;
273 }None
274 }
275
276 pub fn lookup_work_bound(&self)->usize{self.lookup_work_bound}
278
279 pub fn lookup_brick_cell(&self,bx:u32,by:u32,bz:u32,lx:u8,ly:u8,lz:u8,order:u32)->Option<EmfCellRecord64>{
281 self.lookup_brick_cell_at_time(bx,by,bz,lx,ly,lz,order,0)
282 }
283
284 pub fn lookup_brick_cell_at_time(&self,bx:u32,by:u32,bz:u32,lx:u8,ly:u8,lz:u8,order:u32,time_slice:u32)->Option<EmfCellRecord64>{
285 if lx>=8||ly>=8||lz>=8||order==0||order>21{return None;}
286 if bx>=(1u32<<order)||by>=(1u32<<order)||bz>=(1u32<<order){return None;}
287 let hilbert_key=encode_hilbert3d_u64(bx,by,bz,order);
288 let chunk=self.find_chunk_at_time(hilbert_key,time_slice)?;
289 self.cell(chunk,emf_brick_voxel_linear_index(lx,ly,lz))
290 }
291
292 pub fn dense_volume(&self) -> Result<DenseEmfVolume, EmfError> {
298 let dims = self.header.grid_dims_bricks;
299 if dims[0] == 0 || dims[1] == 0 || dims[2] == 0 {
300 return Err(EmfError::ChunkLimitExceeded);
301 }
302 let side = EMF_BRICK_SIDE as u32;
303 let nx = dims[0]
304 .checked_mul(side)
305 .ok_or(EmfError::ArithmeticOverflow)?;
306 let ny = dims[1]
307 .checked_mul(side)
308 .ok_or(EmfError::ArithmeticOverflow)?;
309 let nz = dims[2]
310 .checked_mul(side)
311 .ok_or(EmfError::ArithmeticOverflow)?;
312
313 let mut max_t = 0u32;
314 for i in 0..self.chunk_count {
315 let entry = self.chunk_entry(i).ok_or(EmfError::DirectoryBounds)?;
316 max_t = max_t.max(entry.time_slice_index);
317 }
318 let nt = max_t
319 .checked_add(1)
320 .ok_or(EmfError::ArithmeticOverflow)?;
321
322 let cell_count = (nx as usize)
323 .checked_mul(ny as usize)
324 .and_then(|n| n.checked_mul(nz as usize))
325 .and_then(|n| n.checked_mul(nt as usize))
326 .ok_or(EmfError::ArithmeticOverflow)?;
327 if cell_count > MAX_DENSE_CELLS {
328 return Err(EmfError::DenseVolumeCap {
329 cells: cell_count,
330 cap: MAX_DENSE_CELLS,
331 });
332 }
333 let mut cells = vec![EmfCellRecord64::default(); cell_count];
334 let mut sigma_sum = 0.0f64;
335 let mut sigma_n = 0u64;
336 let order = match &self.extension {
337 Some(extension)=>u32::from(extension.metadata().field.hilbert_order),
338 None=>infer_container_hilbert_order(self,dims)?,
339 };
340
341 for i in 0..self.chunk_count {
342 let entry = self.chunk_entry(i).ok_or(EmfError::DirectoryBounds)?;
343 let (bx, by, bz) = decode_hilbert3d_u64(entry.spatial_hilbert_index, order);
344 if bx >= dims[0] || by >= dims[1] || bz >= dims[2] {
345 return Err(EmfError::HilbertOrder);
346 }
347 let t = entry.time_slice_index;
348 let n_cells = usize::try_from(entry.cell_count).map_err(|_| EmfError::ArithmeticOverflow)?;
349 for vi in 0..n_cells {
350 let cell = self.cell(i, vi).ok_or(EmfError::PayloadOutOfBounds)?;
351 let (lx, ly, lz) = emf_brick_voxel_coords(vi);
352 let x = bx
353 .checked_mul(side)
354 .and_then(|v| v.checked_add(lx as u32))
355 .ok_or(EmfError::ArithmeticOverflow)?;
356 let y = by
357 .checked_mul(side)
358 .and_then(|v| v.checked_add(ly as u32))
359 .ok_or(EmfError::ArithmeticOverflow)?;
360 let z = bz
361 .checked_mul(side)
362 .and_then(|v| v.checked_add(lz as u32))
363 .ok_or(EmfError::ArithmeticOverflow)?;
364 if x >= nx || y >= ny || z >= nz {
365 return Err(EmfError::PayloadOutOfBounds);
366 }
367 let idx = dense_cell_index(x, y, z, t, nx, ny, nz)?;
368 cells[idx] = cell;
369 if cell.e_magnitude() > 0.0
370 || cell.wavelength_nm > 0.0
371 || cell.spectral_sigma != 0.0
372 {
373 sigma_sum += cell.effective_sigma() as f64;
374 sigma_n += 1;
375 }
376 }
377 }
378
379 let mean_sigma = if sigma_n > 0 {
380 (sigma_sum / sigma_n as f64) as f32
381 } else {
382 0.0
383 };
384
385 Ok(DenseEmfVolume {
386 origin_meters: self.header.origin_meters,
387 voxel_size_meters: self.header.voxel_size_meters,
388 nx,
389 ny,
390 nz,
391 nt,
392 cells,
393 mean_sigma,
394 })
395 }
396}
397
398#[derive(Clone, Debug)]
400pub struct DenseEmfVolume {
401 pub origin_meters: [f32; 3],
402 pub voxel_size_meters: f32,
403 pub nx: u32,
404 pub ny: u32,
405 pub nz: u32,
406 pub nt: u32,
407 pub cells: Vec<EmfCellRecord64>,
408 pub mean_sigma: f32,
409}
410
411impl DenseEmfVolume {
412 pub fn bounds_meters(&self) -> [f32; 6] {
414 let ox = self.origin_meters[0];
415 let oy = self.origin_meters[1];
416 let oz = self.origin_meters[2];
417 let vx = self.voxel_size_meters * self.nx as f32;
418 let vy = self.voxel_size_meters * self.ny as f32;
419 let vz = self.voxel_size_meters * self.nz as f32;
420 [ox, ox + vx, oy, oy + vy, oz, oz + vz]
421 }
422}
423
424fn dense_cell_index(
425 x: u32,
426 y: u32,
427 z: u32,
428 t: u32,
429 nx: u32,
430 ny: u32,
431 nz: u32,
432) -> Result<usize, EmfError> {
433 let plane = (nx as usize)
434 .checked_mul(ny as usize)
435 .ok_or(EmfError::ArithmeticOverflow)?;
436 let volume = plane
437 .checked_mul(nz as usize)
438 .ok_or(EmfError::ArithmeticOverflow)?;
439 let t_off = (t as usize)
440 .checked_mul(volume)
441 .ok_or(EmfError::ArithmeticOverflow)?;
442 let z_off = (z as usize)
443 .checked_mul(plane)
444 .ok_or(EmfError::ArithmeticOverflow)?;
445 let y_off = (y as usize)
446 .checked_mul(nx as usize)
447 .ok_or(EmfError::ArithmeticOverflow)?;
448 t_off
449 .checked_add(z_off)
450 .and_then(|n| n.checked_add(y_off))
451 .and_then(|n| n.checked_add(x as usize))
452 .ok_or(EmfError::ArithmeticOverflow)
453}
454
455fn infer_container_hilbert_order(
456 reader: &EmfContainerReader<'_>,
457 dims: [u32; 3],
458) -> Result<u32, EmfError> {
459 if reader.chunk_count == 0 {
460 return Ok(2);
461 }
462 for order in [2, 3, 1, 4, 5, 6, 7, 8] {
465 let mut seen = std::collections::BTreeSet::new();
466 let mut ok = true;
467 for i in 0..reader.chunk_count {
468 let entry = reader.chunk_entry(i).ok_or(EmfError::DirectoryBounds)?;
469 let (bx, by, bz) = decode_hilbert3d_u64(entry.spatial_hilbert_index, order);
470 if bx >= dims[0]
471 || by >= dims[1]
472 || bz >= dims[2]
473 || encode_hilbert3d_u64(bx, by, bz, order) != entry.spatial_hilbert_index
474 || !seen.insert((bx, by, bz, entry.time_slice_index))
475 {
476 ok = false;
477 break;
478 }
479 }
480 if ok {
481 return Ok(order);
482 }
483 }
484 Err(EmfError::HilbertOrder)
485}