Skip to main content

qualia_core_db/q42/emf_container/
writer.rs

1//! Container Writer and Builder for EMF-64 Containers.
2//!
3//! Provides zero-copy / caller-buffered serialisation with automatic
4//! CRC-32C calculations and full 256-bit BLAKE3 chunk trailers.
5
6use super::chunk_directory::{EmfChunkDirectoryEntry, CHUNK_DIRECTORY_ENTRY_BYTES};
7use crate::q42::chunk_integrity::{blake3_trailer, BLAKE3_TRAILER_BYTES};
8use super::header::{
9    EmfCellRecord64, EmfContainerHeader, EmfError, EMF_CONTAINER_MAGIC, EMF_CONTAINER_VERSION,
10    EMF_FLAG_3D_BRICK_8X8X8, EMF_FLAG_BAO_VERIFIED, EMF_CHUNK_FLAG_BLAKE3_FULL,
11};
12
13/// An uncompressed chunk staged for writing into an EMF-64 container.
14#[derive(Clone, Debug)]
15pub struct StagedEmfChunk {
16    pub spatial_hilbert_index: u64,
17    pub cells: Vec<EmfCellRecord64>,
18    pub time_slice_index: u32,
19    pub spectral_band_mask: u16,
20    pub flags: u16,
21}
22
23/// Builder for constructing valid EMF-64 container files.
24#[derive(Clone, Debug)]
25pub struct EmfContainerWriter {
26    pub origin_meters: [f32; 3],
27    pub voxel_size_meters: f32,
28    pub grid_dims_bricks: [u32; 3],
29    pub reference_epoch_s: f64,
30    pub flags: u16,
31    chunks: Vec<StagedEmfChunk>,
32}
33
34impl EmfContainerWriter {
35    /// Create a new EMF container builder.
36    pub fn new(
37        origin_meters: [f32; 3],
38        voxel_size_meters: f32,
39        grid_dims_bricks: [u32; 3],
40        reference_epoch_s: f64,
41    ) -> Self {
42        Self {
43            origin_meters,
44            voxel_size_meters,
45            grid_dims_bricks,
46            reference_epoch_s,
47            flags: EMF_FLAG_3D_BRICK_8X8X8,
48            chunks: Vec::new(),
49        }
50    }
51
52    /// Add an 8x8x8 volumetric brick chunk.
53    pub fn add_brick_chunk(
54        &mut self,
55        spatial_hilbert_index: u64,
56        cells: &[EmfCellRecord64],
57        time_slice_index: u32,
58        spectral_band_mask: u16,
59        flags: u16,
60    ) -> Result<(), EmfError> {
61        self.chunks.push(StagedEmfChunk {
62            spatial_hilbert_index,
63            cells: cells.to_vec(),
64            time_slice_index,
65            spectral_band_mask,
66            flags,
67        });
68        Ok(())
69    }
70
71    /// Directory table size in bytes for `chunk_count` rows.
72    pub(crate) fn checked_directory_bytes(chunk_count: usize) -> Result<usize, EmfError> {
73        chunk_count
74            .checked_mul(CHUNK_DIRECTORY_ENTRY_BYTES)
75            .ok_or(EmfError::ArithmeticOverflow)
76    }
77
78    /// Payload size in bytes for `cell_count` 64-byte cells.
79    pub(crate) fn checked_payload_bytes(cell_count: usize) -> Result<usize, EmfError> {
80        cell_count
81            .checked_mul(64)
82            .and_then(|n| n.checked_add(BLAKE3_TRAILER_BYTES))
83            .ok_or(EmfError::ArithmeticOverflow)
84    }
85
86    /// Calculate the exact total byte size needed to serialize this container.
87    pub fn total_required_bytes(&self) -> Result<usize, EmfError> {
88        let directory_bytes = Self::checked_directory_bytes(self.chunks.len())?;
89        let mut payload_bytes = 0usize;
90        for c in &self.chunks {
91            let chunk_payload = Self::checked_payload_bytes(c.cells.len())?;
92            payload_bytes = payload_bytes
93                .checked_add(chunk_payload)
94                .ok_or(EmfError::ArithmeticOverflow)?;
95        }
96        64usize
97            .checked_add(directory_bytes)
98            .and_then(|n| n.checked_add(payload_bytes))
99            .ok_or(EmfError::ArithmeticOverflow)
100    }
101
102    /// Serialize into caller output; cold sorting uses bounded-per-chunk scratch.
103    pub fn write_to_slice(&self, out: &mut [u8]) -> Result<usize, EmfError> {
104        let required = self.total_required_bytes()?;
105        if out.len() < required {
106            return Err(EmfError::BufferTooSmall {
107                required,
108                provided: out.len(),
109            });
110        }
111
112        // Sort chunks by Hilbert index so directory is sorted for binary search
113        let mut sorted_indices: Vec<usize> = (0..self.chunks.len()).collect();
114        sorted_indices.sort_by_key(|&idx| self.chunks[idx].spatial_hilbert_index);
115
116        let chunk_count =
117            u32::try_from(self.chunks.len()).map_err(|_| EmfError::ArithmeticOverflow)?;
118        let directory_offset = 64u64;
119        let directory_bytes = u64::try_from(Self::checked_directory_bytes(self.chunks.len())?)
120            .map_err(|_| EmfError::ArithmeticOverflow)?;
121        let mut current_payload_offset = directory_offset
122            .checked_add(directory_bytes)
123            .ok_or(EmfError::ArithmeticOverflow)?;
124
125        // Write directory and payloads
126        for (i, &chunk_idx) in sorted_indices.iter().enumerate() {
127            let chunk = &self.chunks[chunk_idx];
128            let cell_count =
129                u32::try_from(chunk.cells.len()).map_err(|_| EmfError::ArithmeticOverflow)?;
130            let byte_length = cell_count
131                .checked_mul(64)
132                .and_then(|n| n.checked_add(BLAKE3_TRAILER_BYTES as u32))
133                .ok_or(EmfError::ArithmeticOverflow)?;
134
135            // Serialize cell records to payload region
136            let p_start = usize::try_from(current_payload_offset)
137                .map_err(|_| EmfError::ArithmeticOverflow)?;
138            let payload_len = usize::try_from(byte_length).map_err(|_| EmfError::ArithmeticOverflow)? - BLAKE3_TRAILER_BYTES;
139            let p_end = p_start
140                .checked_add(payload_len)
141                .ok_or(EmfError::ArithmeticOverflow)?;
142            let payload_slice = bytemuck::cast_slice(&chunk.cells);
143            out.get_mut(p_start..p_end)
144                .ok_or(EmfError::PayloadOutOfBounds)?
145                .copy_from_slice(payload_slice);
146
147            let trailer = blake3_trailer(&out[p_start..p_end]);
148            out[p_end..p_end + BLAKE3_TRAILER_BYTES].copy_from_slice(&trailer);
149
150            // Construct directory entry
151            let mut entry = EmfChunkDirectoryEntry {
152                spatial_hilbert_index: chunk.spatial_hilbert_index,
153                byte_offset: current_payload_offset,
154                byte_length,
155                cell_count,
156                spectral_band_mask: chunk.spectral_band_mask,
157                flags: (chunk.flags & !2) | EMF_CHUNK_FLAG_BLAKE3_FULL,
158                time_slice_index: chunk.time_slice_index,
159                bao_hash_prefix: [0; 16],
160                reserved: [0u8; 12],
161                entry_crc32c: 0,
162            };
163            entry.update_crc();
164
165            // Write directory row
166            let d_start = usize::try_from(directory_offset)
167                .map_err(|_| EmfError::ArithmeticOverflow)?
168                .checked_add(
169                    i.checked_mul(CHUNK_DIRECTORY_ENTRY_BYTES)
170                        .ok_or(EmfError::ArithmeticOverflow)?,
171                )
172                .ok_or(EmfError::ArithmeticOverflow)?;
173            let d_end = d_start
174                .checked_add(CHUNK_DIRECTORY_ENTRY_BYTES)
175                .ok_or(EmfError::ArithmeticOverflow)?;
176            out.get_mut(d_start..d_end)
177                .ok_or(EmfError::DirectoryBounds)?
178                .copy_from_slice(bytemuck::bytes_of(&entry));
179
180            current_payload_offset = current_payload_offset
181                .checked_add(u64::from(byte_length))
182                .ok_or(EmfError::ArithmeticOverflow)?;
183        }
184
185        // Build and write container header
186        let mut header = EmfContainerHeader {
187            magic: EMF_CONTAINER_MAGIC,
188            version: EMF_CONTAINER_VERSION,
189            flags: self.flags & !EMF_FLAG_BAO_VERIFIED,
190            chunk_count,
191            directory_offset,
192            reference_epoch_s: self.reference_epoch_s,
193            grid_dims_bricks: self.grid_dims_bricks,
194            voxel_size_meters: self.voxel_size_meters,
195            origin_meters: self.origin_meters,
196            header_crc32c: 0,
197        };
198        header.update_crc();
199        out[..64].copy_from_slice(bytemuck::bytes_of(&header));
200
201        Ok(required)
202    }
203
204    /// Opt-in v3. Cells and their repaired v2 trailers remain byte-identical.
205    /// All extension/file limits are checked before allocating the output buffer.
206    pub fn to_bytes_v3(&self,field:&super::extension::EmfFieldInterpretation,tracks:&[super::extension::EmfAudioTrack<'_>],limits:super::extension::EmfExtensionLimits)->Result<Vec<u8>,EmfError>{
207        if !cfg!(target_endian="little")||self.flags&!7!=0||self.chunks.iter().any(|c|c.flags & !EMF_CHUNK_FLAG_BLAKE3_FULL !=0){return Err(EmfError::ExtensionMetadata);}
208        let probe=EmfContainerHeader{reference_epoch_s:self.reference_epoch_s,voxel_size_meters:self.voxel_size_meters,
209            grid_dims_bricks:self.grid_dims_bricks,origin_meters:self.origin_meters,..EmfContainerHeader::default()};
210        field.validate_header(&probe)?;
211        let prepared=super::extension::prepare_extension(field,tracks,limits)?;
212        let field_end=self.total_required_bytes()?;let total=prepared.required_file_bytes(field_end)?;
213        let mut bytes=vec![0u8;total];self.write_to_slice(&mut bytes[..field_end])?;
214        prepared.write_into(tracks,&mut bytes,field_end)?;
215        let mut header:EmfContainerHeader=bytemuck::pod_read_unaligned(&bytes[..64]);
216        header.version=super::header::EMF_EXTENDED_VERSION;header.update_crc();
217        bytes[..64].copy_from_slice(bytemuck::bytes_of(&header));
218        super::reader::EmfContainerReader::parse_with_extension_limits(&bytes,super::reader::EmfReadLimits{max_file_bytes:limits.max_file_bytes,max_chunks:self.chunks.len()},limits)?;
219        Ok(bytes)
220    }
221
222    /// Serialize the container into a heap-allocated `Vec<u8>` (Tier-2 Cold Construction).
223    pub fn to_bytes(&self) -> Result<Vec<u8>, EmfError> {
224        let required = self.total_required_bytes()?;
225        let mut out = vec![0u8; required];
226        self.write_to_slice(&mut out)?;
227        Ok(out)
228    }
229}