Skip to main content

qualia_core_db/q42/volume/
wasm_slice_reader.rs

1//! Byte-backed `.q42` volume and raw `NQuin` slice reader for WASM and native hosts.
2//!
3//! Operates directly over `&[u8]` (such as an `.hmc` / `.qpack` entry slice or
4//! OPFS buffer) without requiring `std::fs::File` or `memmap2`. Hot-path block
5//! queries, object BIDX searches, and lexicon lookups write into caller-supplied
6//! slices with zero heap allocations.
7
8use std::io;
9
10use super::super::{
11    MAX_COMPRESSED_SUPERBLOCK_SIZE, Q42_MAGIC, QUIN_SIZE, SUPERBLOCK_SIZE,
12};
13use super::opfs_source::OpfsSliceRangeSource;
14use super::range::{Q42ByteRange, Q42RangeSource};
15use super::range_volume::{
16    Q42ObjectMatchPage, Q42ObjectSearchCursor, Q42RangeQueryCursor, Q42RangeQueryPage,
17    Q42RangeQueryPattern, Q42RangeQueryPlan, Q42RangeVolume,
18};
19use crate::q42_lex::{
20    INDEX_ENTRY_SIZE, LEX_HEADER_SIZE, LEX_MAGIC, LEX_TAG_STRING, LEX_VERSION_PAGED,
21    LEX_VERSION_V4,
22};
23use crate::NQuin;
24
25fn invalid(msg: impl Into<String>) -> io::Error {
26    io::Error::new(io::ErrorKind::InvalidData, msg.into())
27}
28
29enum SliceBacking<'a> {
30    UnifiedVolume(Q42RangeVolume<OpfsSliceRangeSource<&'a [u8]>>),
31    RawQuinSlice(&'a [u8]),
32}
33
34/// Transport-neutral reader over a borrowed `.q42` volume or 48-byte `NQuin` slice.
35pub struct WasmSliceQ42Reader<'a> {
36    bytes: &'a [u8],
37    backing: SliceBacking<'a>,
38}
39
40impl<'a> WasmSliceQ42Reader<'a> {
41    /// Open and validate a `.q42` volume slice or a raw 48-byte `NQuin` array slice.
42    pub fn open(bytes: &'a [u8]) -> io::Result<Self> {
43        if bytes.is_empty() {
44            return Err(invalid("empty Q42 byte slice"));
45        }
46        if bytes.len() >= 4 && bytes[0..4] == Q42_MAGIC {
47            let source = OpfsSliceRangeSource::new(bytes);
48            let volume = Q42RangeVolume::open(source)?;
49            return Ok(Self {
50                bytes,
51                backing: SliceBacking::UnifiedVolume(volume),
52            });
53        }
54        if bytes.len() % QUIN_SIZE == 0 {
55            for chunk in bytes.chunks_exact(QUIN_SIZE) {
56                let quin = bytemuck::pod_read_unaligned::<NQuin>(chunk);
57                if !quin.verify_ecc_parity() {
58                    return Err(invalid("raw NQuin slice failed parity integrity check"));
59                }
60            }
61            return Ok(Self {
62                bytes,
63                backing: SliceBacking::RawQuinSlice(bytes),
64            });
65        }
66        Err(invalid(
67            "byte slice is neither a Q42VOL unified volume nor a 48-byte-aligned NQuin array",
68        ))
69    }
70
71    /// Returns true if backed by a full `Q42VOL` headered volume rather than raw 48B quins.
72    #[inline]
73    pub fn is_unified_volume(&self) -> bool {
74        matches!(self.backing, SliceBacking::UnifiedVolume(_))
75    }
76
77    /// Version number of the volume (`3` or `4` for unified `.q42`, `0` for raw `NQuin` slice).
78    #[inline]
79    pub fn version(&self) -> u32 {
80        match &self.backing {
81            SliceBacking::UnifiedVolume(vol) => u32::from(vol.header().version),
82            SliceBacking::RawQuinSlice(_) => 0,
83        }
84    }
85
86    /// Number of SuperBlocks (`0` for raw `NQuin` slices).
87    #[inline]
88    pub fn block_count(&self) -> u64 {
89        match &self.backing {
90            SliceBacking::UnifiedVolume(vol) => vol.block_count(),
91            SliceBacking::RawQuinSlice(_) => 0,
92        }
93    }
94
95    /// Underlying byte slice length.
96    #[inline]
97    pub fn byte_len(&self) -> usize {
98        self.bytes.len()
99    }
100
101    /// Zero-heap paged pattern query into caller-supplied buffers.
102    ///
103    /// For unified `.q42` volumes, `compressed` must be at least
104    /// `MAX_COMPRESSED_SUPERBLOCK_SIZE` and `decoded` must be at least `SUPERBLOCK_SIZE`.
105    pub fn execute_query_page_into(
106        &self,
107        plan: Q42RangeQueryPlan,
108        cursor: Q42RangeQueryCursor,
109        compressed: &mut [u8],
110        decoded: &mut [u8],
111        out: &mut [NQuin],
112    ) -> io::Result<Q42RangeQueryPage> {
113        match &self.backing {
114            SliceBacking::UnifiedVolume(vol) => {
115                vol.execute_query_page_into(plan, cursor, compressed, decoded, out)
116            }
117            SliceBacking::RawQuinSlice(raw) => {
118                if out.is_empty() {
119                    return Err(io::Error::new(
120                        io::ErrorKind::InvalidInput,
121                        "Q42 query output buffer is empty",
122                    ));
123                }
124                let total_quins = raw.len() / QUIN_SIZE;
125                if cursor.block_index > 0 || cursor.quin_offset >= total_quins {
126                    return Ok(Q42RangeQueryPage {
127                        returned: 0,
128                        next_cursor: None,
129                    });
130                }
131                let mut idx = cursor.quin_offset;
132                let mut returned = 0usize;
133                let pat = plan.pattern;
134                while idx < total_quins {
135                    let start = idx * QUIN_SIZE;
136                    let quin = bytemuck::pod_read_unaligned::<NQuin>(&raw[start..start + QUIN_SIZE]);
137                    idx += 1;
138                    if pat.subject.is_some_and(|s| s != quin.subject)
139                        || pat.predicate.is_some_and(|p| p != quin.predicate)
140                        || pat.object.is_some_and(|o| o != quin.object)
141                        || pat.context.is_some_and(|c| c != quin.context)
142                    {
143                        continue;
144                    }
145                    out[returned] = quin;
146                    returned += 1;
147                    if returned == out.len() {
148                        let next_cursor = (idx < total_quins).then_some(Q42RangeQueryCursor {
149                            block_index: 0,
150                            quin_offset: idx,
151                        });
152                        return Ok(Q42RangeQueryPage {
153                            returned,
154                            next_cursor,
155                        });
156                    }
157                }
158                Ok(Q42RangeQueryPage {
159                    returned,
160                    next_cursor: None,
161                })
162            }
163        }
164    }
165
166    /// Zero-heap exact object lookup using BIDX on unified volumes or linear scan on raw slices.
167    pub fn find_object_into(
168        &self,
169        object_hash: u64,
170        cursor: Q42ObjectSearchCursor,
171        compressed: &mut [u8],
172        decoded: &mut [u8],
173        out: &mut [NQuin],
174    ) -> io::Result<Option<Q42ObjectMatchPage>> {
175        match &self.backing {
176            SliceBacking::UnifiedVolume(vol) => {
177                vol.find_object_into(object_hash, cursor, compressed, decoded, out)
178            }
179            SliceBacking::RawQuinSlice(_) => {
180                let plan = Q42RangeQueryPlan::for_pattern(Q42RangeQueryPattern {
181                    subject: None,
182                    predicate: None,
183                    object: Some(object_hash),
184                    context: None,
185                });
186                let page = self.execute_query_page_into(
187                    plan,
188                    Q42RangeQueryCursor {
189                        block_index: cursor.block_offset,
190                        quin_offset: cursor.quin_offset,
191                    },
192                    compressed,
193                    decoded,
194                    out,
195                )?;
196                if page.returned == 0 && cursor.block_offset == 0 && cursor.quin_offset == 0 {
197                    return Ok(None);
198                }
199                Ok(Some(Q42ObjectMatchPage {
200                    block_range: super::index::BidxBlockRange { start: 0, end: 1 },
201                    returned: page.returned,
202                    next_cursor: page.next_cursor.map(|c| Q42ObjectSearchCursor {
203                        block_offset: c.block_index,
204                        quin_offset: c.quin_offset,
205                    }),
206                }))
207            }
208        }
209    }
210
211    /// Zero-heap lexicon hash lookup supporting both flat v1 `Q42LEX` and paged v2/v4 `Q42LEX`.
212    pub fn lookup_lexicon_hash_into(
213        &self,
214        hash: u64,
215        page_scratch: &mut [u8],
216        out: &mut [u8],
217    ) -> io::Result<Option<usize>> {
218        let SliceBacking::UnifiedVolume(vol) = &self.backing else {
219            return Ok(None);
220        };
221        let lex_offset = vol.header().lex_offset;
222        let lex_length = vol.header().lex_length;
223        if lex_length < LEX_HEADER_SIZE as u64 {
224            return Ok(None);
225        }
226        let mut header = [0u8; LEX_HEADER_SIZE];
227        vol.source().read_range_into(
228            Q42ByteRange {
229                offset: lex_offset,
230                length: LEX_HEADER_SIZE,
231            },
232            &mut header,
233        )?;
234        if header[0..8] != LEX_MAGIC {
235            return Err(invalid("Q42 lexicon section has invalid magic"));
236        }
237        let version = u64::from_le_bytes(header[24..32].try_into().unwrap());
238        if version == LEX_VERSION_PAGED || version == LEX_VERSION_V4 {
239            return vol.lookup_lexicon_hash_into(hash, page_scratch, out);
240        }
241        if version != 1 {
242            return Err(invalid("unsupported Q42LEX version"));
243        }
244        let entry_count = u64::from_le_bytes(header[8..16].try_into().unwrap()) as usize;
245        let strings_offset = u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize;
246        let lex_start = lex_offset as usize;
247        let lex_end = lex_start
248            .checked_add(lex_length as usize)
249            .ok_or_else(|| invalid("Q42 lexicon slice overflow"))?;
250        if lex_end > self.bytes.len() {
251            return Err(invalid("Q42 lexicon extends past volume slice"));
252        }
253        let lex_bytes = &self.bytes[lex_start..lex_end];
254        let index_end = LEX_HEADER_SIZE
255            .checked_add(entry_count.saturating_mul(INDEX_ENTRY_SIZE))
256            .ok_or_else(|| invalid("Q42 lexicon index overflow"))?;
257        if strings_offset < index_end || strings_offset > lex_bytes.len() {
258            return Err(invalid("Q42 v1 lexicon strings offset out of bounds"));
259        }
260        let mut lo = 0usize;
261        let mut hi = entry_count;
262        while lo < hi {
263            let mid = lo + (hi - lo) / 2;
264            let pos = LEX_HEADER_SIZE + mid * INDEX_ENTRY_SIZE;
265            let entry_hash = u64::from_le_bytes(lex_bytes[pos..pos + 8].try_into().unwrap());
266            if entry_hash < hash {
267                lo = mid + 1;
268            } else if entry_hash > hash {
269                hi = mid;
270            } else {
271                let rel = u64::from_le_bytes(lex_bytes[pos + 8..pos + 16].try_into().unwrap())
272                    as usize;
273                let start = strings_offset
274                    .checked_add(rel)
275                    .ok_or_else(|| invalid("Q42 lexicon string offset overflow"))?;
276                if start + 3 > lex_bytes.len() || lex_bytes[start] != LEX_TAG_STRING {
277                    return Err(invalid("Q42 lexicon v1 entry malformed"));
278                }
279                let str_len =
280                    u16::from_le_bytes(lex_bytes[start + 1..start + 3].try_into().unwrap())
281                        as usize;
282                let end = start + 3 + str_len;
283                if end > lex_bytes.len() {
284                    return Err(invalid("Q42 lexicon v1 string truncated"));
285                }
286                if str_len > out.len() {
287                    return Err(io::Error::new(
288                        io::ErrorKind::WriteZero,
289                        "Q42 lexicon output buffer too small",
290                    ));
291                }
292                std::str::from_utf8(&lex_bytes[start + 3..end])
293                    .map_err(|_| invalid("Q42 lexicon v1 entry is not UTF-8"))?;
294                out[..str_len].copy_from_slice(&lex_bytes[start + 3..end]);
295                return Ok(Some(str_len));
296            }
297        }
298        Ok(None)
299    }
300
301    /// Cold-path helper to collect all `NQuin` records from the slice.
302    pub fn collect_all_quins(&self) -> io::Result<Vec<NQuin>> {
303        match &self.backing {
304            SliceBacking::RawQuinSlice(raw) => {
305                let mut out = Vec::with_capacity(raw.len() / QUIN_SIZE);
306                for chunk in raw.chunks_exact(QUIN_SIZE) {
307                    out.push(bytemuck::pod_read_unaligned::<NQuin>(chunk));
308                }
309                Ok(out)
310            }
311            SliceBacking::UnifiedVolume(vol) => {
312                let mut compressed = vec![0u8; MAX_COMPRESSED_SUPERBLOCK_SIZE];
313                let mut decoded = vec![0u8; SUPERBLOCK_SIZE];
314                let mut page_buf = [NQuin::default(); 256];
315                let mut all = Vec::new();
316                let plan = Q42RangeQueryPlan::for_pattern(Q42RangeQueryPattern::default());
317                let mut cursor = Q42RangeQueryCursor::default();
318                loop {
319                    let page = vol.execute_query_page_into(
320                        plan,
321                        cursor,
322                        &mut compressed,
323                        &mut decoded,
324                        &mut page_buf,
325                    )?;
326                    all.extend_from_slice(&page_buf[..page.returned]);
327                    match page.next_cursor {
328                        Some(next) => cursor = next,
329                        None => break,
330                    }
331                }
332                Ok(all)
333            }
334        }
335    }
336}
337
338#[cfg(test)]
339mod tests {
340    use super::*;
341    use crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc;
342
343    #[test]
344    fn raw_quin_slice_reader_queries_zero_heap() {
345        let mut q1 = NQuin {
346            subject: 11,
347            predicate: 22,
348            object: 33,
349            context: 44,
350            metadata: 0,
351            parity: 0,
352        };
353        q1.parity =
354            NQuin::calculate_parity(q1.subject, q1.predicate, q1.object, q1.context, q1.metadata);
355        let mut q2 = NQuin {
356            subject: 11,
357            predicate: 22,
358            object: 99,
359            context: 44,
360            metadata: 0,
361            parity: 0,
362        };
363        q2.parity =
364            NQuin::calculate_parity(q2.subject, q2.predicate, q2.object, q2.context, q2.metadata);
365        let quins = [q1, q2];
366        let raw_bytes: &[u8] = bytemuck::cast_slice(&quins);
367
368        let reader = WasmSliceQ42Reader::open(raw_bytes).expect("open raw slice");
369        assert!(!reader.is_unified_volume());
370
371        let mut comp = [0u8; 64];
372        let mut dec = [0u8; 64];
373        let mut out = [NQuin::default(); 4];
374        let mut matched = 0usize;
375
376        assert_zero_alloc("wasm_slice_raw_find_object", || {
377            let page = reader
378                .find_object_into(
379                    99,
380                    Q42ObjectSearchCursor::default(),
381                    &mut comp,
382                    &mut dec,
383                    &mut out,
384                )
385                .unwrap()
386                .unwrap();
387            matched = page.returned;
388        });
389        assert_eq!(matched, 1);
390        assert_eq!(out[0].object, 99);
391    }
392}