qualia_core_db/
archive.rs1#[cfg(not(target_arch = "wasm32"))]
2use memmap2::MmapOptions;
3#[cfg(not(target_arch = "wasm32"))]
4use std::fs::File;
5#[cfg(not(target_arch = "wasm32"))]
6use std::io;
7#[cfg(not(target_arch = "wasm32"))]
8use std::io::{Cursor, Read};
9#[cfg(not(target_arch = "wasm32"))]
10use std::path::Path;
11
12pub const Q42_MAGIC: [u8; 4] = [0x51, 0x34, 0x32, 0x00]; #[repr(C, packed)]
17#[derive(Debug, Clone, Copy)]
18pub struct Q42Preamble {
19 pub magic: [u8; 4],
20 pub version: u16,
21 pub global_flags: u16,
22
23 pub dict_manifest_standard: [u16; 2],
26 pub dict_manifest_permissive: [u16; 2],
27 pub dict_manifest_bilateral: [u16; 2],
28 pub dict_manifest_spatiotemporal: [u16; 2],
29
30 pub index_manifest_standard: [u32; 2],
33 pub index_manifest_permissive: [u32; 2],
34 pub index_manifest_bilateral: [u32; 2],
35 pub index_manifest_spatiotemporal: [u32; 2],
36
37 pub eof_marker: u64,
38}
39
40#[repr(C, packed)]
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub struct Q42JumpEntry {
44 pub virtual_chunk_id: u32,
45 pub physical_offset_low: u32,
46 pub physical_offset_high: u16, pub frame_size: u16,
48}
49
50impl Q42JumpEntry {
51 pub fn physical_offset(&self) -> u64 {
54 (self.physical_offset_low as u64) | ((self.physical_offset_high as u64) << 32)
55 }
56}
57
58#[cfg(not(target_arch = "wasm32"))]
60pub struct Q42Archive {
61 mmap: memmap2::Mmap,
62}
63
64#[cfg(not(target_arch = "wasm32"))]
65impl Q42Archive {
66 pub fn open<P: AsRef<Path>>(path: P) -> io::Result<Self> {
68 let file = File::open(path)?;
69 let mmap = unsafe { MmapOptions::new().map(&file)? };
70
71 if mmap.len() < 64 {
72 return Err(io::Error::new(
73 io::ErrorKind::UnexpectedEof,
74 "File too small to contain Q42 Preamble",
75 ));
76 }
77
78 let archive = Self { mmap };
79 let preamble = archive.preamble();
80
81 if preamble.magic != Q42_MAGIC {
82 return Err(io::Error::new(
83 io::ErrorKind::InvalidData,
84 "Invalid Q42 Magic Number",
85 ));
86 }
87
88 Ok(archive)
89 }
90
91 pub fn preamble(&self) -> &Q42Preamble {
93 unsafe { &*(self.mmap.as_ptr() as *const Q42Preamble) }
94 }
95
96 pub fn read_jump_table(&self, offset: u32, size_bytes: u32) -> &[Q42JumpEntry] {
98 let start = offset as usize;
99 let end = start + size_bytes as usize;
100 let count = size_bytes as usize / std::mem::size_of::<Q42JumpEntry>();
101
102 unsafe {
103 std::slice::from_raw_parts(self.mmap[start..end].as_ptr() as *const Q42JumpEntry, count)
104 }
105 }
106
107 pub fn read_dictionary(&self, offset: u16, size_bytes: u16) -> &[u8] {
109 let start = 0x40 + offset as usize;
110 let end = start + size_bytes as usize;
111 &self.mmap[start..end]
112 }
113
114 pub fn decompress_frame(&self, entry: &Q42JumpEntry, dict: &[u8]) -> io::Result<Vec<u8>> {
116 let start = entry.physical_offset() as usize;
117 let end = start + entry.frame_size as usize;
118 let compressed_data = &self.mmap[start..end];
119
120 let mut decoder =
121 zstd::stream::Decoder::with_dictionary(Cursor::new(compressed_data), dict)?;
122 let mut output = Vec::with_capacity(128 * 1024);
123 decoder.read_to_end(&mut output)?;
124
125 Ok(output)
126 }
127}
128
129#[cfg(test)]
130mod tests {
131 use super::*;
132 use std::io::Write;
133 use tempfile::NamedTempFile;
134
135 #[test]
136 fn test_zero_deserialization_header_alignment() {
137 assert_eq!(std::mem::size_of::<Q42Preamble>(), 64);
138 assert_eq!(std::mem::size_of::<Q42JumpEntry>(), 12);
139 }
140
141 #[test]
142 fn test_q42_archive_mapping() {
143 let mut file = NamedTempFile::new().unwrap();
144
145 let preamble = Q42Preamble {
146 magic: Q42_MAGIC,
147 version: 1,
148 global_flags: 0,
149 dict_manifest_standard: [0, 0],
150 dict_manifest_permissive: [0, 0],
151 dict_manifest_bilateral: [0, 0],
152 dict_manifest_spatiotemporal: [0, 0],
153 index_manifest_standard: [0, 0],
154 index_manifest_permissive: [0, 0],
155 index_manifest_bilateral: [0, 0],
156 index_manifest_spatiotemporal: [0, 0],
157 eof_marker: 64,
158 };
159
160 unsafe {
161 let bytes = std::slice::from_raw_parts(&preamble as *const _ as *const u8, 64);
162 file.write_all(bytes).unwrap();
163 }
164
165 let archive = Q42Archive::open(file.path()).unwrap();
166 let mapped_preamble = archive.preamble();
167
168 let magic = mapped_preamble.magic;
169 let version = mapped_preamble.version;
170 let eof = mapped_preamble.eof_marker;
171
172 assert_eq!(magic, Q42_MAGIC);
173 assert_eq!(version, 1);
174 assert_eq!(eof, 64);
175 }
176}