qualia_core_db/inference/qwen/
expert_tile.rs1use std::fs::File;
8use std::io::{Read, Seek, SeekFrom, Write};
9use std::path::{Path, PathBuf};
10
11
12use crate::bundle::{BundleReader, BundleWriter};
13use crate::gguf_sharder::GgufTensorInfo;
14
15pub use etalon_inference::qwen::expert_tile::{QwenExpertPlane, QwenExpertTileDescriptor, QwenExpertTileAdmission, QwenExpertTileError, QwenExpertTilePlaneKind, QWEN_EXPERT_TILE_FORMAT, QWEN_EXPERT_TILE_SUFFIX};
16const COPY_CHUNK_BYTES: usize = 1024 * 1024;
17pub struct QwenExpertTileReader { inner: etalon_inference::qwen::expert_tile::QwenExpertTileReader }
18impl core::ops::Deref for QwenExpertTileReader {
19 type Target = etalon_inference::qwen::expert_tile::QwenExpertTileReader;
20 fn deref(&self) -> &Self::Target { &self.inner }
21}
22
23fn plane_for_expert(
24 tensor: &GgufTensorInfo,
25 expert: u16,
26) -> Result<QwenExpertPlane, QwenExpertTileError> {
27 if tensor.n_dims != 3 || tensor.dims[0] == 0 || tensor.dims[1] == 0 || tensor.dims[2] == 0 {
28 return Err(QwenExpertTileError::InvalidTensor);
29 }
30 if expert as u64 >= tensor.dims[2] {
31 return Err(QwenExpertTileError::ExpertOutOfRange);
32 }
33 let row_bytes = crate::ggml_quants::ggml_row_bytes(tensor.ggml_type, tensor.dims[0] as usize)
34 .ok_or(QwenExpertTileError::InvalidTensor)? as u64;
35 let bytes = row_bytes
36 .checked_mul(tensor.dims[1])
37 .ok_or(QwenExpertTileError::OffsetOverflow)?;
38 let plane_offset = bytes
39 .checked_mul(expert as u64)
40 .ok_or(QwenExpertTileError::OffsetOverflow)?;
41 let source_byte_offset = tensor
42 .byte_offset
43 .checked_add(plane_offset)
44 .ok_or(QwenExpertTileError::OffsetOverflow)?;
45 Ok(QwenExpertPlane {
46 dims: [tensor.dims[0], tensor.dims[1], 1, 1],
47 n_dims: 2,
48 ggml_type: tensor.ggml_type,
49 source_byte_offset,
50 bytes,
51 })
52}
53
54fn read_plane(
55 source: &mut File,
56 tensor_data_start: u64,
57 plane: QwenExpertPlane,
58) -> Result<Vec<u8>, QwenExpertTileError> {
59 let start = tensor_data_start
60 .checked_add(plane.source_byte_offset)
61 .ok_or(QwenExpertTileError::OffsetOverflow)?;
62 let length = usize::try_from(plane.bytes).map_err(|_| QwenExpertTileError::OffsetOverflow)?;
63 let mut bytes = vec![0u8; length];
64 source
65 .seek(SeekFrom::Start(start))
66 .map_err(|_| QwenExpertTileError::SourceIo)?;
67 let mut copied = 0usize;
69 while copied < bytes.len() {
70 let end = (copied + COPY_CHUNK_BYTES).min(bytes.len());
71 source
72 .read_exact(&mut bytes[copied..end])
73 .map_err(|_| QwenExpertTileError::SourceIo)?;
74 copied = end;
75 }
76 Ok(bytes)
77}
78
79pub fn admit_expert_tile(
83 cache_dir: &Path,
84 requested_bytes: u64,
85 cache_limit_bytes: u64,
86) -> Result<QwenExpertTileAdmission, QwenExpertTileError> {
87 let mut used = 0u64;
88 if cache_dir.exists() {
89 for entry in std::fs::read_dir(cache_dir).map_err(|_| QwenExpertTileError::OutputPath)? {
90 let entry = entry.map_err(|_| QwenExpertTileError::OutputPath)?;
91 let path = entry.path();
92 if path.is_file()
93 && path
94 .file_name()
95 .and_then(|name| name.to_str())
96 .is_some_and(|name| name.ends_with(QWEN_EXPERT_TILE_SUFFIX))
97 {
98 used = used.saturating_add(
99 entry
100 .metadata()
101 .map_err(|_| QwenExpertTileError::OutputPath)?
102 .len(),
103 );
104 }
105 }
106 }
107 let admission = QwenExpertTileAdmission {
108 cache_used_bytes: used,
109 requested_bytes,
110 cache_limit_bytes,
111 };
112 if requested_bytes > cache_limit_bytes
113 || used
114 .checked_add(requested_bytes)
115 .ok_or(QwenExpertTileError::OffsetOverflow)?
116 > cache_limit_bytes
117 {
118 return Err(QwenExpertTileError::CacheBudgetExceeded);
119 }
120 Ok(admission)
121}
122
123#[allow(clippy::too_many_arguments)]
127pub fn promote_expert_tile(
128 source_path: &Path,
129 source_byte_len: u64,
130 tensor_data_start: u64,
131 layer: u16,
132 expert: u16,
133 gate: &GgufTensorInfo,
134 up: &GgufTensorInfo,
135 down: &GgufTensorInfo,
136 cache_dir: &Path,
137 cache_limit_bytes: u64,
138) -> Result<(PathBuf, QwenExpertTileAdmission), QwenExpertTileError> {
139 let gate_plane = plane_for_expert(gate, expert)?;
140 let up_plane = plane_for_expert(up, expert)?;
141 let down_plane = plane_for_expert(down, expert)?;
142 let requested = gate_plane
143 .bytes
144 .checked_add(up_plane.bytes)
145 .and_then(|value| value.checked_add(down_plane.bytes))
146 .ok_or(QwenExpertTileError::OffsetOverflow)?;
147 let admission = admit_expert_tile(cache_dir, requested, cache_limit_bytes)?;
148 std::fs::create_dir_all(cache_dir).map_err(|_| QwenExpertTileError::OutputPath)?;
149 let output = cache_dir.join(format!(
150 "layer-{layer:02}-expert-{expert:03}{QWEN_EXPERT_TILE_SUFFIX}"
151 ));
152 if output.exists() {
153 return Err(QwenExpertTileError::OutputExists);
154 }
155 let source_name = source_path
156 .file_name()
157 .and_then(|name| name.to_str())
158 .ok_or(QwenExpertTileError::OutputPath)?;
159 let descriptor = QwenExpertTileDescriptor {
160 format: QWEN_EXPERT_TILE_FORMAT.to_string(),
161 layer,
162 expert,
163 source_file_name: source_name.to_string(),
164 source_byte_len,
165 gate: gate_plane,
166 up: up_plane,
167 down: down_plane,
168 };
169 let mut descriptor_bytes = Vec::new();
170 ciborium::into_writer(&descriptor, &mut descriptor_bytes)
171 .map_err(|_| QwenExpertTileError::Serialize)?;
172 let mut source = File::open(source_path).map_err(|_| QwenExpertTileError::SourceIo)?;
173 let gate_bytes = read_plane(&mut source, tensor_data_start, gate_plane)?;
174 let up_bytes = read_plane(&mut source, tensor_data_start, up_plane)?;
175 let down_bytes = read_plane(&mut source, tensor_data_start, down_plane)?;
176 let mut bundle = BundleWriter::new();
177 bundle
178 .add_file("descriptor", "cbor", descriptor_bytes, None)
179 .map_err(|_| QwenExpertTileError::Bundle)?;
180 bundle
181 .add_file("gate", "ggml-plane", gate_bytes, None)
182 .map_err(|_| QwenExpertTileError::Bundle)?;
183 bundle
184 .add_file("up", "ggml-plane", up_bytes, None)
185 .map_err(|_| QwenExpertTileError::Bundle)?;
186 bundle
187 .add_file("down", "ggml-plane", down_bytes, None)
188 .map_err(|_| QwenExpertTileError::Bundle)?;
189 let bytes = bundle.build().map_err(|_| QwenExpertTileError::Bundle)?;
190 let mut temporary =
191 tempfile::NamedTempFile::new_in(cache_dir).map_err(|_| QwenExpertTileError::OutputPath)?;
192 if temporary.write_all(&bytes).is_err() || temporary.flush().is_err() {
193 return Err(QwenExpertTileError::Write);
194 }
195 temporary
196 .persist_noclobber(&output)
197 .map_err(|_| QwenExpertTileError::OutputExists)?;
198 Ok((output, admission))
199}
200
201pub fn load_expert_tile_descriptor(
204 path: &Path,
205) -> Result<QwenExpertTileDescriptor, QwenExpertTileError> {
206 let bytes = std::fs::read(path).map_err(|_| QwenExpertTileError::InvalidPackage)?;
207 let bundle = BundleReader::parse(&bytes).map_err(|_| QwenExpertTileError::InvalidPackage)?;
208 let descriptor_bytes = bundle
209 .get("descriptor")
210 .ok_or(QwenExpertTileError::InvalidPackage)?;
211 let descriptor: QwenExpertTileDescriptor =
212 ciborium::from_reader(descriptor_bytes).map_err(|_| QwenExpertTileError::InvalidPackage)?;
213 if descriptor.format != QWEN_EXPERT_TILE_FORMAT
214 || descriptor.gate.n_dims != 2
215 || descriptor.up.n_dims != 2
216 || descriptor.down.n_dims != 2
217 || descriptor.gate.bytes == 0
218 || descriptor.up.bytes == 0
219 || descriptor.down.bytes == 0
220 {
221 return Err(QwenExpertTileError::InvalidPackage);
222 }
223 Ok(descriptor)
224}
225
226impl QwenExpertTileReader {
227 pub fn open(path: &Path) -> Result<Self, QwenExpertTileError> {
229 let file = File::open(path).map_err(|_| QwenExpertTileError::InvalidPackage)?;
230 let mapped = unsafe { memmap2::MmapOptions::new().map(&file) }
231 .map_err(|_| QwenExpertTileError::InvalidPackage)?;
232 let bundle =
233 BundleReader::parse(&mapped).map_err(|_| QwenExpertTileError::InvalidPackage)?;
234 let descriptor_bytes = bundle
235 .get("descriptor")
236 .ok_or(QwenExpertTileError::InvalidPackage)?;
237 let descriptor: QwenExpertTileDescriptor = ciborium::from_reader(descriptor_bytes)
238 .map_err(|_| QwenExpertTileError::InvalidPackage)?;
239 if descriptor.format != QWEN_EXPERT_TILE_FORMAT {
240 return Err(QwenExpertTileError::InvalidPackage);
241 }
242 let entry_offset = |key: &str, expected: u64| {
243 let entry = bundle
244 .entry(key)
245 .ok_or(QwenExpertTileError::InvalidPackage)?;
246 if entry.length != expected {
247 return Err(QwenExpertTileError::InvalidPackage);
248 }
249 usize::try_from(entry.offset).map_err(|_| QwenExpertTileError::InvalidPackage)
250 };
251 let gate_offset = entry_offset("gate", descriptor.gate.bytes)?;
252 let up_offset = entry_offset("up", descriptor.up.bytes)?;
253 let down_offset = entry_offset("down", descriptor.down.bytes)?;
254 drop(bundle);
255 Ok(Self { inner: etalon_inference::qwen::expert_tile::QwenExpertTileReader::from_mapping(mapped, descriptor, [gate_offset, up_offset, down_offset])? })
256 }
257
258}
259
260#[cfg(test)]
261mod tests {
262 use super::*;
263
264 #[test]
265 fn expert_plane_is_layer_local_and_byte_exact() {
266 let tensor = GgufTensorInfo {
267 dims: [32, 4, 3, 0],
268 n_dims: 3,
269 ggml_type: crate::ggml_quants::GGML_TYPE_F32,
270 byte_offset: 128,
271 };
272 let second = plane_for_expert(&tensor, 1).unwrap();
273 assert_eq!(second.bytes, 512);
274 assert_eq!(second.source_byte_offset, 640);
275 assert_eq!(second.dims, [32, 4, 1, 1]);
276 }
277
278 #[test]
279 fn cache_admission_never_overcommits() {
280 let directory = tempfile::tempdir().unwrap();
281 assert_eq!(
282 admit_expert_tile(directory.path(), 9, 8).unwrap_err(),
283 QwenExpertTileError::CacheBudgetExceeded
284 );
285 }
286
287 #[test]
288 fn promoted_tile_replays_a_quantized_plane_from_its_hmc_segment() {
289 let source_dir = tempfile::tempdir().unwrap();
290 let source_path = source_dir.path().join("source.gguf");
291 let mut bytes = Vec::new();
292 for value in [
293 1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0,
294 ] {
295 bytes.extend_from_slice(&value.to_le_bytes());
296 }
297 std::fs::write(&source_path, &bytes).unwrap();
298 let plane = |offset| GgufTensorInfo {
299 dims: [2, 2, 1, 0],
300 n_dims: 3,
301 ggml_type: crate::ggml_quants::GGML_TYPE_F32,
302 byte_offset: offset,
303 };
304 let cache_dir = source_dir.path().join("cache");
305 let (path, _) = promote_expert_tile(
306 &source_path,
307 bytes.len() as u64,
308 0,
309 0,
310 0,
311 &plane(0),
312 &plane(16),
313 &plane(32),
314 &cache_dir,
315 1024,
316 )
317 .unwrap();
318 let reader = QwenExpertTileReader::open(&path).unwrap();
319 let mut out = [0.0f32; 2];
320 let mut row = [0.0f32; 2];
321 assert_eq!(
322 reader
323 .gemv_into(
324 QwenExpertTilePlaneKind::Gate,
325 &[1.0, 1.0],
326 &mut out,
327 &mut row
328 )
329 .unwrap(),
330 2
331 );
332 assert_eq!(out, [3.0, 7.0]);
333 }
334}