Skip to main content

qualia_core_db/inference/qwen/
expert_tile.rs

1//! Bounded C:-resident HMC tiles for router-selected Qwen4Exp experts.
2//!
3//! A tile contains exactly the gate, up, and down planes for one `(layer,
4//! expert)` pair.  Promotion is cold-path, explicit, size-admitted, and never
5//! copies a full GGUF or whole layer to the NVMe cache.
6
7use std::fs::File;
8use std::io::{Read, Seek, SeekFrom, Write};
9use std::path::{Path, PathBuf};
10
11
12use crate::bundle::{BundleReader, BundleWriter};
13use crate::gguf_sharder::GgufTensorInfo;
14
15pub use etalon_inference::qwen::expert_tile::{QwenExpertPlane, QwenExpertTileDescriptor, QwenExpertTileAdmission, QwenExpertTileError, QwenExpertTilePlaneKind, QWEN_EXPERT_TILE_FORMAT, QWEN_EXPERT_TILE_SUFFIX};
16const COPY_CHUNK_BYTES: usize = 1024 * 1024;
17pub struct QwenExpertTileReader { inner: etalon_inference::qwen::expert_tile::QwenExpertTileReader }
18impl core::ops::Deref for QwenExpertTileReader {
19 type Target = etalon_inference::qwen::expert_tile::QwenExpertTileReader;
20 fn deref(&self) -> &Self::Target { &self.inner }
21}
22
23fn plane_for_expert(
24    tensor: &GgufTensorInfo,
25    expert: u16,
26) -> Result<QwenExpertPlane, QwenExpertTileError> {
27    if tensor.n_dims != 3 || tensor.dims[0] == 0 || tensor.dims[1] == 0 || tensor.dims[2] == 0 {
28        return Err(QwenExpertTileError::InvalidTensor);
29    }
30    if expert as u64 >= tensor.dims[2] {
31        return Err(QwenExpertTileError::ExpertOutOfRange);
32    }
33    let row_bytes = crate::ggml_quants::ggml_row_bytes(tensor.ggml_type, tensor.dims[0] as usize)
34        .ok_or(QwenExpertTileError::InvalidTensor)? as u64;
35    let bytes = row_bytes
36        .checked_mul(tensor.dims[1])
37        .ok_or(QwenExpertTileError::OffsetOverflow)?;
38    let plane_offset = bytes
39        .checked_mul(expert as u64)
40        .ok_or(QwenExpertTileError::OffsetOverflow)?;
41    let source_byte_offset = tensor
42        .byte_offset
43        .checked_add(plane_offset)
44        .ok_or(QwenExpertTileError::OffsetOverflow)?;
45    Ok(QwenExpertPlane {
46        dims: [tensor.dims[0], tensor.dims[1], 1, 1],
47        n_dims: 2,
48        ggml_type: tensor.ggml_type,
49        source_byte_offset,
50        bytes,
51    })
52}
53
54fn read_plane(
55    source: &mut File,
56    tensor_data_start: u64,
57    plane: QwenExpertPlane,
58) -> Result<Vec<u8>, QwenExpertTileError> {
59    let start = tensor_data_start
60        .checked_add(plane.source_byte_offset)
61        .ok_or(QwenExpertTileError::OffsetOverflow)?;
62    let length = usize::try_from(plane.bytes).map_err(|_| QwenExpertTileError::OffsetOverflow)?;
63    let mut bytes = vec![0u8; length];
64    source
65        .seek(SeekFrom::Start(start))
66        .map_err(|_| QwenExpertTileError::SourceIo)?;
67    // Keep the read bounded even when a future checkpoint uses larger planes.
68    let mut copied = 0usize;
69    while copied < bytes.len() {
70        let end = (copied + COPY_CHUNK_BYTES).min(bytes.len());
71        source
72            .read_exact(&mut bytes[copied..end])
73            .map_err(|_| QwenExpertTileError::SourceIo)?;
74        copied = end;
75    }
76    Ok(bytes)
77}
78
79/// Compute the requested tile's budget admission without creating any files.
80/// Existing cache files are only counted when their name carries the dedicated
81/// tile suffix; unrelated C: artifacts are never scanned or deleted.
82pub fn admit_expert_tile(
83    cache_dir: &Path,
84    requested_bytes: u64,
85    cache_limit_bytes: u64,
86) -> Result<QwenExpertTileAdmission, QwenExpertTileError> {
87    let mut used = 0u64;
88    if cache_dir.exists() {
89        for entry in std::fs::read_dir(cache_dir).map_err(|_| QwenExpertTileError::OutputPath)? {
90            let entry = entry.map_err(|_| QwenExpertTileError::OutputPath)?;
91            let path = entry.path();
92            if path.is_file()
93                && path
94                    .file_name()
95                    .and_then(|name| name.to_str())
96                    .is_some_and(|name| name.ends_with(QWEN_EXPERT_TILE_SUFFIX))
97            {
98                used = used.saturating_add(
99                    entry
100                        .metadata()
101                        .map_err(|_| QwenExpertTileError::OutputPath)?
102                        .len(),
103                );
104            }
105        }
106    }
107    let admission = QwenExpertTileAdmission {
108        cache_used_bytes: used,
109        requested_bytes,
110        cache_limit_bytes,
111    };
112    if requested_bytes > cache_limit_bytes
113        || used
114            .checked_add(requested_bytes)
115            .ok_or(QwenExpertTileError::OffsetOverflow)?
116            > cache_limit_bytes
117    {
118        return Err(QwenExpertTileError::CacheBudgetExceeded);
119    }
120    Ok(admission)
121}
122
123/// Extract one selected expert's quantized source planes into an HMC tile.
124/// The output path must be new.  The caller supplies the C: cache directory
125/// and budget; this function never evicts or creates an unbounded cache.
126#[allow(clippy::too_many_arguments)]
127pub fn promote_expert_tile(
128    source_path: &Path,
129    source_byte_len: u64,
130    tensor_data_start: u64,
131    layer: u16,
132    expert: u16,
133    gate: &GgufTensorInfo,
134    up: &GgufTensorInfo,
135    down: &GgufTensorInfo,
136    cache_dir: &Path,
137    cache_limit_bytes: u64,
138) -> Result<(PathBuf, QwenExpertTileAdmission), QwenExpertTileError> {
139    let gate_plane = plane_for_expert(gate, expert)?;
140    let up_plane = plane_for_expert(up, expert)?;
141    let down_plane = plane_for_expert(down, expert)?;
142    let requested = gate_plane
143        .bytes
144        .checked_add(up_plane.bytes)
145        .and_then(|value| value.checked_add(down_plane.bytes))
146        .ok_or(QwenExpertTileError::OffsetOverflow)?;
147    let admission = admit_expert_tile(cache_dir, requested, cache_limit_bytes)?;
148    std::fs::create_dir_all(cache_dir).map_err(|_| QwenExpertTileError::OutputPath)?;
149    let output = cache_dir.join(format!(
150        "layer-{layer:02}-expert-{expert:03}{QWEN_EXPERT_TILE_SUFFIX}"
151    ));
152    if output.exists() {
153        return Err(QwenExpertTileError::OutputExists);
154    }
155    let source_name = source_path
156        .file_name()
157        .and_then(|name| name.to_str())
158        .ok_or(QwenExpertTileError::OutputPath)?;
159    let descriptor = QwenExpertTileDescriptor {
160        format: QWEN_EXPERT_TILE_FORMAT.to_string(),
161        layer,
162        expert,
163        source_file_name: source_name.to_string(),
164        source_byte_len,
165        gate: gate_plane,
166        up: up_plane,
167        down: down_plane,
168    };
169    let mut descriptor_bytes = Vec::new();
170    ciborium::into_writer(&descriptor, &mut descriptor_bytes)
171        .map_err(|_| QwenExpertTileError::Serialize)?;
172    let mut source = File::open(source_path).map_err(|_| QwenExpertTileError::SourceIo)?;
173    let gate_bytes = read_plane(&mut source, tensor_data_start, gate_plane)?;
174    let up_bytes = read_plane(&mut source, tensor_data_start, up_plane)?;
175    let down_bytes = read_plane(&mut source, tensor_data_start, down_plane)?;
176    let mut bundle = BundleWriter::new();
177    bundle
178        .add_file("descriptor", "cbor", descriptor_bytes, None)
179        .map_err(|_| QwenExpertTileError::Bundle)?;
180    bundle
181        .add_file("gate", "ggml-plane", gate_bytes, None)
182        .map_err(|_| QwenExpertTileError::Bundle)?;
183    bundle
184        .add_file("up", "ggml-plane", up_bytes, None)
185        .map_err(|_| QwenExpertTileError::Bundle)?;
186    bundle
187        .add_file("down", "ggml-plane", down_bytes, None)
188        .map_err(|_| QwenExpertTileError::Bundle)?;
189    let bytes = bundle.build().map_err(|_| QwenExpertTileError::Bundle)?;
190    let mut temporary =
191        tempfile::NamedTempFile::new_in(cache_dir).map_err(|_| QwenExpertTileError::OutputPath)?;
192    if temporary.write_all(&bytes).is_err() || temporary.flush().is_err() {
193        return Err(QwenExpertTileError::Write);
194    }
195    temporary
196        .persist_noclobber(&output)
197        .map_err(|_| QwenExpertTileError::OutputExists)?;
198    Ok((output, admission))
199}
200
201/// Read and validate a compact expert-tile descriptor.  Tensor payloads remain
202/// in the HMC until a caller explicitly opens that small tile for execution.
203pub fn load_expert_tile_descriptor(
204    path: &Path,
205) -> Result<QwenExpertTileDescriptor, QwenExpertTileError> {
206    let bytes = std::fs::read(path).map_err(|_| QwenExpertTileError::InvalidPackage)?;
207    let bundle = BundleReader::parse(&bytes).map_err(|_| QwenExpertTileError::InvalidPackage)?;
208    let descriptor_bytes = bundle
209        .get("descriptor")
210        .ok_or(QwenExpertTileError::InvalidPackage)?;
211    let descriptor: QwenExpertTileDescriptor =
212        ciborium::from_reader(descriptor_bytes).map_err(|_| QwenExpertTileError::InvalidPackage)?;
213    if descriptor.format != QWEN_EXPERT_TILE_FORMAT
214        || descriptor.gate.n_dims != 2
215        || descriptor.up.n_dims != 2
216        || descriptor.down.n_dims != 2
217        || descriptor.gate.bytes == 0
218        || descriptor.up.bytes == 0
219        || descriptor.down.bytes == 0
220    {
221        return Err(QwenExpertTileError::InvalidPackage);
222    }
223    Ok(descriptor)
224}
225
226impl QwenExpertTileReader {
227    /// Opens the resource.
228    pub fn open(path: &Path) -> Result<Self, QwenExpertTileError> {
229        let file = File::open(path).map_err(|_| QwenExpertTileError::InvalidPackage)?;
230        let mapped = unsafe { memmap2::MmapOptions::new().map(&file) }
231            .map_err(|_| QwenExpertTileError::InvalidPackage)?;
232        let bundle =
233            BundleReader::parse(&mapped).map_err(|_| QwenExpertTileError::InvalidPackage)?;
234        let descriptor_bytes = bundle
235            .get("descriptor")
236            .ok_or(QwenExpertTileError::InvalidPackage)?;
237        let descriptor: QwenExpertTileDescriptor = ciborium::from_reader(descriptor_bytes)
238            .map_err(|_| QwenExpertTileError::InvalidPackage)?;
239        if descriptor.format != QWEN_EXPERT_TILE_FORMAT {
240            return Err(QwenExpertTileError::InvalidPackage);
241        }
242        let entry_offset = |key: &str, expected: u64| {
243            let entry = bundle
244                .entry(key)
245                .ok_or(QwenExpertTileError::InvalidPackage)?;
246            if entry.length != expected {
247                return Err(QwenExpertTileError::InvalidPackage);
248            }
249            usize::try_from(entry.offset).map_err(|_| QwenExpertTileError::InvalidPackage)
250        };
251        let gate_offset = entry_offset("gate", descriptor.gate.bytes)?;
252        let up_offset = entry_offset("up", descriptor.up.bytes)?;
253        let down_offset = entry_offset("down", descriptor.down.bytes)?;
254        drop(bundle);
255        Ok(Self { inner: etalon_inference::qwen::expert_tile::QwenExpertTileReader::from_mapping(mapped, descriptor, [gate_offset, up_offset, down_offset])? })
256    }
257
258}
259
260#[cfg(test)]
261mod tests {
262    use super::*;
263
264    #[test]
265    fn expert_plane_is_layer_local_and_byte_exact() {
266        let tensor = GgufTensorInfo {
267            dims: [32, 4, 3, 0],
268            n_dims: 3,
269            ggml_type: crate::ggml_quants::GGML_TYPE_F32,
270            byte_offset: 128,
271        };
272        let second = plane_for_expert(&tensor, 1).unwrap();
273        assert_eq!(second.bytes, 512);
274        assert_eq!(second.source_byte_offset, 640);
275        assert_eq!(second.dims, [32, 4, 1, 1]);
276    }
277
278    #[test]
279    fn cache_admission_never_overcommits() {
280        let directory = tempfile::tempdir().unwrap();
281        assert_eq!(
282            admit_expert_tile(directory.path(), 9, 8).unwrap_err(),
283            QwenExpertTileError::CacheBudgetExceeded
284        );
285    }
286
287    #[test]
288    fn promoted_tile_replays_a_quantized_plane_from_its_hmc_segment() {
289        let source_dir = tempfile::tempdir().unwrap();
290        let source_path = source_dir.path().join("source.gguf");
291        let mut bytes = Vec::new();
292        for value in [
293            1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0,
294        ] {
295            bytes.extend_from_slice(&value.to_le_bytes());
296        }
297        std::fs::write(&source_path, &bytes).unwrap();
298        let plane = |offset| GgufTensorInfo {
299            dims: [2, 2, 1, 0],
300            n_dims: 3,
301            ggml_type: crate::ggml_quants::GGML_TYPE_F32,
302            byte_offset: offset,
303        };
304        let cache_dir = source_dir.path().join("cache");
305        let (path, _) = promote_expert_tile(
306            &source_path,
307            bytes.len() as u64,
308            0,
309            0,
310            0,
311            &plane(0),
312            &plane(16),
313            &plane(32),
314            &cache_dir,
315            1024,
316        )
317        .unwrap();
318        let reader = QwenExpertTileReader::open(&path).unwrap();
319        let mut out = [0.0f32; 2];
320        let mut row = [0.0f32; 2];
321        assert_eq!(
322            reader
323                .gemv_into(
324                    QwenExpertTilePlaneKind::Gate,
325                    &[1.0, 1.0],
326                    &mut out,
327                    &mut row
328                )
329                .unwrap(),
330            2
331        );
332        assert_eq!(out, [3.0, 7.0]);
333    }
334}