Skip to main content

qualia_core_db/inference/moe/
ftw_loader.rs

1//! FTW (FreeToken Weight) format loader and multi-shard memory mapper (Work Package F9/F10).
2//!
3//! Loads `freetoken_weight.json` and memory-maps `.ftw` binary shards, providing
4//! zero-copy access to shared weights and MoE expert banks (256 experts per layer).
5
6use memmap2::Mmap;
7use serde::Deserialize;
8use std::collections::HashMap;
9use std::fs::File;
10use std::path::Path;
11
12use crate::inference::gguf_sharder::{GgufTensorIndex, GgufTensorInfo};
13use crate::inference::tensor_roles::name_to_role;
14use etalon_model::inference::ftw_config::FtwConfig;
15
16#[derive(Debug, Clone, Deserialize)]
17pub struct FtwTensorEntry {
18    pub name: String,
19    pub kind: String,
20    pub dtype: String,
21    pub shape: Vec<u64>,
22    pub global_off: u64,
23    pub nbytes: u64,
24}
25
26#[derive(Debug, Clone, Deserialize)]
27pub struct FtwShardEntry {
28    pub file: String,
29    pub global_off: u64,
30    pub nbytes: u64,
31}
32
33#[derive(Debug, Clone, Deserialize)]
34pub struct FtwManifest {
35    pub format: String,
36    pub version: u32,
37    pub total_bytes: u64,
38    pub tensors: Vec<FtwTensorEntry>,
39    pub shards: Vec<FtwShardEntry>,
40    #[serde(default)]
41    pub quant_format: String,
42    #[serde(default)]
43    pub expert_bank_num_layers: Option<u32>,
44}
45
46/// Slices within the memory-mapped shards for a single expert in a given layer.
47#[derive(Debug, Clone, Copy, Default)]
48pub struct FtwExpertBankSlice {
49    /// Legacy primary shard (gate/up packed). Components can cross shard boundaries.
50    pub shard_idx: usize,
51    /// gate/up packed, scale, global; down packed, scale, global.
52    pub component_shards: [usize; 6],
53    pub gate_up_packed_off: usize,
54    pub gate_up_packed_len: usize,
55    pub gate_up_scale_off: usize,
56    pub gate_up_scale_len: usize,
57    pub gate_up_global_off: usize,
58    pub gate_up_global_len: usize,
59    pub down_packed_off: usize,
60    pub down_packed_len: usize,
61    pub down_scale_off: usize,
62    pub down_scale_len: usize,
63    pub down_global_off: usize,
64    pub down_global_len: usize,
65}
66
67pub use etalon_inference::ordinary::expert_source::FtwExpertData;
68
69use std::sync::Arc;
70
71/// Container owning memory-mapped FTW shards and tensor index.
72#[derive(Clone)]
73pub struct FtwModelPackage {
74    prepared_shards: Option<Vec<crate::gguf_bridge::ModelByteHandle>>,
75    pub manifest: FtwManifest,
76    pub shards: Vec<Arc<Mmap>>,
77    pub tensor_index: GgufTensorIndex,
78    pub expert_slices: HashMap<(u16, u16), FtwExpertBankSlice>,
79    tensor_locations: HashMap<String, (usize, usize, usize)>, // name -> (shard_idx, local_off, len)
80}
81
82#[path = "ftw_loader/prepared.rs"]
83mod prepared;
84
85impl FtwModelPackage {
86    /// Load an FTW model package from a directory containing `freetoken_weight.json` and `.ftw` shards.
87    pub fn open_from_dir(dir: &Path) -> Result<Self, String> {
88        let manifest_path = dir.join("freetoken_weight.json");
89        if !manifest_path.is_file() {
90            return Err(format!(
91                "FTW manifest not found at {}",
92                manifest_path.display()
93            ));
94        }
95        let manifest_bytes = std::fs::read(&manifest_path)
96            .map_err(|e| format!("Failed to read {}: {}", manifest_path.display(), e))?;
97        let manifest: FtwManifest = serde_json::from_slice(&manifest_bytes)
98            .map_err(|e| format!("Failed to parse FTW manifest: {}", e))?;
99
100        // Open and mmap each shard
101        let mut shards = Vec::with_capacity(manifest.shards.len());
102        for s in &manifest.shards {
103            let shard_path = dir.join(&s.file);
104            let file = File::open(&shard_path)
105                .map_err(|e| format!("Failed to open shard {}: {}", shard_path.display(), e))?;
106            let mmap = unsafe { Mmap::map(&file) }
107                .map_err(|e| format!("Failed to mmap shard {}: {}", shard_path.display(), e))?;
108            shards.push(Arc::new(mmap));
109        }
110
111        let config_path = dir.join("config.json");
112        let parsed_cfg: FtwConfig = if config_path.is_file() {
113            let bytes = std::fs::read(&config_path)
114                .map_err(|e| format!("Failed to read FTW config: {e}"))?;
115            serde_json::from_slice(&bytes)
116                .map_err(|e| format!("Failed to parse FTW config: {e}"))?
117        } else {
118            FtwConfig::default()
119        };
120        let mut tensor_locations = HashMap::new();
121        let mut named_tensors = Vec::new();
122        let mut bank_tensors: HashMap<String, (usize, usize, usize)> = HashMap::new();
123
124        let mut inferred_layers = 0u32;
125        let mut inferred_embd = 0u32;
126
127        for t in &manifest.tensors {
128            // Find which shard contains t.global_off
129            let mut shard_idx = None;
130            let mut local_off = 0usize;
131            for (idx, s) in manifest.shards.iter().enumerate() {
132                let start = s.global_off;
133                let end = start + s.nbytes;
134                if t.global_off >= start && t.global_off < end {
135                    shard_idx = Some(idx);
136                    local_off = (t.global_off - start) as usize;
137                    break;
138                }
139            }
140            let s_idx = match shard_idx {
141                Some(i) => i,
142                None => continue,
143            };
144            let len = t.nbytes as usize;
145
146            if t.kind == "experts_bank" {
147                bank_tensors.insert(t.name.clone(), (s_idx, local_off, len));
148                continue;
149            }
150
151            tensor_locations.insert(t.name.clone(), (s_idx, local_off, len));
152
153            let ggml_type = match t.dtype.as_str() {
154                "bfloat16" | "bf16" => crate::ggml_quants::GGML_TYPE_BF16,
155                "float16" | "f16" => crate::ggml_quants::GGML_TYPE_F16,
156                "float32" | "f32" => crate::ggml_quants::GGML_TYPE_F32,
157                "float8_e4m3fn" | "fp8_e4m3" => crate::ggml_quants::GGML_TYPE_FP8_E4M3,
158                "uint8" => crate::ggml_quants::GGML_TYPE_Q8_0,
159                _ => crate::ggml_quants::GGML_TYPE_Q8_0,
160            };
161
162            let mut dims = [0u64; 4];
163            for (i, &d) in t.shape.iter().rev().enumerate().take(4) {
164                dims[i] = d;
165            }
166            let n_dims = t.shape.len().min(4) as u32;
167
168            let info = GgufTensorInfo {
169                dims,
170                n_dims,
171                ggml_type,
172                byte_offset: t.global_off,
173            };
174
175            // Canonicalize to standard GGUF tensor names for QTensorEngine compatibility
176            if let Some(r) = name_to_role(&t.name) {
177                if r.layer == crate::p64_weight::P64_LAYER_GLOBAL {
178                    match r.role {
179                        crate::p64_weight::P64_ROLE_TOKEN_EMBD => {
180                            if t.shape.len() >= 2 {
181                                inferred_embd = u32::try_from(t.shape[1])
182                                    .map_err(|_| "FTW embedding width exceeds u32")?;
183                            }
184                            named_tensors.push((b"token_embd.weight".to_vec(), info));
185                            tensor_locations
186                                .insert("token_embd.weight".to_string(), (s_idx, local_off, len));
187                        }
188                        crate::p64_weight::P64_ROLE_OUTPUT => {
189                            named_tensors.push((b"output.weight".to_vec(), info));
190                            tensor_locations
191                                .insert("output.weight".to_string(), (s_idx, local_off, len));
192                        }
193                        crate::p64_weight::P64_ROLE_OUTPUT_NORM => {
194                            named_tensors.push((b"output_norm.weight".to_vec(), info));
195                            tensor_locations
196                                .insert("output_norm.weight".to_string(), (s_idx, local_off, len));
197                        }
198                        _ => {}
199                    }
200                } else {
201                    inferred_layers = inferred_layers.max(u32::from(r.layer) + 1);
202                    if let Some(suffix) = crate::inference::safetensor_loader::role_suffix(r.role) {
203                        let mut buf = [0u8; 96];
204                        let blk_len = crate::gguf_sharder::write_blk_tensor_name(
205                            r.layer as u32,
206                            suffix,
207                            &mut buf,
208                        );
209                        if blk_len > 0 {
210                            named_tensors.push((buf[..blk_len].to_vec(), info));
211                            if let Ok(c_str) = std::str::from_utf8(&buf[..blk_len]) {
212                                tensor_locations.insert(c_str.to_string(), (s_idx, local_off, len));
213                            }
214                        }
215                    }
216                }
217            }
218
219            // Always retain the original HuggingFace / FreeToken tensor name
220            named_tensors.push((t.name.as_bytes().to_vec(), info));
221
222            if t.name.ends_with("embed_tokens.weight") && t.shape.len() >= 2 {
223                inferred_embd =
224                    u32::try_from(t.shape[1]).map_err(|_| "FTW embedding width exceeds u32")?;
225            }
226        }
227
228        // Build expert bank slices per (layer, expert_id)
229        let num_layers = manifest
230            .expert_bank_num_layers
231            .unwrap_or(inferred_layers)
232            .max(1);
233        let mut expert_slices = HashMap::new();
234
235        for layer in 0..num_layers {
236            let layer_tag = format!("#L{:05}", layer);
237            let gup_key = format!("gate_up_packed{}", layer_tag);
238            let gus_key = format!("gate_up_scale{}", layer_tag);
239            let gug_key = format!("gate_up_global{}", layer_tag);
240            let dnp_key = format!("down_packed{}", layer_tag);
241            let dns_key = format!("down_scale{}", layer_tag);
242            let dng_key = format!("down_global{}", layer_tag);
243
244            if let (Some(&gup), Some(&gus), Some(&gug), Some(&dnp), Some(&dns), Some(&dng)) = (
245                bank_tensors.get(&gup_key),
246                bank_tensors.get(&gus_key),
247                bank_tensors.get(&gug_key),
248                bank_tensors.get(&dnp_key),
249                bank_tensors.get(&dns_key),
250                bank_tensors.get(&dng_key),
251            ) {
252                // 256 experts per layer
253                let num_experts = 256usize;
254                let gup_step = gup.2 / num_experts;
255                let gus_step = gus.2 / num_experts;
256                let gug_step = gug.2 / num_experts;
257                let dnp_step = dnp.2 / num_experts;
258                let dns_step = dns.2 / num_experts;
259                let dng_step = dng.2 / num_experts;
260
261                for e in 0..num_experts {
262                    let slice = FtwExpertBankSlice {
263                        shard_idx: gup.0,
264                        component_shards: [gup.0, gus.0, gug.0, dnp.0, dns.0, dng.0],
265                        gate_up_packed_off: gup.1 + e * gup_step,
266                        gate_up_packed_len: gup_step,
267                        gate_up_scale_off: gus.1 + e * gus_step,
268                        gate_up_scale_len: gus_step,
269                        gate_up_global_off: gug.1 + e * gug_step,
270                        gate_up_global_len: gug_step,
271                        down_packed_off: dnp.1 + e * dnp_step,
272                        down_packed_len: dnp_step,
273                        down_scale_off: dns.1 + e * dns_step,
274                        down_scale_len: dns_step,
275                        down_global_off: dng.1 + e * dng_step,
276                        down_global_len: dng_step,
277                    };
278                    expert_slices.insert((layer as u16, e as u16), slice);
279                }
280            }
281        }
282
283        let has_hybrid_tensors = manifest
284            .tensors
285            .iter()
286            .any(|t| t.name.contains(".linear_attn.") || t.name.contains(".ssm_"));
287        let hyperparams = parsed_cfg
288            .hyperparams(inferred_layers, inferred_embd, has_hybrid_tensors)?
289            .into();
290        let references: Vec<(&[u8], GgufTensorInfo)> = named_tensors
291            .iter()
292            .map(|(name, info)| (name.as_slice(), *info))
293            .collect();
294
295        let mut tensor_index = GgufTensorIndex::from_components(&references, hyperparams, 0);
296        tensor_index.moe_routing = parsed_cfg.moe_routing()?;
297
298        Ok(Self {
299            prepared_shards: None,
300            manifest,
301            shards,
302            tensor_index,
303            expert_slices,
304            tensor_locations,
305        })
306    }
307
308    /// Read raw bytes for a shared weight tensor by name.
309    pub fn fetch_tensor_bytes(&self, name: &str) -> Option<&[u8]> {
310        let &(shard_idx, off, len) = self.tensor_locations.get(name)?;
311        let shard = self.shard_bytes(shard_idx)?;
312        shard.get(off..off + len)
313    }
314
315    /// Read raw bytes for a tensor by global offset and length across shards.
316    pub fn fetch_by_global_offset(&self, global_off: u64, len: usize) -> Option<&[u8]> {
317        let end = global_off.checked_add(u64::try_from(len).ok()?)?;
318        for (idx, s) in self.manifest.shards.iter().enumerate() {
319            if global_off >= s.global_off && end <= s.global_off.checked_add(s.nbytes)? {
320                let local_off = usize::try_from(global_off.checked_sub(s.global_off)?).ok()?;
321                let shard = self.shard_bytes(idx)?;
322                return shard.get(local_off..local_off.checked_add(len)?);
323            }
324        }
325        None
326    }
327
328    /// Get zero-copy byte slices for one expert in a given layer.
329    pub fn get_expert_data(&self, layer: u16, expert_idx: u16) -> Option<FtwExpertData<'_>> {
330        let slice = self.expert_slices.get(&(layer, expert_idx))?;
331        let component = |index: usize, off: usize, len: usize| {
332            self.shard_bytes(slice.component_shards[index])?
333                .get(off..off.checked_add(len)?)
334        };
335
336        Some(FtwExpertData {
337            gate_up_packed: component(0, slice.gate_up_packed_off, slice.gate_up_packed_len)?,
338            gate_up_scale: component(1, slice.gate_up_scale_off, slice.gate_up_scale_len)?,
339            gate_up_global: component(2, slice.gate_up_global_off, slice.gate_up_global_len)?,
340            down_packed: component(3, slice.down_packed_off, slice.down_packed_len)?,
341            down_scale: component(4, slice.down_scale_off, slice.down_scale_len)?,
342            down_global: component(5, slice.down_global_off, slice.down_global_len)?,
343        })
344    }
345}
346
347#[cfg(test)]
348#[path = "ftw_loader/shard_tests.rs"]
349mod shard_tests;
350#[cfg(test)]
351mod tests {
352    use super::*;
353
354    #[test]
355    fn test_ftw_manifest_parsing() {
356        let raw = r#"{
357            "format": "freetoken_weight",
358            "version": 1,
359            "total_bytes": 1024,
360            "tensors": [
361                { "name": "model.embed_tokens.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10, 64], "global_off": 0, "nbytes": 1280 }
362            ],
363            "shards": [
364                { "file": "freetoken-00000.ftw", "global_off": 0, "nbytes": 1280 }
365            ]
366        }"#;
367
368        let manifest: FtwManifest = serde_json::from_str(raw).expect("manifest parsed");
369        assert_eq!(manifest.format, "freetoken_weight");
370        assert_eq!(manifest.tensors.len(), 1);
371        assert_eq!(manifest.shards.len(), 1);
372    }
373
374    #[test]
375    fn test_ftw_synthetic_package_open() {
376        let temp_dir = tempfile::tempdir().unwrap();
377        let ftw_path = temp_dir.path().join("freetoken-00000.ftw");
378        let dummy_bytes = vec![0u8; 1280];
379        std::fs::write(&ftw_path, &dummy_bytes).unwrap();
380
381        let manifest_json = r#"{
382            "format": "freetoken_weight",
383            "version": 1,
384            "total_bytes": 1280,
385            "tensors": [
386                { "name": "model.embed_tokens.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10, 64], "global_off": 0, "nbytes": 1280 }
387            ],
388            "shards": [
389                { "file": "freetoken-00000.ftw", "global_off": 0, "nbytes": 1280 }
390            ]
391        }"#;
392        std::fs::write(temp_dir.path().join("freetoken_weight.json"), manifest_json).unwrap();
393
394        let config_json = r#"{
395            "model_type": "qwen2",
396            "text_config": {
397                "hidden_size": 64,
398                "num_hidden_layers": 40,
399                "num_attention_heads": 8,
400                "num_key_value_heads": 2
401            }
402        }"#;
403        std::fs::write(temp_dir.path().join("config.json"), config_json).unwrap();
404
405        let pkg = FtwModelPackage::open_from_dir(temp_dir.path()).expect("package open");
406        assert_eq!(pkg.tensor_index.emb_dim(), 64);
407        assert_eq!(pkg.tensor_index.hyperparams.n_layer, 40);
408        assert_eq!(pkg.tensor_index.hyperparams.n_head, 8);
409        assert_eq!(pkg.tensor_index.hyperparams.n_kv_head, 2);
410        assert!(pkg.fetch_tensor_bytes("token_embd.weight").is_some());
411        assert!(pkg
412            .fetch_tensor_bytes("model.embed_tokens.weight")
413            .is_some());
414
415        for config in [
416            "{",
417            r#"{"model_type":"unsupported_hybrid"}"#,
418            r#"{"model_type":"qwen3_5_moe","hidden_size":64}"#,
419        ] {
420            std::fs::write(temp_dir.path().join("config.json"), config).unwrap();
421            assert!(FtwModelPackage::open_from_dir(temp_dir.path()).is_err());
422        }
423    }
424
425    #[test]
426    fn test_ftw_physical_qwen_package_if_present() {
427        let p = Path::new(r#"E:\LLM_Models\Qwen3.6-35B-A3B-NVFP4"#);
428        if !p.exists() || !p.join("freetoken_weight.json").exists() {
429            return;
430        }
431
432        let pkg = FtwModelPackage::open_from_dir(p).expect("real package open");
433        assert_eq!(pkg.manifest.format, "freetoken_weight");
434        assert!(!pkg.shards.is_empty());
435        assert!(pkg.tensor_index.emb_dim() > 0);
436
437        let exp_0_0 = pkg.get_expert_data(0, 0);
438        assert!(exp_0_0.is_some(), "layer 0 expert 0 data present");
439        let exp_data = exp_0_0.unwrap();
440        assert!(!exp_data.gate_up_packed.is_empty());
441        assert!(!exp_data.down_packed.is_empty());
442
443        let view = exp_data.to_view(pkg.tensor_index.emb_dim() as usize, 1024);
444        assert!(!view.gate_up_packed.is_empty());
445        assert!(!view.down_packed.is_empty());
446    }
447}
448
449impl etalon_inference::ordinary::expert_source::ExpertSource for FtwModelPackage {
450    fn fetch_tensor_bytes(&self, name: &str) -> Option<&[u8]> {
451        FtwModelPackage::fetch_tensor_bytes(self, name)
452    }
453    fn get_expert_data(&self, layer: u16, expert: u16) -> Option<FtwExpertData<'_>> {
454        FtwModelPackage::get_expert_data(self, layer, expert)
455    }
456}