1use memmap2::Mmap;
7use serde::Deserialize;
8use std::collections::HashMap;
9use std::fs::File;
10use std::path::Path;
11
12use crate::inference::gguf_sharder::{GgufTensorIndex, GgufTensorInfo};
13use crate::inference::tensor_roles::name_to_role;
14use etalon_model::inference::ftw_config::FtwConfig;
15
16#[derive(Debug, Clone, Deserialize)]
17pub struct FtwTensorEntry {
18 pub name: String,
19 pub kind: String,
20 pub dtype: String,
21 pub shape: Vec<u64>,
22 pub global_off: u64,
23 pub nbytes: u64,
24}
25
26#[derive(Debug, Clone, Deserialize)]
27pub struct FtwShardEntry {
28 pub file: String,
29 pub global_off: u64,
30 pub nbytes: u64,
31}
32
33#[derive(Debug, Clone, Deserialize)]
34pub struct FtwManifest {
35 pub format: String,
36 pub version: u32,
37 pub total_bytes: u64,
38 pub tensors: Vec<FtwTensorEntry>,
39 pub shards: Vec<FtwShardEntry>,
40 #[serde(default)]
41 pub quant_format: String,
42 #[serde(default)]
43 pub expert_bank_num_layers: Option<u32>,
44}
45
46#[derive(Debug, Clone, Copy, Default)]
48pub struct FtwExpertBankSlice {
49 pub shard_idx: usize,
51 pub component_shards: [usize; 6],
53 pub gate_up_packed_off: usize,
54 pub gate_up_packed_len: usize,
55 pub gate_up_scale_off: usize,
56 pub gate_up_scale_len: usize,
57 pub gate_up_global_off: usize,
58 pub gate_up_global_len: usize,
59 pub down_packed_off: usize,
60 pub down_packed_len: usize,
61 pub down_scale_off: usize,
62 pub down_scale_len: usize,
63 pub down_global_off: usize,
64 pub down_global_len: usize,
65}
66
67pub use etalon_inference::ordinary::expert_source::FtwExpertData;
68
69use std::sync::Arc;
70
71#[derive(Clone)]
73pub struct FtwModelPackage {
74 prepared_shards: Option<Vec<crate::gguf_bridge::ModelByteHandle>>,
75 pub manifest: FtwManifest,
76 pub shards: Vec<Arc<Mmap>>,
77 pub tensor_index: GgufTensorIndex,
78 pub expert_slices: HashMap<(u16, u16), FtwExpertBankSlice>,
79 tensor_locations: HashMap<String, (usize, usize, usize)>, }
81
82#[path = "ftw_loader/prepared.rs"]
83mod prepared;
84
85impl FtwModelPackage {
86 pub fn open_from_dir(dir: &Path) -> Result<Self, String> {
88 let manifest_path = dir.join("freetoken_weight.json");
89 if !manifest_path.is_file() {
90 return Err(format!(
91 "FTW manifest not found at {}",
92 manifest_path.display()
93 ));
94 }
95 let manifest_bytes = std::fs::read(&manifest_path)
96 .map_err(|e| format!("Failed to read {}: {}", manifest_path.display(), e))?;
97 let manifest: FtwManifest = serde_json::from_slice(&manifest_bytes)
98 .map_err(|e| format!("Failed to parse FTW manifest: {}", e))?;
99
100 let mut shards = Vec::with_capacity(manifest.shards.len());
102 for s in &manifest.shards {
103 let shard_path = dir.join(&s.file);
104 let file = File::open(&shard_path)
105 .map_err(|e| format!("Failed to open shard {}: {}", shard_path.display(), e))?;
106 let mmap = unsafe { Mmap::map(&file) }
107 .map_err(|e| format!("Failed to mmap shard {}: {}", shard_path.display(), e))?;
108 shards.push(Arc::new(mmap));
109 }
110
111 let config_path = dir.join("config.json");
112 let parsed_cfg: FtwConfig = if config_path.is_file() {
113 let bytes = std::fs::read(&config_path)
114 .map_err(|e| format!("Failed to read FTW config: {e}"))?;
115 serde_json::from_slice(&bytes)
116 .map_err(|e| format!("Failed to parse FTW config: {e}"))?
117 } else {
118 FtwConfig::default()
119 };
120 let mut tensor_locations = HashMap::new();
121 let mut named_tensors = Vec::new();
122 let mut bank_tensors: HashMap<String, (usize, usize, usize)> = HashMap::new();
123
124 let mut inferred_layers = 0u32;
125 let mut inferred_embd = 0u32;
126
127 for t in &manifest.tensors {
128 let mut shard_idx = None;
130 let mut local_off = 0usize;
131 for (idx, s) in manifest.shards.iter().enumerate() {
132 let start = s.global_off;
133 let end = start + s.nbytes;
134 if t.global_off >= start && t.global_off < end {
135 shard_idx = Some(idx);
136 local_off = (t.global_off - start) as usize;
137 break;
138 }
139 }
140 let s_idx = match shard_idx {
141 Some(i) => i,
142 None => continue,
143 };
144 let len = t.nbytes as usize;
145
146 if t.kind == "experts_bank" {
147 bank_tensors.insert(t.name.clone(), (s_idx, local_off, len));
148 continue;
149 }
150
151 tensor_locations.insert(t.name.clone(), (s_idx, local_off, len));
152
153 let ggml_type = match t.dtype.as_str() {
154 "bfloat16" | "bf16" => crate::ggml_quants::GGML_TYPE_BF16,
155 "float16" | "f16" => crate::ggml_quants::GGML_TYPE_F16,
156 "float32" | "f32" => crate::ggml_quants::GGML_TYPE_F32,
157 "float8_e4m3fn" | "fp8_e4m3" => crate::ggml_quants::GGML_TYPE_FP8_E4M3,
158 "uint8" => crate::ggml_quants::GGML_TYPE_Q8_0,
159 _ => crate::ggml_quants::GGML_TYPE_Q8_0,
160 };
161
162 let mut dims = [0u64; 4];
163 for (i, &d) in t.shape.iter().rev().enumerate().take(4) {
164 dims[i] = d;
165 }
166 let n_dims = t.shape.len().min(4) as u32;
167
168 let info = GgufTensorInfo {
169 dims,
170 n_dims,
171 ggml_type,
172 byte_offset: t.global_off,
173 };
174
175 if let Some(r) = name_to_role(&t.name) {
177 if r.layer == crate::p64_weight::P64_LAYER_GLOBAL {
178 match r.role {
179 crate::p64_weight::P64_ROLE_TOKEN_EMBD => {
180 if t.shape.len() >= 2 {
181 inferred_embd = u32::try_from(t.shape[1])
182 .map_err(|_| "FTW embedding width exceeds u32")?;
183 }
184 named_tensors.push((b"token_embd.weight".to_vec(), info));
185 tensor_locations
186 .insert("token_embd.weight".to_string(), (s_idx, local_off, len));
187 }
188 crate::p64_weight::P64_ROLE_OUTPUT => {
189 named_tensors.push((b"output.weight".to_vec(), info));
190 tensor_locations
191 .insert("output.weight".to_string(), (s_idx, local_off, len));
192 }
193 crate::p64_weight::P64_ROLE_OUTPUT_NORM => {
194 named_tensors.push((b"output_norm.weight".to_vec(), info));
195 tensor_locations
196 .insert("output_norm.weight".to_string(), (s_idx, local_off, len));
197 }
198 _ => {}
199 }
200 } else {
201 inferred_layers = inferred_layers.max(u32::from(r.layer) + 1);
202 if let Some(suffix) = crate::inference::safetensor_loader::role_suffix(r.role) {
203 let mut buf = [0u8; 96];
204 let blk_len = crate::gguf_sharder::write_blk_tensor_name(
205 r.layer as u32,
206 suffix,
207 &mut buf,
208 );
209 if blk_len > 0 {
210 named_tensors.push((buf[..blk_len].to_vec(), info));
211 if let Ok(c_str) = std::str::from_utf8(&buf[..blk_len]) {
212 tensor_locations.insert(c_str.to_string(), (s_idx, local_off, len));
213 }
214 }
215 }
216 }
217 }
218
219 named_tensors.push((t.name.as_bytes().to_vec(), info));
221
222 if t.name.ends_with("embed_tokens.weight") && t.shape.len() >= 2 {
223 inferred_embd =
224 u32::try_from(t.shape[1]).map_err(|_| "FTW embedding width exceeds u32")?;
225 }
226 }
227
228 let num_layers = manifest
230 .expert_bank_num_layers
231 .unwrap_or(inferred_layers)
232 .max(1);
233 let mut expert_slices = HashMap::new();
234
235 for layer in 0..num_layers {
236 let layer_tag = format!("#L{:05}", layer);
237 let gup_key = format!("gate_up_packed{}", layer_tag);
238 let gus_key = format!("gate_up_scale{}", layer_tag);
239 let gug_key = format!("gate_up_global{}", layer_tag);
240 let dnp_key = format!("down_packed{}", layer_tag);
241 let dns_key = format!("down_scale{}", layer_tag);
242 let dng_key = format!("down_global{}", layer_tag);
243
244 if let (Some(&gup), Some(&gus), Some(&gug), Some(&dnp), Some(&dns), Some(&dng)) = (
245 bank_tensors.get(&gup_key),
246 bank_tensors.get(&gus_key),
247 bank_tensors.get(&gug_key),
248 bank_tensors.get(&dnp_key),
249 bank_tensors.get(&dns_key),
250 bank_tensors.get(&dng_key),
251 ) {
252 let num_experts = 256usize;
254 let gup_step = gup.2 / num_experts;
255 let gus_step = gus.2 / num_experts;
256 let gug_step = gug.2 / num_experts;
257 let dnp_step = dnp.2 / num_experts;
258 let dns_step = dns.2 / num_experts;
259 let dng_step = dng.2 / num_experts;
260
261 for e in 0..num_experts {
262 let slice = FtwExpertBankSlice {
263 shard_idx: gup.0,
264 component_shards: [gup.0, gus.0, gug.0, dnp.0, dns.0, dng.0],
265 gate_up_packed_off: gup.1 + e * gup_step,
266 gate_up_packed_len: gup_step,
267 gate_up_scale_off: gus.1 + e * gus_step,
268 gate_up_scale_len: gus_step,
269 gate_up_global_off: gug.1 + e * gug_step,
270 gate_up_global_len: gug_step,
271 down_packed_off: dnp.1 + e * dnp_step,
272 down_packed_len: dnp_step,
273 down_scale_off: dns.1 + e * dns_step,
274 down_scale_len: dns_step,
275 down_global_off: dng.1 + e * dng_step,
276 down_global_len: dng_step,
277 };
278 expert_slices.insert((layer as u16, e as u16), slice);
279 }
280 }
281 }
282
283 let has_hybrid_tensors = manifest
284 .tensors
285 .iter()
286 .any(|t| t.name.contains(".linear_attn.") || t.name.contains(".ssm_"));
287 let hyperparams = parsed_cfg
288 .hyperparams(inferred_layers, inferred_embd, has_hybrid_tensors)?
289 .into();
290 let references: Vec<(&[u8], GgufTensorInfo)> = named_tensors
291 .iter()
292 .map(|(name, info)| (name.as_slice(), *info))
293 .collect();
294
295 let mut tensor_index = GgufTensorIndex::from_components(&references, hyperparams, 0);
296 tensor_index.moe_routing = parsed_cfg.moe_routing()?;
297
298 Ok(Self {
299 prepared_shards: None,
300 manifest,
301 shards,
302 tensor_index,
303 expert_slices,
304 tensor_locations,
305 })
306 }
307
308 pub fn fetch_tensor_bytes(&self, name: &str) -> Option<&[u8]> {
310 let &(shard_idx, off, len) = self.tensor_locations.get(name)?;
311 let shard = self.shard_bytes(shard_idx)?;
312 shard.get(off..off + len)
313 }
314
315 pub fn fetch_by_global_offset(&self, global_off: u64, len: usize) -> Option<&[u8]> {
317 let end = global_off.checked_add(u64::try_from(len).ok()?)?;
318 for (idx, s) in self.manifest.shards.iter().enumerate() {
319 if global_off >= s.global_off && end <= s.global_off.checked_add(s.nbytes)? {
320 let local_off = usize::try_from(global_off.checked_sub(s.global_off)?).ok()?;
321 let shard = self.shard_bytes(idx)?;
322 return shard.get(local_off..local_off.checked_add(len)?);
323 }
324 }
325 None
326 }
327
328 pub fn get_expert_data(&self, layer: u16, expert_idx: u16) -> Option<FtwExpertData<'_>> {
330 let slice = self.expert_slices.get(&(layer, expert_idx))?;
331 let component = |index: usize, off: usize, len: usize| {
332 self.shard_bytes(slice.component_shards[index])?
333 .get(off..off.checked_add(len)?)
334 };
335
336 Some(FtwExpertData {
337 gate_up_packed: component(0, slice.gate_up_packed_off, slice.gate_up_packed_len)?,
338 gate_up_scale: component(1, slice.gate_up_scale_off, slice.gate_up_scale_len)?,
339 gate_up_global: component(2, slice.gate_up_global_off, slice.gate_up_global_len)?,
340 down_packed: component(3, slice.down_packed_off, slice.down_packed_len)?,
341 down_scale: component(4, slice.down_scale_off, slice.down_scale_len)?,
342 down_global: component(5, slice.down_global_off, slice.down_global_len)?,
343 })
344 }
345}
346
347#[cfg(test)]
348#[path = "ftw_loader/shard_tests.rs"]
349mod shard_tests;
350#[cfg(test)]
351mod tests {
352 use super::*;
353
354 #[test]
355 fn test_ftw_manifest_parsing() {
356 let raw = r#"{
357 "format": "freetoken_weight",
358 "version": 1,
359 "total_bytes": 1024,
360 "tensors": [
361 { "name": "model.embed_tokens.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10, 64], "global_off": 0, "nbytes": 1280 }
362 ],
363 "shards": [
364 { "file": "freetoken-00000.ftw", "global_off": 0, "nbytes": 1280 }
365 ]
366 }"#;
367
368 let manifest: FtwManifest = serde_json::from_str(raw).expect("manifest parsed");
369 assert_eq!(manifest.format, "freetoken_weight");
370 assert_eq!(manifest.tensors.len(), 1);
371 assert_eq!(manifest.shards.len(), 1);
372 }
373
374 #[test]
375 fn test_ftw_synthetic_package_open() {
376 let temp_dir = tempfile::tempdir().unwrap();
377 let ftw_path = temp_dir.path().join("freetoken-00000.ftw");
378 let dummy_bytes = vec![0u8; 1280];
379 std::fs::write(&ftw_path, &dummy_bytes).unwrap();
380
381 let manifest_json = r#"{
382 "format": "freetoken_weight",
383 "version": 1,
384 "total_bytes": 1280,
385 "tensors": [
386 { "name": "model.embed_tokens.weight", "kind": "weight", "dtype": "bfloat16", "shape": [10, 64], "global_off": 0, "nbytes": 1280 }
387 ],
388 "shards": [
389 { "file": "freetoken-00000.ftw", "global_off": 0, "nbytes": 1280 }
390 ]
391 }"#;
392 std::fs::write(temp_dir.path().join("freetoken_weight.json"), manifest_json).unwrap();
393
394 let config_json = r#"{
395 "model_type": "qwen2",
396 "text_config": {
397 "hidden_size": 64,
398 "num_hidden_layers": 40,
399 "num_attention_heads": 8,
400 "num_key_value_heads": 2
401 }
402 }"#;
403 std::fs::write(temp_dir.path().join("config.json"), config_json).unwrap();
404
405 let pkg = FtwModelPackage::open_from_dir(temp_dir.path()).expect("package open");
406 assert_eq!(pkg.tensor_index.emb_dim(), 64);
407 assert_eq!(pkg.tensor_index.hyperparams.n_layer, 40);
408 assert_eq!(pkg.tensor_index.hyperparams.n_head, 8);
409 assert_eq!(pkg.tensor_index.hyperparams.n_kv_head, 2);
410 assert!(pkg.fetch_tensor_bytes("token_embd.weight").is_some());
411 assert!(pkg
412 .fetch_tensor_bytes("model.embed_tokens.weight")
413 .is_some());
414
415 for config in [
416 "{",
417 r#"{"model_type":"unsupported_hybrid"}"#,
418 r#"{"model_type":"qwen3_5_moe","hidden_size":64}"#,
419 ] {
420 std::fs::write(temp_dir.path().join("config.json"), config).unwrap();
421 assert!(FtwModelPackage::open_from_dir(temp_dir.path()).is_err());
422 }
423 }
424
425 #[test]
426 fn test_ftw_physical_qwen_package_if_present() {
427 let p = Path::new(r#"E:\LLM_Models\Qwen3.6-35B-A3B-NVFP4"#);
428 if !p.exists() || !p.join("freetoken_weight.json").exists() {
429 return;
430 }
431
432 let pkg = FtwModelPackage::open_from_dir(p).expect("real package open");
433 assert_eq!(pkg.manifest.format, "freetoken_weight");
434 assert!(!pkg.shards.is_empty());
435 assert!(pkg.tensor_index.emb_dim() > 0);
436
437 let exp_0_0 = pkg.get_expert_data(0, 0);
438 assert!(exp_0_0.is_some(), "layer 0 expert 0 data present");
439 let exp_data = exp_0_0.unwrap();
440 assert!(!exp_data.gate_up_packed.is_empty());
441 assert!(!exp_data.down_packed.is_empty());
442
443 let view = exp_data.to_view(pkg.tensor_index.emb_dim() as usize, 1024);
444 assert!(!view.gate_up_packed.is_empty());
445 assert!(!view.down_packed.is_empty());
446 }
447}
448
449impl etalon_inference::ordinary::expert_source::ExpertSource for FtwModelPackage {
450 fn fetch_tensor_bytes(&self, name: &str) -> Option<&[u8]> {
451 FtwModelPackage::fetch_tensor_bytes(self, name)
452 }
453 fn get_expert_data(&self, layer: u16, expert: u16) -> Option<FtwExpertData<'_>> {
454 FtwModelPackage::get_expert_data(self, layer, expert)
455 }
456}