qualia_core_db/inference/
safetensor_loader.rs1use crate::inference::gguf_sharder::{
8 GgufHyperparams, GgufTensorIndex, GgufTensorInfo, ARCH_LLAMA, ARCH_QWEN2,
9 DEFAULT_ROPE_FREQ_BASE,
10};
11use crate::inference::safetensor::{parse_safetensor_header, safetensor_dtype_to_ggml};
12use crate::inference::tensor_roles::name_to_role;
13use crate::p64_weight::{
14 P64_LAYER_GLOBAL, P64_ROLE_ATTN_K, P64_ROLE_ATTN_Q, P64_ROLE_OUTPUT, P64_ROLE_OUTPUT_NORM,
15 P64_ROLE_TOKEN_EMBD,
16};
17
18pub(crate) use crate::inference::tensor_roles::canonical_suffix as role_suffix;
19
20pub const GGML_TYPE_NVFP4_PACKED: u32 = 114;
22
23#[derive(Debug, Clone, Copy, PartialEq)]
25pub struct SafetensorNvfp4Matrix<'a> {
26 pub packed: &'a [u8],
27 pub block_scales: &'a [u8],
28 pub global_scale: f32,
29 pub rows: usize,
30 pub cols: usize,
31}
32
33impl<'a> SafetensorNvfp4Matrix<'a> {
34 pub fn gemv_zero_heap(&self, input: &[f32], out: &mut [f32]) -> Result<(), crate::inference::moe::Nvfp4Error> {
36 crate::inference::moe::nvfp4_gemv_zero_heap(
37 input,
38 self.packed,
39 self.block_scales,
40 self.global_scale,
41 self.rows,
42 self.cols,
43 out,
44 )
45 }
46}
47
48pub fn resolve_nvfp4_matrix<'a>(
50 mmap: &'a [u8],
51 plan: &crate::inference::safetensor::SafeTensorPlan,
52 weight_name: &str,
53) -> Option<SafetensorNvfp4Matrix<'a>> {
54 let base = weight_name.strip_suffix(".weight")?;
55 let scale_name = format!("{base}.weight_scale");
56 let scale2_name = format!("{base}.weight_scale_2");
57 let w = plan.tensors.iter().find(|t| t.name == weight_name && t.dtype == "U8")?;
58 let s = plan.tensors.iter().find(|t| t.name == scale_name && t.dtype == "F8_E4M3")?;
59 let s2 = plan.tensors.iter().find(|t| t.name == scale2_name && t.dtype == "F32")?;
60 if w.shape.len() != 2 || s2.byte_len() < 4 {
61 return None;
62 }
63 let rows = w.shape[0];
64 let cols = w.shape[1] * 2;
65 let ds = plan.data_start;
66 let packed = mmap.get(ds + w.begin..ds + w.end)?;
67 let block_scales = mmap.get(ds + s.begin..ds + s.end)?;
68 let s2_bytes = mmap.get(ds + s2.begin..ds + s2.begin + 4)?;
69 let global_scale = f32::from_le_bytes(s2_bytes.try_into().ok()?);
70 Some(SafetensorNvfp4Matrix {
71 packed,
72 block_scales,
73 global_scale,
74 rows,
75 cols,
76 })
77}
78#[derive(Debug, Clone, Default)]
79pub struct ModelJsonConfig {
80 pub num_hidden_layers: Option<u32>,
81 pub hidden_size: Option<u32>,
82 pub num_attention_heads: Option<u32>,
83 pub num_key_value_heads: Option<u32>,
84 pub head_dim: Option<u32>,
85 pub rope_theta: Option<f32>,
86 pub model_type: Option<String>,
87 pub moe_intermediate_size: Option<u32>,
88 pub num_experts: Option<u32>,
89 pub num_experts_per_tok: Option<u32>,
90}
91
92impl ModelJsonConfig {
93 pub fn from_json_str(s: &str) -> Option<Self> {
95 let val: serde_json::Value = serde_json::from_str(s).ok()?;
96 let obj = val.as_object()?;
97 let tc = obj.get("text_config").and_then(|v| v.as_object());
98 let get_u32 = |k: &str| {
99 obj.get(k)
100 .or_else(|| tc.and_then(|t| t.get(k)))
101 .and_then(|v| v.as_u64())
102 .map(|v| v as u32)
103 };
104 let get_f32 = |k: &str| {
105 obj.get(k)
106 .or_else(|| tc.and_then(|t| t.get(k)))
107 .and_then(|v| v.as_f64())
108 .map(|v| v as f32)
109 };
110 let get_str = |k: &str| {
111 obj.get(k)
112 .or_else(|| tc.and_then(|t| t.get(k)))
113 .and_then(|v| v.as_str())
114 .map(|v| v.to_string())
115 };
116
117 Some(Self {
118 num_hidden_layers: get_u32("num_hidden_layers"),
119 hidden_size: get_u32("hidden_size"),
120 num_attention_heads: get_u32("num_attention_heads"),
121 num_key_value_heads: get_u32("num_key_value_heads"),
122 head_dim: get_u32("head_dim"),
123 rope_theta: get_f32("rope_theta"),
124 model_type: get_str("model_type"),
125 moe_intermediate_size: get_u32("moe_intermediate_size"),
126 num_experts: get_u32("num_experts"),
127 num_experts_per_tok: get_u32("num_experts_per_tok"),
128 })
129 }
130}
131
132pub fn parse_safetensor_to_index(
134 mmap: &[u8],
135 config_json: Option<&str>,
136) -> Result<GgufTensorIndex, String> {
137 let plan = parse_safetensor_header(mmap)?;
138 let parsed_cfg = config_json.and_then(ModelJsonConfig::from_json_str);
139
140 let mut named_tensors: Vec<(Vec<u8>, GgufTensorInfo)> = Vec::with_capacity(plan.tensors.len());
141 let mut inferred_layers = 0u32;
142 let mut inferred_embd = 0u32;
143 let mut inferred_q_heads = 0u32;
144 let mut inferred_kv_heads = 0u32;
145
146 for tensor in &plan.tensors {
147 if tensor.name.ends_with(".input_scale")
148 || tensor.name.ends_with(".weight_scale")
149 || tensor.name.ends_with(".weight_scale_2")
150 || tensor.name.starts_with("model.vision_tower.")
151 || tensor.name.starts_with("model.embed_vision.")
152 {
153 continue;
154 }
155 let ggml_type = if tensor.dtype == "U8" && tensor.name.ends_with(".weight") {
156 GGML_TYPE_NVFP4_PACKED
157 } else if let Some(t) = safetensor_dtype_to_ggml(&tensor.dtype) {
158 t
159 } else {
160 return Err(format!(
161 "safetensor: unsupported dtype '{}' on tensor '{}'",
162 tensor.dtype, tensor.name
163 ));
164 };
165
166 let role = name_to_role(&tensor.name);
168 let canonical_name: Vec<u8> = if let Some(r) = role {
169 if r.layer == P64_LAYER_GLOBAL {
170 match r.role {
171 P64_ROLE_TOKEN_EMBD => {
172 if tensor.shape.len() >= 2 {
173 inferred_embd = inferred_embd.max(tensor.shape[1] as u32);
174 }
175 b"token_embd.weight".to_vec()
176 }
177 P64_ROLE_OUTPUT => b"output.weight".to_vec(),
178 P64_ROLE_OUTPUT_NORM => b"output_norm.weight".to_vec(),
179 _ => tensor.name.as_bytes().to_vec(),
180 }
181 } else {
182 inferred_layers = inferred_layers.max(r.layer as u32 + 1);
183 if let Some(suffix) = role_suffix(r.role) {
184 let mut buf = [0u8; 96];
185 let len = crate::gguf_sharder::write_blk_tensor_name(
186 r.layer as u32,
187 suffix,
188 &mut buf,
189 );
190 if r.role == P64_ROLE_ATTN_Q && tensor.shape.len() >= 2 {
191 let out_dim = tensor.shape[0] as u32;
192 let in_dim = tensor.shape[1] as u32;
193 inferred_embd = inferred_embd.max(in_dim);
194 let h_dim = 64u32; inferred_q_heads = inferred_q_heads.max(out_dim / h_dim);
196 } else if r.role == P64_ROLE_ATTN_K && tensor.shape.len() >= 2 {
197 let out_dim = tensor.shape[0] as u32;
198 let h_dim = 64u32;
199 inferred_kv_heads = inferred_kv_heads.max(out_dim / h_dim);
200 }
201 buf[..len].to_vec()
202 } else {
203 tensor.name.as_bytes().to_vec()
204 }
205 }
206 } else {
207 tensor.name.as_bytes().to_vec()
208 };
209
210 let mut dims = [0u64; 4];
211 let n_dims = tensor.shape.len().min(4) as u32;
212 for (i, d) in tensor.shape.iter().take(4).enumerate() {
213 if n_dims == 2 {
216 let in_elems = if ggml_type == GGML_TYPE_NVFP4_PACKED {
217 (tensor.shape[1] * 2) as u64
218 } else {
219 tensor.shape[1] as u64
220 };
221 dims[0] = in_elems;
222 dims[1] = tensor.shape[0] as u64;
223 break;
224 } else {
225 dims[i] = *d as u64;
226 }
227 }
228
229 named_tensors.push((
230 canonical_name,
231 GgufTensorInfo {
232 dims,
233 n_dims,
234 ggml_type,
235 byte_offset: tensor.begin as u64,
236 },
237 ));
238 }
239
240 let n_layer = parsed_cfg
242 .as_ref()
243 .and_then(|c| c.num_hidden_layers)
244 .unwrap_or(inferred_layers);
245 let n_embd = parsed_cfg
246 .as_ref()
247 .and_then(|c| c.hidden_size)
248 .unwrap_or(inferred_embd);
249 let n_head = parsed_cfg
250 .as_ref()
251 .and_then(|c| c.num_attention_heads)
252 .unwrap_or_else(|| inferred_q_heads.max(1));
253 let n_kv_head = parsed_cfg
254 .as_ref()
255 .and_then(|c| c.num_key_value_heads)
256 .unwrap_or_else(|| inferred_kv_heads.max(1));
257 let head_dim = parsed_cfg
258 .as_ref()
259 .and_then(|c| c.head_dim)
260 .unwrap_or_else(|| if n_head > 0 { n_embd / n_head } else { 64 });
261 let rope_freq_base = parsed_cfg
262 .as_ref()
263 .and_then(|c| c.rope_theta)
264 .unwrap_or(DEFAULT_ROPE_FREQ_BASE);
265
266 let architecture = if let Some(cfg) = &parsed_cfg {
267 match cfg.model_type.as_deref() {
268 Some("qwen2") | Some("qwen") => ARCH_QWEN2,
269 _ => ARCH_LLAMA,
270 }
271 } else {
272 ARCH_LLAMA
273 };
274
275 let hyperparams = GgufHyperparams {
276 n_layer,
277 n_embd,
278 n_head,
279 n_kv_head,
280 rope_freq_base,
281 rope_scale: 1.0,
282 head_dim,
283 head_dim_swa: head_dim,
284 sliding_window: 0,
285 shared_kv_layers: 0,
286 logit_softcap: 0.0,
287 ssm_conv_kernel: 0,
288 ssm_state_size: 0,
289 ssm_group_count: 0,
290 ssm_time_step_rank: 0,
291 ssm_inner_size: 0,
292 full_attention_interval: 0,
293 architecture,
294 arch_flags: 0,
295 ..GgufHyperparams::default()
296 };
297
298 let references: Vec<(&[u8], GgufTensorInfo)> = named_tensors
299 .iter()
300 .map(|(name, info)| (name.as_slice(), *info))
301 .collect();
302
303 Ok(GgufTensorIndex::from_components(
304 &references,
305 hyperparams,
306 plan.data_start as u64,
307 ))
308}
309
310#[cfg(test)]
311mod tests {
312 use super::*;
313 use crate::ggml_quants::GGML_TYPE_BF16;
314
315 #[test]
316 fn test_parse_synthetic_bf16_safetensor() {
317 let json_header = r#"{
319 "model.embed_tokens.weight": {
320 "dtype": "BF16",
321 "shape": [10, 64],
322 "data_offsets": [0, 1280]
323 },
324 "model.layers.0.self_attn.q_proj.weight": {
325 "dtype": "BF16",
326 "shape": [64, 64],
327 "data_offsets": [1280, 9472]
328 },
329 "lm_head.weight": {
330 "dtype": "BF16",
331 "shape": [10, 64],
332 "data_offsets": [9472, 10752]
333 }
334 }"#;
335
336 let json_bytes = json_header.as_bytes();
337 let hlen = json_bytes.len() as u64;
338
339 let mut buf = Vec::new();
340 buf.extend_from_slice(&hlen.to_le_bytes());
341 buf.extend_from_slice(json_bytes);
342 buf.resize(buf.len() + 10752, 0x3fu8); let index = parse_safetensor_to_index(&buf, None).expect("safetensor parse ok");
345 assert_eq!(index.hyperparams.n_layer, 1);
346 assert_eq!(index.hyperparams.n_embd, 64);
347 assert!(index.token_embd.is_some());
348 assert_eq!(index.token_embd.unwrap().ggml_type, GGML_TYPE_BF16);
349 }
350
351 #[test]
352 fn real_qwen35_4b_nvfp4_and_gemma4_26b_nvfp4_safetensor_gemv() {
353 let qwen_path = r"C:\LLM_files\qwen\Qwen3.5-4B-NVFP4\model.safetensors";
354 if std::path::Path::new(qwen_path).exists() {
355 let cfg_str = std::fs::read_to_string(r"C:\LLM_files\qwen\Qwen3.5-4B-NVFP4\config.json")
356 .ok();
357 let file = std::fs::File::open(qwen_path).unwrap();
358 let mmap = unsafe { memmap2::MmapOptions::new().map(&file).unwrap() };
359 let plan = parse_safetensor_header(&mmap[..]).expect("qwen nvfp4 header");
360 let index = parse_safetensor_to_index(&mmap[..], cfg_str.as_deref())
361 .expect("qwen nvfp4 index");
362 assert_eq!(index.hyperparams.n_layer, 32);
363 assert_eq!(index.hyperparams.n_embd, 2560);
364 let mat = resolve_nvfp4_matrix(
365 &mmap[..],
366 &plan,
367 "model.language_model.layers.0.mlp.gate_proj.weight",
368 )
369 .expect("resolve qwen gate_proj nvfp4");
370 assert_eq!(mat.rows, 9216);
371 assert_eq!(mat.cols, 2560);
372 let input = [0.125f32; 2560];
373 let mut out = vec![0.0f32; 9216];
374 mat.gemv_zero_heap(&input, &mut out).expect("nvfp4 gemv");
375 assert!(out.iter().all(|v| v.is_finite()));
376 assert!(out.iter().any(|&v| v.abs() > 1e-6));
377 }
378
379 let gemma_path =
380 r"C:\LLM_files\gemma\Gemma-4-26B-A4B-NVFP4\model-00001-of-00002.safetensors";
381 if std::path::Path::new(gemma_path).exists() {
382 let file = std::fs::File::open(gemma_path).unwrap();
383 let mmap = unsafe { memmap2::MmapOptions::new().map(&file).unwrap() };
384 let plan = parse_safetensor_header(&mmap[..]).expect("gemma nvfp4 header");
385 let mat = resolve_nvfp4_matrix(
386 &mmap[..],
387 &plan,
388 "model.language_model.layers.0.experts.0.gate_proj.weight",
389 )
390 .expect("resolve gemma expert 0 gate_proj nvfp4");
391 assert_eq!(mat.rows, 704);
392 assert_eq!(mat.cols, 2816);
393 let input = [0.125f32; 2816];
394 let mut out = [0.0f32; 704];
395 mat.gemv_zero_heap(&input, &mut out).expect("gemma nvfp4 gemv");
396 assert!(out.iter().all(|v| v.is_finite()));
397 assert!(out.iter().any(|&v| v.abs() > 1e-6));
398 }
399 }
400}