Skip to main content

qualia_core_db/inference/
safetensor_loader.rs

1//! Safetensor model loader — native zero-copy adoption of Safetensors models (including BF16).
2//!
3//! Maps Hugging Face safetensors format (`model.layers.N...`, `lm_head`, `model.embed_tokens`)
4//! and GGUF-named safetensors into canonical `GgufTensorIndex` references without transcoding.
5//! Supports `BF16` (GGML 30), `F16` (GGML 1), and `F32` (GGML 0).
6
7use crate::inference::gguf_sharder::{
8    GgufHyperparams, GgufTensorIndex, GgufTensorInfo, ARCH_LLAMA, ARCH_QWEN2,
9    DEFAULT_ROPE_FREQ_BASE,
10};
11use crate::inference::safetensor::{parse_safetensor_header, safetensor_dtype_to_ggml};
12use crate::inference::tensor_roles::name_to_role;
13use crate::p64_weight::{
14    P64_LAYER_GLOBAL, P64_ROLE_ATTN_K, P64_ROLE_ATTN_Q, P64_ROLE_OUTPUT, P64_ROLE_OUTPUT_NORM,
15    P64_ROLE_TOKEN_EMBD,
16};
17
18pub(crate) use crate::inference::tensor_roles::canonical_suffix as role_suffix;
19
20/// GGML synthetic type tag for packed NVFP4 (`U8` weight bytes paired with `F8_E4M3` `.weight_scale` and `F32` `.weight_scale_2`).
21pub const GGML_TYPE_NVFP4_PACKED: u32 = 114;
22
23/// Zero-copy view of one NVFP4-quantized matrix inside a Safetensors mmap (`U8` packed nibbles + `F8_E4M3` block scales + `f32` global scale).
24#[derive(Debug, Clone, Copy, PartialEq)]
25pub struct SafetensorNvfp4Matrix<'a> {
26    pub packed: &'a [u8],
27    pub block_scales: &'a [u8],
28    pub global_scale: f32,
29    pub rows: usize,
30    pub cols: usize,
31}
32
33impl<'a> SafetensorNvfp4Matrix<'a> {
34    /// Execute zero-heap NVFP4 matrix-vector multiplication $y = A \cdot x$ directly from the Safetensors mmap.
35    pub fn gemv_zero_heap(&self, input: &[f32], out: &mut [f32]) -> Result<(), crate::inference::moe::Nvfp4Error> {
36        crate::inference::moe::nvfp4_gemv_zero_heap(
37            input,
38            self.packed,
39            self.block_scales,
40            self.global_scale,
41            self.rows,
42            self.cols,
43            out,
44        )
45    }
46}
47
48/// Resolve an NVFP4 matrix triplet (`{prefix}.weight`, `{prefix}.weight_scale`, `{prefix}.weight_scale_2`) from a Safetensors mmap.
49pub fn resolve_nvfp4_matrix<'a>(
50    mmap: &'a [u8],
51    plan: &crate::inference::safetensor::SafeTensorPlan,
52    weight_name: &str,
53) -> Option<SafetensorNvfp4Matrix<'a>> {
54    let base = weight_name.strip_suffix(".weight")?;
55    let scale_name = format!("{base}.weight_scale");
56    let scale2_name = format!("{base}.weight_scale_2");
57    let w = plan.tensors.iter().find(|t| t.name == weight_name && t.dtype == "U8")?;
58    let s = plan.tensors.iter().find(|t| t.name == scale_name && t.dtype == "F8_E4M3")?;
59    let s2 = plan.tensors.iter().find(|t| t.name == scale2_name && t.dtype == "F32")?;
60    if w.shape.len() != 2 || s2.byte_len() < 4 {
61        return None;
62    }
63    let rows = w.shape[0];
64    let cols = w.shape[1] * 2;
65    let ds = plan.data_start;
66    let packed = mmap.get(ds + w.begin..ds + w.end)?;
67    let block_scales = mmap.get(ds + s.begin..ds + s.end)?;
68    let s2_bytes = mmap.get(ds + s2.begin..ds + s2.begin + 4)?;
69    let global_scale = f32::from_le_bytes(s2_bytes.try_into().ok()?);
70    Some(SafetensorNvfp4Matrix {
71        packed,
72        block_scales,
73        global_scale,
74        rows,
75        cols,
76    })
77}
78#[derive(Debug, Clone, Default)]
79pub struct ModelJsonConfig {
80    pub num_hidden_layers: Option<u32>,
81    pub hidden_size: Option<u32>,
82    pub num_attention_heads: Option<u32>,
83    pub num_key_value_heads: Option<u32>,
84    pub head_dim: Option<u32>,
85    pub rope_theta: Option<f32>,
86    pub model_type: Option<String>,
87    pub moe_intermediate_size: Option<u32>,
88    pub num_experts: Option<u32>,
89    pub num_experts_per_tok: Option<u32>,
90}
91
92impl ModelJsonConfig {
93    /// Parse from a JSON string slice, supporting top-level or nested `text_config`.
94    pub fn from_json_str(s: &str) -> Option<Self> {
95        let val: serde_json::Value = serde_json::from_str(s).ok()?;
96        let obj = val.as_object()?;
97        let tc = obj.get("text_config").and_then(|v| v.as_object());
98        let get_u32 = |k: &str| {
99            obj.get(k)
100                .or_else(|| tc.and_then(|t| t.get(k)))
101                .and_then(|v| v.as_u64())
102                .map(|v| v as u32)
103        };
104        let get_f32 = |k: &str| {
105            obj.get(k)
106                .or_else(|| tc.and_then(|t| t.get(k)))
107                .and_then(|v| v.as_f64())
108                .map(|v| v as f32)
109        };
110        let get_str = |k: &str| {
111            obj.get(k)
112                .or_else(|| tc.and_then(|t| t.get(k)))
113                .and_then(|v| v.as_str())
114                .map(|v| v.to_string())
115        };
116
117        Some(Self {
118            num_hidden_layers: get_u32("num_hidden_layers"),
119            hidden_size: get_u32("hidden_size"),
120            num_attention_heads: get_u32("num_attention_heads"),
121            num_key_value_heads: get_u32("num_key_value_heads"),
122            head_dim: get_u32("head_dim"),
123            rope_theta: get_f32("rope_theta"),
124            model_type: get_str("model_type"),
125            moe_intermediate_size: get_u32("moe_intermediate_size"),
126            num_experts: get_u32("num_experts"),
127            num_experts_per_tok: get_u32("num_experts_per_tok"),
128        })
129    }
130}
131
132/// Parse a safetensor mmap and build a canonical `GgufTensorIndex`.
133pub fn parse_safetensor_to_index(
134    mmap: &[u8],
135    config_json: Option<&str>,
136) -> Result<GgufTensorIndex, String> {
137    let plan = parse_safetensor_header(mmap)?;
138    let parsed_cfg = config_json.and_then(ModelJsonConfig::from_json_str);
139
140    let mut named_tensors: Vec<(Vec<u8>, GgufTensorInfo)> = Vec::with_capacity(plan.tensors.len());
141    let mut inferred_layers = 0u32;
142    let mut inferred_embd = 0u32;
143    let mut inferred_q_heads = 0u32;
144    let mut inferred_kv_heads = 0u32;
145
146    for tensor in &plan.tensors {
147        if tensor.name.ends_with(".input_scale")
148            || tensor.name.ends_with(".weight_scale")
149            || tensor.name.ends_with(".weight_scale_2")
150            || tensor.name.starts_with("model.vision_tower.")
151            || tensor.name.starts_with("model.embed_vision.")
152        {
153            continue;
154        }
155        let ggml_type = if tensor.dtype == "U8" && tensor.name.ends_with(".weight") {
156            GGML_TYPE_NVFP4_PACKED
157        } else if let Some(t) = safetensor_dtype_to_ggml(&tensor.dtype) {
158            t
159        } else {
160            return Err(format!(
161                "safetensor: unsupported dtype '{}' on tensor '{}'",
162                tensor.dtype, tensor.name
163            ));
164        };
165
166        // Canonicalize name
167        let role = name_to_role(&tensor.name);
168        let canonical_name: Vec<u8> = if let Some(r) = role {
169            if r.layer == P64_LAYER_GLOBAL {
170                match r.role {
171                    P64_ROLE_TOKEN_EMBD => {
172                        if tensor.shape.len() >= 2 {
173                            inferred_embd = inferred_embd.max(tensor.shape[1] as u32);
174                        }
175                        b"token_embd.weight".to_vec()
176                    }
177                    P64_ROLE_OUTPUT => b"output.weight".to_vec(),
178                    P64_ROLE_OUTPUT_NORM => b"output_norm.weight".to_vec(),
179                    _ => tensor.name.as_bytes().to_vec(),
180                }
181            } else {
182                inferred_layers = inferred_layers.max(r.layer as u32 + 1);
183                if let Some(suffix) = role_suffix(r.role) {
184                    let mut buf = [0u8; 96];
185                    let len = crate::gguf_sharder::write_blk_tensor_name(
186                        r.layer as u32,
187                        suffix,
188                        &mut buf,
189                    );
190                    if r.role == P64_ROLE_ATTN_Q && tensor.shape.len() >= 2 {
191                        let out_dim = tensor.shape[0] as u32;
192                        let in_dim = tensor.shape[1] as u32;
193                        inferred_embd = inferred_embd.max(in_dim);
194                        let h_dim = 64u32; // standard floor
195                        inferred_q_heads = inferred_q_heads.max(out_dim / h_dim);
196                    } else if r.role == P64_ROLE_ATTN_K && tensor.shape.len() >= 2 {
197                        let out_dim = tensor.shape[0] as u32;
198                        let h_dim = 64u32;
199                        inferred_kv_heads = inferred_kv_heads.max(out_dim / h_dim);
200                    }
201                    buf[..len].to_vec()
202                } else {
203                    tensor.name.as_bytes().to_vec()
204                }
205            }
206        } else {
207            tensor.name.as_bytes().to_vec()
208        };
209
210        let mut dims = [0u64; 4];
211        let n_dims = tensor.shape.len().min(4) as u32;
212        for (i, d) in tensor.shape.iter().take(4).enumerate() {
213            // Reverse dimensions to match GGUF column-major order convention if necessary
214            // For 2D matmuls [out_features, in_features], GGUF expects [in_features, out_features].
215            if n_dims == 2 {
216                let in_elems = if ggml_type == GGML_TYPE_NVFP4_PACKED {
217                    (tensor.shape[1] * 2) as u64
218                } else {
219                    tensor.shape[1] as u64
220                };
221                dims[0] = in_elems;
222                dims[1] = tensor.shape[0] as u64;
223                break;
224            } else {
225                dims[i] = *d as u64;
226            }
227        }
228
229        named_tensors.push((
230            canonical_name,
231            GgufTensorInfo {
232                dims,
233                n_dims,
234                ggml_type,
235                byte_offset: tensor.begin as u64,
236            },
237        ));
238    }
239
240    // Resolve hyperparameters from config.json or inferred dimensions
241    let n_layer = parsed_cfg
242        .as_ref()
243        .and_then(|c| c.num_hidden_layers)
244        .unwrap_or(inferred_layers);
245    let n_embd = parsed_cfg
246        .as_ref()
247        .and_then(|c| c.hidden_size)
248        .unwrap_or(inferred_embd);
249    let n_head = parsed_cfg
250        .as_ref()
251        .and_then(|c| c.num_attention_heads)
252        .unwrap_or_else(|| inferred_q_heads.max(1));
253    let n_kv_head = parsed_cfg
254        .as_ref()
255        .and_then(|c| c.num_key_value_heads)
256        .unwrap_or_else(|| inferred_kv_heads.max(1));
257    let head_dim = parsed_cfg
258        .as_ref()
259        .and_then(|c| c.head_dim)
260        .unwrap_or_else(|| if n_head > 0 { n_embd / n_head } else { 64 });
261    let rope_freq_base = parsed_cfg
262        .as_ref()
263        .and_then(|c| c.rope_theta)
264        .unwrap_or(DEFAULT_ROPE_FREQ_BASE);
265
266    let architecture = if let Some(cfg) = &parsed_cfg {
267        match cfg.model_type.as_deref() {
268            Some("qwen2") | Some("qwen") => ARCH_QWEN2,
269            _ => ARCH_LLAMA,
270        }
271    } else {
272        ARCH_LLAMA
273    };
274
275    let hyperparams = GgufHyperparams {
276        n_layer,
277        n_embd,
278        n_head,
279        n_kv_head,
280        rope_freq_base,
281        rope_scale: 1.0,
282        head_dim,
283        head_dim_swa: head_dim,
284        sliding_window: 0,
285        shared_kv_layers: 0,
286        logit_softcap: 0.0,
287        ssm_conv_kernel: 0,
288        ssm_state_size: 0,
289        ssm_group_count: 0,
290        ssm_time_step_rank: 0,
291        ssm_inner_size: 0,
292        full_attention_interval: 0,
293        architecture,
294        arch_flags: 0,
295        ..GgufHyperparams::default()
296    };
297
298    let references: Vec<(&[u8], GgufTensorInfo)> = named_tensors
299        .iter()
300        .map(|(name, info)| (name.as_slice(), *info))
301        .collect();
302
303    Ok(GgufTensorIndex::from_components(
304        &references,
305        hyperparams,
306        plan.data_start as u64,
307    ))
308}
309
310#[cfg(test)]
311mod tests {
312    use super::*;
313    use crate::ggml_quants::GGML_TYPE_BF16;
314
315    #[test]
316    fn test_parse_synthetic_bf16_safetensor() {
317        // Construct a synthetic Safetensors byte buffer with BF16 tensor
318        let json_header = r#"{
319            "model.embed_tokens.weight": {
320                "dtype": "BF16",
321                "shape": [10, 64],
322                "data_offsets": [0, 1280]
323            },
324            "model.layers.0.self_attn.q_proj.weight": {
325                "dtype": "BF16",
326                "shape": [64, 64],
327                "data_offsets": [1280, 9472]
328            },
329            "lm_head.weight": {
330                "dtype": "BF16",
331                "shape": [10, 64],
332                "data_offsets": [9472, 10752]
333            }
334        }"#;
335
336        let json_bytes = json_header.as_bytes();
337        let hlen = json_bytes.len() as u64;
338
339        let mut buf = Vec::new();
340        buf.extend_from_slice(&hlen.to_le_bytes());
341        buf.extend_from_slice(json_bytes);
342        buf.resize(buf.len() + 10752, 0x3fu8); // dummy payload
343
344        let index = parse_safetensor_to_index(&buf, None).expect("safetensor parse ok");
345        assert_eq!(index.hyperparams.n_layer, 1);
346        assert_eq!(index.hyperparams.n_embd, 64);
347        assert!(index.token_embd.is_some());
348        assert_eq!(index.token_embd.unwrap().ggml_type, GGML_TYPE_BF16);
349    }
350
351    #[test]
352    fn real_qwen35_4b_nvfp4_and_gemma4_26b_nvfp4_safetensor_gemv() {
353        let qwen_path = r"C:\LLM_files\qwen\Qwen3.5-4B-NVFP4\model.safetensors";
354        if std::path::Path::new(qwen_path).exists() {
355            let cfg_str = std::fs::read_to_string(r"C:\LLM_files\qwen\Qwen3.5-4B-NVFP4\config.json")
356                .ok();
357            let file = std::fs::File::open(qwen_path).unwrap();
358            let mmap = unsafe { memmap2::MmapOptions::new().map(&file).unwrap() };
359            let plan = parse_safetensor_header(&mmap[..]).expect("qwen nvfp4 header");
360            let index = parse_safetensor_to_index(&mmap[..], cfg_str.as_deref())
361                .expect("qwen nvfp4 index");
362            assert_eq!(index.hyperparams.n_layer, 32);
363            assert_eq!(index.hyperparams.n_embd, 2560);
364            let mat = resolve_nvfp4_matrix(
365                &mmap[..],
366                &plan,
367                "model.language_model.layers.0.mlp.gate_proj.weight",
368            )
369            .expect("resolve qwen gate_proj nvfp4");
370            assert_eq!(mat.rows, 9216);
371            assert_eq!(mat.cols, 2560);
372            let input = [0.125f32; 2560];
373            let mut out = vec![0.0f32; 9216];
374            mat.gemv_zero_heap(&input, &mut out).expect("nvfp4 gemv");
375            assert!(out.iter().all(|v| v.is_finite()));
376            assert!(out.iter().any(|&v| v.abs() > 1e-6));
377        }
378
379        let gemma_path =
380            r"C:\LLM_files\gemma\Gemma-4-26B-A4B-NVFP4\model-00001-of-00002.safetensors";
381        if std::path::Path::new(gemma_path).exists() {
382            let file = std::fs::File::open(gemma_path).unwrap();
383            let mmap = unsafe { memmap2::MmapOptions::new().map(&file).unwrap() };
384            let plan = parse_safetensor_header(&mmap[..]).expect("gemma nvfp4 header");
385            let mat = resolve_nvfp4_matrix(
386                &mmap[..],
387                &plan,
388                "model.language_model.layers.0.experts.0.gate_proj.weight",
389            )
390            .expect("resolve gemma expert 0 gate_proj nvfp4");
391            assert_eq!(mat.rows, 704);
392            assert_eq!(mat.cols, 2816);
393            let input = [0.125f32; 2816];
394            let mut out = [0.0f32; 704];
395            mat.gemv_zero_heap(&input, &mut out).expect("gemma nvfp4 gemv");
396            assert!(out.iter().all(|v| v.is_finite()));
397            assert!(out.iter().any(|&v| v.abs() > 1e-6));
398        }
399    }
400}