Skip to main content

qualia_core_db/gguf_bridge/qwen35/ftw/
attention.rs

1//! Architecture-owned attention admission and allocation-free execution.
2//! Packing, norm storage and rotary geometry arrive from the admitted exporter
3//! profile; neither tensor names nor a successful matrix multiply establish them.
4use super::Profile;
5use crate::inference::moe::ftw_loader::FtwModelPackage;
6use etalon_inference::ordinary::qwen35::{self, full_attention, Error, Host};
7use etalon_model::inference::{
8    qwen_attention::Qwen35AttentionRoles,
9    qwen_roles::{Projection, Qwen35Roles},
10    TensorInfo, WeightEncoding,
11};
12
13fn tensor(pkg: &FtwModelPackage, layer: u32, suffix: &str) -> Result<Option<TensorInfo>, String> {
14    let name = format!("model.layers.{layer}.{suffix}");
15    let mut matches = pkg
16        .manifest
17        .tensors
18        .iter()
19        .filter(|entry| entry.name == name);
20    let Some(entry) = matches.next() else {
21        return Ok(None);
22    };
23    if matches.next().is_some() {
24        return Err(format!("ambiguous Qwen tensor {name}"));
25    }
26    if entry.shape.is_empty() || entry.shape.len() > 4 || entry.shape.contains(&0) {
27        return Err(format!("invalid Qwen tensor shape {name}"));
28    }
29    let ty = match entry.dtype.as_str() {
30        "float32" => crate::ggml_quants::GGML_TYPE_F32,
31        "float16" => crate::ggml_quants::GGML_TYPE_F16,
32        "bfloat16" => crate::ggml_quants::GGML_TYPE_BF16,
33        "float8_e4m3fn" => crate::ggml_quants::GGML_TYPE_FP8_E4M3,
34        _ => return Err(format!("unsupported Qwen attention tensor dtype {name}")),
35    };
36    let mut dims = [0; 4];
37    for (dst, &value) in dims.iter_mut().zip(entry.shape.iter().rev()) {
38        *dst = value;
39    }
40    let info = TensorInfo {
41        dims,
42        n_dims: entry.shape.len() as u32,
43        ggml_type: ty,
44        byte_offset: entry.global_off,
45    };
46    let bytes = crate::ggml_quants::tensor_byte_len(&info).ok_or("Qwen tensor size overflow")?;
47    if bytes as u64 != entry.nbytes
48        || pkg
49            .fetch_by_global_offset(info.byte_offset, bytes)
50            .is_none()
51    {
52        return Err(format!("Qwen attention tensor bounds {name}"));
53    }
54    Ok(Some(info))
55}
56fn projection(
57    pkg: &FtwModelPackage,
58    layer: u32,
59    stem: &str,
60    cols: usize,
61    rows: usize,
62) -> Result<Projection, String> {
63    let weight = tensor(pkg, layer, &format!("{stem}.weight"))?.ok_or("missing Qwen projection")?;
64    let row_scale = tensor(pkg, layer, &format!("{stem}.weight_scale"))?;
65    if weight.n_dims != 2 || weight.dims[..2] != [cols as u64, rows as u64] {
66        return Err("Qwen attention projection shape".into());
67    }
68    match weight.encoding() {
69        WeightEncoding::Float8E4M3 => {
70            let scale = row_scale.ok_or("missing Qwen FP8 row scales")?;
71            if scale.n_dims != 1
72                || scale.dims[0] != rows as u64
73                || !matches!(
74                    scale.encoding(),
75                    WeightEncoding::IeeeF32 | WeightEncoding::IeeeF16 | WeightEncoding::BrainF16
76                )
77            {
78                return Err("Qwen FP8 row scale shape/encoding".into());
79            }
80        }
81        WeightEncoding::IeeeF32 | WeightEncoding::IeeeF16 | WeightEncoding::BrainF16
82            if row_scale.is_none() => {}
83        _ => return Err("unsupported Qwen attention projection encoding".into()),
84    }
85    Ok(Projection { weight, row_scale })
86}
87
88#[derive(Clone, Copy, Debug)]
89pub enum AttentionPlan {
90    Linear(Qwen35Roles),
91    Full(Qwen35AttentionRoles),
92}
93
94#[cfg(test)]
95#[path = "attention_tests.rs"]
96mod tests;
97
98#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
99pub struct AttentionRequirements {
100    pub history: usize,
101    pub recurrent: usize,
102    /// Float count for each of keys and values, not their sum.
103    pub kv: usize,
104    pub workspace: usize,
105}
106
107pub fn resolve(
108    pkg: &FtwModelPackage,
109    profile: &Profile,
110    layer: usize,
111    is_linear: bool,
112) -> Result<AttentionPlan, String> {
113    let layer = u32::try_from(layer).map_err(|_| "Qwen layer index overflow")?;
114    let index = &pkg.tensor_index;
115    if is_linear {
116        let roles = Qwen35Roles::resolve(index, layer, profile.packing)
117            .map_err(|e| format!("Qwen linear layer {layer} roles: {e:?}"))?;
118        if roles.geometry != profile.linear_geometry {
119            return Err(format!("Qwen linear layer {layer} geometry mismatch"));
120        }
121        // A mixed checkpoint must not silently choose one of two layer kinds.
122        if tensor(pkg, layer, "self_attn.q_proj.weight")
123            .map_err(|e| format!("Qwen layer {layer} attention ambiguity: {e:?}"))?
124            .is_some()
125            || tensor(pkg, layer, "self_attn.qkv_proj.weight")?.is_some()
126        {
127            return Err(format!("Qwen linear layer {layer} also has full attention"));
128        }
129        Ok(AttentionPlan::Linear(roles))
130    } else {
131        let g = profile.attention_geometry;
132        g.validate()
133            .map_err(|e| format!("Qwen attention geometry: {e:?}"))?;
134        let query = g
135            .query_heads
136            .checked_mul(g.head_dim)
137            .ok_or("Qwen query overflow")?;
138        let kv = g
139            .kv_heads
140            .checked_mul(g.head_dim)
141            .ok_or("Qwen KV overflow")?;
142        let qg = query.checked_mul(2).ok_or("Qwen query gate overflow")?;
143        let rows = kv
144            .checked_mul(2)
145            .and_then(|n| n.checked_add(qg))
146            .ok_or("Qwen QKV overflow")?;
147        // FreeToken's mixed ModelOpt exporter concatenates Q/gate, K, V rows,
148        // with Q/gate interleaved per query head. Slice the scale vector at
149        // exactly the same row boundaries; never reinterpret packed columns.
150        let fused = projection(pkg, layer, "self_attn.qkv_proj", g.hidden, rows)
151            .map_err(|e| format!("Qwen fused attention layer {layer}: {e:?}"))?;
152        let required = |name| {
153            tensor(pkg, layer, name)
154                .map_err(|e| format!("Qwen attention layer {layer}: {e:?}"))?
155                .ok_or_else(|| format!("Qwen attention layer {layer} missing {name}"))
156        };
157        let roles = Qwen35AttentionRoles {
158            geometry: g,
159            q_gate: row_range(fused, 0, qg)?,
160            k: row_range(fused, qg, kv)?,
161            v: row_range(fused, qg + kv, kv)?,
162            out: projection(pkg, layer, "self_attn.o_proj", query, g.hidden)
163                .map_err(|e| format!("Qwen attention output layer {layer}: {e:?}"))?,
164            q_norm: required("self_attn.q_norm.weight")?,
165            k_norm: required("self_attn.k_norm.weight")?,
166            norm: profile.ordinary_norm,
167        };
168        roles
169            .validate()
170            .map_err(|e| format!("Qwen attention layer {layer} roles: {e:?}"))?;
171        for name in [
172            "linear_attn.in_proj_qkvz.weight",
173            "linear_attn.in_proj_qkv.weight",
174            "self_attn.q_proj.weight",
175            "self_attn.k_proj.weight",
176            "self_attn.v_proj.weight",
177            "self_attn.qkv_proj.bias",
178            "self_attn.q_proj.bias",
179            "self_attn.k_proj.bias",
180            "self_attn.v_proj.bias",
181            "self_attn.o_proj.bias",
182        ] {
183            if tensor(pkg, layer, name)
184                .map_err(|e| format!("Qwen layer {layer} linear ambiguity: {e:?}"))?
185                .is_some()
186            {
187                return Err(format!(
188                    "Qwen full attention layer {layer} has unsupported mixed role {name}"
189                ));
190            }
191        }
192        Ok(AttentionPlan::Full(roles))
193    }
194}
195
196/// Cold descriptor slicing includes its independently stored row scales.
197fn row_range(mut projection: Projection, start: usize, rows: usize) -> Result<Projection, String> {
198    if rows == 0
199        || start
200            .checked_add(rows)
201            .is_none_or(|end| end > projection.rows())
202    {
203        return Err("Qwen fused projection row bounds".into());
204    }
205    let width = match projection.weight.encoding() {
206        WeightEncoding::IeeeF32 => 4,
207        WeightEncoding::IeeeF16 | WeightEncoding::BrainF16 => 2,
208        WeightEncoding::Float8E4M3 => 1,
209        _ => return Err("Qwen fused projection encoding".into()),
210    };
211    let offset = start
212        .checked_mul(projection.cols())
213        .and_then(|n| n.checked_mul(width))
214        .and_then(|n| u64::try_from(n).ok())
215        .ok_or("Qwen fused projection offset overflow")?;
216    projection.weight.byte_offset = projection
217        .weight
218        .byte_offset
219        .checked_add(offset)
220        .ok_or("Qwen fused projection address overflow")?;
221    projection.weight.dims[1] = rows as u64;
222    if let Some(scale) = &mut projection.row_scale {
223        // qwen35_projection validates a scalar F32/F16/BF16 scale per row.
224        let width = match scale.encoding() {
225            WeightEncoding::IeeeF32 => 4,
226            WeightEncoding::IeeeF16 | WeightEncoding::BrainF16 => 2,
227            _ => return Err("Qwen fused scale encoding".into()),
228        };
229        let offset = start
230            .checked_mul(width)
231            .and_then(|n| u64::try_from(n).ok())
232            .ok_or("Qwen fused scale offset overflow")?;
233        scale.byte_offset = scale
234            .byte_offset
235            .checked_add(offset)
236            .ok_or("Qwen fused scale address overflow")?;
237        scale.dims[0] = rows as u64;
238    }
239    Ok(projection)
240}
241
242impl AttentionPlan {
243    pub fn requirements(&self, max_context: usize) -> Result<AttentionRequirements, Error> {
244        match self {
245            Self::Linear(roles) => {
246                let l = qwen35::WorkspaceLayout::checked(roles.geometry)?;
247                Ok(AttentionRequirements {
248                    history: l.history,
249                    recurrent: l.recurrent,
250                    kv: 0,
251                    workspace: l.floats,
252                })
253            }
254            Self::Full(roles) => {
255                let l = full_attention::WorkspaceLayout::checked(roles.geometry, max_context)?;
256                Ok(AttentionRequirements {
257                    history: 0,
258                    recurrent: 0,
259                    kv: l.kv.checked_mul(max_context).ok_or(Error::Overflow)?,
260                    workspace: l.floats,
261                })
262            }
263        }
264    }
265
266    /// The layer kernels publish only on success. The containing token owner
267    /// must poison a partially executed multi-layer token and require reset.
268    #[allow(clippy::too_many_arguments)]
269    pub fn execute(
270        &self,
271        host: &impl Host,
272        normalized: &[f32],
273        hidden: &mut [f32],
274        history: &mut [f32],
275        recurrent: &mut [f32],
276        keys: &mut [f32],
277        values: &mut [f32],
278        position: usize,
279        inv_freq: &[f32],
280        epsilon: f32,
281        workspace: &mut [f32],
282    ) -> Result<(), Error> {
283        match self {
284            Self::Linear(roles) => {
285                if !keys.is_empty() || !values.is_empty() {
286                    return Err(Error::Shape);
287                }
288                qwen35::execute_normalized(
289                    host, roles, normalized, hidden, history, recurrent, epsilon, workspace,
290                )
291            }
292            Self::Full(roles) => {
293                if !history.is_empty() || !recurrent.is_empty() {
294                    return Err(Error::Shape);
295                }
296                let mut cursor = position;
297                full_attention::execute_normalized(
298                    host,
299                    roles,
300                    normalized,
301                    hidden,
302                    keys,
303                    values,
304                    &mut cursor,
305                    full_attention::TextRope { position, inv_freq },
306                    epsilon,
307                    workspace,
308                )
309            }
310        }
311    }
312}