qualia_core_db/gguf_bridge/qwen35/ftw/
attention.rs1use super::Profile;
5use crate::inference::moe::ftw_loader::FtwModelPackage;
6use etalon_inference::ordinary::qwen35::{self, full_attention, Error, Host};
7use etalon_model::inference::{
8 qwen_attention::Qwen35AttentionRoles,
9 qwen_roles::{Projection, Qwen35Roles},
10 TensorInfo, WeightEncoding,
11};
12
13fn tensor(pkg: &FtwModelPackage, layer: u32, suffix: &str) -> Result<Option<TensorInfo>, String> {
14 let name = format!("model.layers.{layer}.{suffix}");
15 let mut matches = pkg
16 .manifest
17 .tensors
18 .iter()
19 .filter(|entry| entry.name == name);
20 let Some(entry) = matches.next() else {
21 return Ok(None);
22 };
23 if matches.next().is_some() {
24 return Err(format!("ambiguous Qwen tensor {name}"));
25 }
26 if entry.shape.is_empty() || entry.shape.len() > 4 || entry.shape.contains(&0) {
27 return Err(format!("invalid Qwen tensor shape {name}"));
28 }
29 let ty = match entry.dtype.as_str() {
30 "float32" => crate::ggml_quants::GGML_TYPE_F32,
31 "float16" => crate::ggml_quants::GGML_TYPE_F16,
32 "bfloat16" => crate::ggml_quants::GGML_TYPE_BF16,
33 "float8_e4m3fn" => crate::ggml_quants::GGML_TYPE_FP8_E4M3,
34 _ => return Err(format!("unsupported Qwen attention tensor dtype {name}")),
35 };
36 let mut dims = [0; 4];
37 for (dst, &value) in dims.iter_mut().zip(entry.shape.iter().rev()) {
38 *dst = value;
39 }
40 let info = TensorInfo {
41 dims,
42 n_dims: entry.shape.len() as u32,
43 ggml_type: ty,
44 byte_offset: entry.global_off,
45 };
46 let bytes = crate::ggml_quants::tensor_byte_len(&info).ok_or("Qwen tensor size overflow")?;
47 if bytes as u64 != entry.nbytes
48 || pkg
49 .fetch_by_global_offset(info.byte_offset, bytes)
50 .is_none()
51 {
52 return Err(format!("Qwen attention tensor bounds {name}"));
53 }
54 Ok(Some(info))
55}
56fn projection(
57 pkg: &FtwModelPackage,
58 layer: u32,
59 stem: &str,
60 cols: usize,
61 rows: usize,
62) -> Result<Projection, String> {
63 let weight = tensor(pkg, layer, &format!("{stem}.weight"))?.ok_or("missing Qwen projection")?;
64 let row_scale = tensor(pkg, layer, &format!("{stem}.weight_scale"))?;
65 if weight.n_dims != 2 || weight.dims[..2] != [cols as u64, rows as u64] {
66 return Err("Qwen attention projection shape".into());
67 }
68 match weight.encoding() {
69 WeightEncoding::Float8E4M3 => {
70 let scale = row_scale.ok_or("missing Qwen FP8 row scales")?;
71 if scale.n_dims != 1
72 || scale.dims[0] != rows as u64
73 || !matches!(
74 scale.encoding(),
75 WeightEncoding::IeeeF32 | WeightEncoding::IeeeF16 | WeightEncoding::BrainF16
76 )
77 {
78 return Err("Qwen FP8 row scale shape/encoding".into());
79 }
80 }
81 WeightEncoding::IeeeF32 | WeightEncoding::IeeeF16 | WeightEncoding::BrainF16
82 if row_scale.is_none() => {}
83 _ => return Err("unsupported Qwen attention projection encoding".into()),
84 }
85 Ok(Projection { weight, row_scale })
86}
87
88#[derive(Clone, Copy, Debug)]
89pub enum AttentionPlan {
90 Linear(Qwen35Roles),
91 Full(Qwen35AttentionRoles),
92}
93
94#[cfg(test)]
95#[path = "attention_tests.rs"]
96mod tests;
97
98#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
99pub struct AttentionRequirements {
100 pub history: usize,
101 pub recurrent: usize,
102 pub kv: usize,
104 pub workspace: usize,
105}
106
107pub fn resolve(
108 pkg: &FtwModelPackage,
109 profile: &Profile,
110 layer: usize,
111 is_linear: bool,
112) -> Result<AttentionPlan, String> {
113 let layer = u32::try_from(layer).map_err(|_| "Qwen layer index overflow")?;
114 let index = &pkg.tensor_index;
115 if is_linear {
116 let roles = Qwen35Roles::resolve(index, layer, profile.packing)
117 .map_err(|e| format!("Qwen linear layer {layer} roles: {e:?}"))?;
118 if roles.geometry != profile.linear_geometry {
119 return Err(format!("Qwen linear layer {layer} geometry mismatch"));
120 }
121 if tensor(pkg, layer, "self_attn.q_proj.weight")
123 .map_err(|e| format!("Qwen layer {layer} attention ambiguity: {e:?}"))?
124 .is_some()
125 || tensor(pkg, layer, "self_attn.qkv_proj.weight")?.is_some()
126 {
127 return Err(format!("Qwen linear layer {layer} also has full attention"));
128 }
129 Ok(AttentionPlan::Linear(roles))
130 } else {
131 let g = profile.attention_geometry;
132 g.validate()
133 .map_err(|e| format!("Qwen attention geometry: {e:?}"))?;
134 let query = g
135 .query_heads
136 .checked_mul(g.head_dim)
137 .ok_or("Qwen query overflow")?;
138 let kv = g
139 .kv_heads
140 .checked_mul(g.head_dim)
141 .ok_or("Qwen KV overflow")?;
142 let qg = query.checked_mul(2).ok_or("Qwen query gate overflow")?;
143 let rows = kv
144 .checked_mul(2)
145 .and_then(|n| n.checked_add(qg))
146 .ok_or("Qwen QKV overflow")?;
147 let fused = projection(pkg, layer, "self_attn.qkv_proj", g.hidden, rows)
151 .map_err(|e| format!("Qwen fused attention layer {layer}: {e:?}"))?;
152 let required = |name| {
153 tensor(pkg, layer, name)
154 .map_err(|e| format!("Qwen attention layer {layer}: {e:?}"))?
155 .ok_or_else(|| format!("Qwen attention layer {layer} missing {name}"))
156 };
157 let roles = Qwen35AttentionRoles {
158 geometry: g,
159 q_gate: row_range(fused, 0, qg)?,
160 k: row_range(fused, qg, kv)?,
161 v: row_range(fused, qg + kv, kv)?,
162 out: projection(pkg, layer, "self_attn.o_proj", query, g.hidden)
163 .map_err(|e| format!("Qwen attention output layer {layer}: {e:?}"))?,
164 q_norm: required("self_attn.q_norm.weight")?,
165 k_norm: required("self_attn.k_norm.weight")?,
166 norm: profile.ordinary_norm,
167 };
168 roles
169 .validate()
170 .map_err(|e| format!("Qwen attention layer {layer} roles: {e:?}"))?;
171 for name in [
172 "linear_attn.in_proj_qkvz.weight",
173 "linear_attn.in_proj_qkv.weight",
174 "self_attn.q_proj.weight",
175 "self_attn.k_proj.weight",
176 "self_attn.v_proj.weight",
177 "self_attn.qkv_proj.bias",
178 "self_attn.q_proj.bias",
179 "self_attn.k_proj.bias",
180 "self_attn.v_proj.bias",
181 "self_attn.o_proj.bias",
182 ] {
183 if tensor(pkg, layer, name)
184 .map_err(|e| format!("Qwen layer {layer} linear ambiguity: {e:?}"))?
185 .is_some()
186 {
187 return Err(format!(
188 "Qwen full attention layer {layer} has unsupported mixed role {name}"
189 ));
190 }
191 }
192 Ok(AttentionPlan::Full(roles))
193 }
194}
195
196fn row_range(mut projection: Projection, start: usize, rows: usize) -> Result<Projection, String> {
198 if rows == 0
199 || start
200 .checked_add(rows)
201 .is_none_or(|end| end > projection.rows())
202 {
203 return Err("Qwen fused projection row bounds".into());
204 }
205 let width = match projection.weight.encoding() {
206 WeightEncoding::IeeeF32 => 4,
207 WeightEncoding::IeeeF16 | WeightEncoding::BrainF16 => 2,
208 WeightEncoding::Float8E4M3 => 1,
209 _ => return Err("Qwen fused projection encoding".into()),
210 };
211 let offset = start
212 .checked_mul(projection.cols())
213 .and_then(|n| n.checked_mul(width))
214 .and_then(|n| u64::try_from(n).ok())
215 .ok_or("Qwen fused projection offset overflow")?;
216 projection.weight.byte_offset = projection
217 .weight
218 .byte_offset
219 .checked_add(offset)
220 .ok_or("Qwen fused projection address overflow")?;
221 projection.weight.dims[1] = rows as u64;
222 if let Some(scale) = &mut projection.row_scale {
223 let width = match scale.encoding() {
225 WeightEncoding::IeeeF32 => 4,
226 WeightEncoding::IeeeF16 | WeightEncoding::BrainF16 => 2,
227 _ => return Err("Qwen fused scale encoding".into()),
228 };
229 let offset = start
230 .checked_mul(width)
231 .and_then(|n| u64::try_from(n).ok())
232 .ok_or("Qwen fused scale offset overflow")?;
233 scale.byte_offset = scale
234 .byte_offset
235 .checked_add(offset)
236 .ok_or("Qwen fused scale address overflow")?;
237 scale.dims[0] = rows as u64;
238 }
239 Ok(projection)
240}
241
242impl AttentionPlan {
243 pub fn requirements(&self, max_context: usize) -> Result<AttentionRequirements, Error> {
244 match self {
245 Self::Linear(roles) => {
246 let l = qwen35::WorkspaceLayout::checked(roles.geometry)?;
247 Ok(AttentionRequirements {
248 history: l.history,
249 recurrent: l.recurrent,
250 kv: 0,
251 workspace: l.floats,
252 })
253 }
254 Self::Full(roles) => {
255 let l = full_attention::WorkspaceLayout::checked(roles.geometry, max_context)?;
256 Ok(AttentionRequirements {
257 history: 0,
258 recurrent: 0,
259 kv: l.kv.checked_mul(max_context).ok_or(Error::Overflow)?,
260 workspace: l.floats,
261 })
262 }
263 }
264 }
265
266 #[allow(clippy::too_many_arguments)]
269 pub fn execute(
270 &self,
271 host: &impl Host,
272 normalized: &[f32],
273 hidden: &mut [f32],
274 history: &mut [f32],
275 recurrent: &mut [f32],
276 keys: &mut [f32],
277 values: &mut [f32],
278 position: usize,
279 inv_freq: &[f32],
280 epsilon: f32,
281 workspace: &mut [f32],
282 ) -> Result<(), Error> {
283 match self {
284 Self::Linear(roles) => {
285 if !keys.is_empty() || !values.is_empty() {
286 return Err(Error::Shape);
287 }
288 qwen35::execute_normalized(
289 host, roles, normalized, hidden, history, recurrent, epsilon, workspace,
290 )
291 }
292 Self::Full(roles) => {
293 if !history.is_empty() || !recurrent.is_empty() {
294 return Err(Error::Shape);
295 }
296 let mut cursor = position;
297 full_attention::execute_normalized(
298 host,
299 roles,
300 normalized,
301 hidden,
302 keys,
303 values,
304 &mut cursor,
305 full_attention::TextRope { position, inv_freq },
306 epsilon,
307 workspace,
308 )
309 }
310 }
311 }
312}