1#![allow(dead_code, unused_variables, unused_mut)]
4use super::*;
5
6#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
7mod fused_tail;
8#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
9mod preproject;
10
11impl QTensorEngine {
12 pub(crate) fn matmul_dims(info: &GgufTensorInfo) -> (usize, usize) {
14 let n_in = info.dims[0] as usize;
15 let n_out = if info.n_dims > 1 && info.dims[1] > 0 {
16 info.dims[1] as usize
17 } else {
18 1
19 };
20 (n_in, n_out)
21 }
22
23 pub(crate) fn attention_gpu_params(
25 h: &crate::gguf_sharder::GgufHyperparams,
26 layout: &KvCacheLayout,
27 layer: u32,
28 token_idx: u32,
29 info: &GgufTensorInfo,
30 raw_len: usize,
31 proj_kind: u32,
32 num_tokens_in_batch: u32,
33 batch_start_token_idx: u32,
34 mask_active: u32,
35 mask_word_count: u32,
36 out_stride_elems: u32,
37 ) -> AttentionGpuParams {
38 AttentionGpuParams {
39 n_embd: h.n_embd,
40 n_head: h.n_head,
41 n_kv_head: h.effective_n_kv_head(),
42 head_dim: h.head_dim(),
43 q_heads_per_kv: h.q_heads_per_kv(),
44 token_idx,
45 max_context: layout.max_context,
46 layer_idx: layer,
47 layer_stride: layout.layer_stride,
48 slot_kv_elems: layout.slot_kv_elems,
49 weight_ggml_type: info.ggml_type,
50 weight_row_elems: info.dims[0] as u32,
51 weight_byte_len: raw_len as u32,
52 proj_kind,
53 rope_theta_base: h.effective_rope_freq_base(),
54 rope_scale: h.effective_rope_scale(),
55 num_tokens_in_batch,
56 batch_start_token_idx,
57 mask_active,
58 mask_word_count,
59 out_stride_elems,
60 proj_row_stride: 0, kv_quant: if layout.int8 { 1 } else { 0 },
62 dict_pack: if layout.dict_k > 0 {
64 layout.dict_k | (layout.dict_n_atoms << 16)
65 } else {
66 0
67 },
68 }
69 }
70
71 #[inline]
72 pub(crate) fn attention_kv_mask_for_dispatch(
74 layout: &KvCacheLayout,
75 token_idx: u32,
76 proj_kind: u32,
77 ) -> ([u32; KV_ATTENTION_MASK_WORDS], u32, u32) {
78 if proj_kind != 0 {
79 return ([0u32; KV_ATTENTION_MASK_WORDS], 0, 0);
80 }
81 let (words, active) =
82 crate::compute_universe::attention_kv_mask_u32(token_idx, layout.max_context);
83 (words, active, KV_ATTENTION_MASK_WORDS as u32)
84 }
85
86 pub(crate) fn dispatch_attention_pass(
88 &self,
89 hidden: &[f32],
90 n_embd: usize,
91 num_tokens_in_batch: u32,
92 batch_start_token_idx: u32,
93 layout: &KvCacheLayout,
94 layer: u32,
95 token_idx: u32,
96 h: &crate::gguf_sharder::GgufHyperparams,
97 info: &GgufTensorInfo,
98 raw_weights: &[u8],
99 proj_kind: u32,
100 n_workgroups: u32,
101 norm_weight: Option<&[f32]>,
102 readback_out: Option<&mut [f32]>,
103 ) -> bool {
104 debug_assert!(token_idx < layout.max_context || layout.max_context == 0);
105 debug_assert!(n_workgroups > 0);
106 if info.ggml_type == crate::ternary::GGML_TYPE_TERNARY_158 {
109 wlog("[attention] FAILED: ternary weights not supported for attention (MAC reserved)");
110 return false;
111 }
112
113 #[cfg(not(target_arch = "wasm32"))]
116 if crate::llm_bench::cpu_attention_enabled() {
117 return self.cpu_attention_pass(
118 hidden,
119 n_embd,
120 num_tokens_in_batch,
121 batch_start_token_idx,
122 layout,
123 layer,
124 h,
125 info,
126 raw_weights,
127 proj_kind,
128 norm_weight,
129 readback_out,
130 );
131 }
132 #[cfg(any(target_arch = "wasm32", not(feature = "gpu-runtime")))]
133 return self.cpu_attention_pass(
134 hidden,
135 n_embd,
136 num_tokens_in_batch,
137 batch_start_token_idx,
138 layout,
139 layer,
140 h,
141 info,
142 raw_weights,
143 proj_kind,
144 norm_weight,
145 readback_out,
146 );
147
148 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
149 if !ggml_gpu_attention_shader_supported(info.ggml_type) {
150 wlog(&format!(
151 "[attn_pass] GUARD unsupported quant kind={proj_kind}"
152 ));
153 return false;
154 }
155 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
156 let batch = num_tokens_in_batch.max(1) as usize;
157 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
158 let hidden_elems = n_embd.checked_mul(batch).unwrap_or(0);
159 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
160 if hidden_elems > hidden.len()
161 || hidden_elems > self.gemm_max_input_floats
162 || raw_weights.len() > self.max_tensor_bytes
163 || self.gemm_input_buf.is_none()
164 || self.kv_cache_gpu.is_none()
165 || self.attention_params_buf.is_none()
166 || self.attention_mask_buf.is_none()
167 {
168 wlog(&format!(
169 "[attn_pass] GUARD buffers kind={proj_kind} hidden_elems={hidden_elems} hidden={} gemm_in={} raw_w={} max_w={} gemm_in_buf={} kv_gpu={} params={} mask={}",
170 hidden.len(),
171 self.gemm_max_input_floats,
172 raw_weights.len(),
173 self.max_tensor_bytes,
174 self.gemm_input_buf.is_some(),
175 self.kv_cache_gpu.is_some(),
176 self.attention_params_buf.is_some(),
177 self.attention_mask_buf.is_some(),
178 ));
179 if crate::gguf_bridge::execution_env::var("QUALIA_LLM_DEBUG_DECODE").is_ok() {
180 eprintln!(
181 "[attn_pass] GUARD kind={proj_kind} gemm_in_buf={} kv_gpu={} params={} mask={} hidden_elems={} gemm_max_in={} raw_w={} max_w={}",
182 self.gemm_input_buf.is_some(),
183 self.kv_cache_gpu.is_some(),
184 self.attention_params_buf.is_some(),
185 self.attention_mask_buf.is_some(),
186 hidden_elems,
187 self.gemm_max_input_floats,
188 raw_weights.len(),
189 self.max_tensor_bytes,
190 );
191 }
192 return false;
193 }
194
195 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
196 {
197 let (mask_words, mask_active, mask_word_count) =
198 Self::attention_kv_mask_for_dispatch(layout, token_idx, proj_kind);
199 let mut params = Self::attention_gpu_params(
200 h,
201 layout,
202 layer,
203 token_idx,
204 info,
205 raw_weights.len(),
206 proj_kind,
207 num_tokens_in_batch.max(1),
208 batch_start_token_idx,
209 mask_active,
210 mask_word_count,
211 0,
212 );
213 if n_embd != h.n_embd as usize {
214 params.proj_row_stride = n_embd as u32;
215 }
216 let input_buf = self.gemm_input_buf.as_ref().unwrap();
217 let weight_buf = self.gemm_weight_buf.as_ref().unwrap();
218 let output_buf = self.gemm_output_buf.as_ref().unwrap();
219 let params_buf = self.attention_params_buf.as_ref().unwrap();
220 let mask_buf = self.attention_mask_buf.as_ref().unwrap();
221 let kv_buf = self.kv_cache_gpu.as_ref().unwrap();
222 let staging = self.gemm_output_staging.as_ref().unwrap();
223
224 etalon_inference::gpu_attention::upload_input(
225 self.gpu_queue(),
226 input_buf,
227 hidden,
228 n_embd,
229 batch,
230 norm_weight,
231 );
232 let resident = if crate::llm_bench::resident_weights_enabled() {
238 self.resident_weight_buffer(raw_weights.as_ptr() as u64, raw_weights)
239 } else {
240 None
241 };
242 let weight_binding: &etalon::Buffer = match resident.as_ref() {
243 Some(r) => r,
244 None => {
245 self.write_weight_words(raw_weights, self.max_tensor_bytes);
246 weight_buf
247 }
248 };
249
250 let resources = etalon_inference::gpu_attention::PassResources {
251 input: input_buf,
252 weight: weight_binding,
253 output: output_buf,
254 params: params_buf,
255 mask: mask_buf,
256 kv: kv_buf,
257 staging,
258 bind_layout: &self.attention_bind_layout,
259 pipeline: &self.attention_pipeline,
260 };
261 let readback_elems = readback_out.as_ref().map(|o| o.len()).unwrap_or(0);
262 let mut encoder = match etalon_inference::gpu_attention::encode_pass(
263 self.gpu_device(),
264 self.gpu_queue(),
265 &resources,
266 ¶ms,
267 &mask_words,
268 n_workgroups,
269 readback_elems,
270 crate::llm_gpu_profiler::pass_writes_both(),
271 ) {
272 Some(encoder) => encoder,
273 None => return false,
274 };
275 crate::llm_gpu_profiler::resolve(&mut encoder);
276 self.submit_commands(Some(encoder.finish()));
277 crate::llm_gpu_profiler::accumulate(crate::llm_gpu_profiler::Phase::Attention);
278
279 if readback_elems == 0 {
280 return true;
281 }
282
283 let out_bytes = (readback_elems * 4) as etalon::BufferAddress;
284 let slice = staging.slice(..out_bytes);
285 let (tx, rx) = futures_channel::oneshot::channel();
286 slice.map_async(etalon::MapMode::Read, move |r| {
287 let _ = tx.send(r);
288 });
289 self.poll_wait();
290 #[cfg(not(target_arch = "wasm32"))]
291 if let Ok(handle) = tokio::runtime::Handle::try_current() {
292 if handle.block_on(rx).ok().map(|m| m.is_ok()).unwrap_or(false) {
293 let data = slice
294 .get_mapped_range()
295 .expect("wgpu buffer map_range failed");
296 let floats: &[f32] = bytemuck::cast_slice(&data);
297 if let Some(out) = readback_out {
298 out[..readback_elems].copy_from_slice(&floats[..readback_elems]);
299 }
300 drop(data);
301 staging.unmap();
302 return true;
303 }
304 }
305 let _ = staging.unmap();
306 false
307 }
308 }
309
310 pub(crate) fn cpu_attention_pass(
314 &self,
315 hidden: &[f32],
316 n_embd: usize,
317 num_tokens_in_batch: u32,
318 batch_start_token_idx: u32,
319 layout: &KvCacheLayout,
320 layer: u32,
321 h: &crate::gguf_sharder::GgufHyperparams,
322 info: &GgufTensorInfo,
323 raw_weights: &[u8],
324 proj_kind: u32,
325 norm_weight: Option<&[f32]>,
326 mut readback_out: Option<&mut [f32]>,
327 ) -> bool {
328 let cache = match self.kv_cache_cpu.as_ref() {
329 Some(cache) => cache,
330 None => return false,
331 };
332 let kv =
334 unsafe { core::slice::from_raw_parts_mut(cache.as_ptr() as *mut f32, cache.len()) };
335 let dict_mode = layout.dict_k > 0;
339 #[cfg(not(target_arch = "wasm32"))]
340 let (k_dict, v_dict) = if dict_mode {
341 match proj_kind {
342 1 => (
343 crate::kv_dict_runtime::clone_layer_dict(layer as usize, true),
344 None,
345 ),
346 2 => (
347 None,
348 crate::kv_dict_runtime::clone_layer_dict(layer as usize, false),
349 ),
350 _ => (
351 crate::kv_dict_runtime::clone_layer_dict(layer as usize, true),
352 crate::kv_dict_runtime::clone_layer_dict(layer as usize, false),
353 ),
354 }
355 } else {
356 (None, None)
357 };
358 #[cfg(target_arch = "wasm32")]
359 let (k_dict, v_dict): (
360 Option<crate::kv_dict::KvDictionary>,
361 Option<crate::kv_dict::KvDictionary>,
362 ) = (None, None);
363
364 let host = super::cpu_attention_adapter::CpuAttentionAdapter {
365 k: k_dict.as_ref(),
366 v: v_dict.as_ref(),
367 };
368 let layout = etalon_inference::ordinary::cpu_attention::CpuKvLayout {
369 max_context: layout.max_context,
370 layer_stride: layout.layer_stride,
371 slot_kv_elems: layout.slot_kv_elems,
372 n_kv_head: layout.n_kv_head,
373 head_dim: layout.head_dim,
374 dict_k: layout.dict_k,
375 };
376 etalon_inference::ordinary::cpu_attention::cpu_attention_pass(
377 &host,
378 kv,
379 hidden,
380 n_embd,
381 num_tokens_in_batch,
382 batch_start_token_idx,
383 &layout,
384 layer,
385 &(*h).into(),
386 info,
387 raw_weights,
388 proj_kind,
389 norm_weight,
390 readback_out,
391 )
392 }
393
394 pub(crate) fn dispatch_attention_layer(
396 &mut self,
397 index: &crate::gguf_sharder::GgufTensorIndex,
398 layer: u32,
399 token_idx: u32,
400 hidden: &[f32],
401 emb_dim: usize,
402 tensors: &crate::gguf_sharder::LayerTensors,
403 scratch_a: &mut [f32],
404 scratch_b: &mut [f32],
405 ) -> Option<usize> {
406 if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
407 return None; }
409 if Self::is_qwen35_gated_attention_layer(index, tensors) {
410 return self.dispatch_qwen35_gated_attention_layer(
411 index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
412 );
413 }
414 etalon_inference::ordinary::attention::AttentionEngine { host: self }
415 .dispatch_attention_layer(
416 index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
417 )
418 }
419
420 pub(crate) fn dispatch_attention_q_ffn_token(
422 &mut self,
423 index: &crate::gguf_sharder::GgufTensorIndex,
424 layer: u32,
425 token_idx: u32,
426 hidden: &mut [f32],
427 emb_dim: usize,
428 tensors: &crate::gguf_sharder::LayerTensors,
429 scratch_a: &mut [f32],
430 scratch_b: &mut [f32],
431 ) -> bool {
432 if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
433 return false;
434 }
435 etalon_inference::ordinary::attention::AttentionEngine { host: self }
436 .dispatch_attention_q_ffn_token(
437 index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
438 )
439 }
440
441 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
445 pub(super) fn try_cuda_soa_attention_layer(
446 &self,
447 hidden_input: &[f32],
448 n_embd: usize,
449 q_dim: usize,
450 kv_dim: usize,
451 n_head: usize,
452 n_kv: usize,
453 head_dim: usize,
454 layout: &KvCacheLayout,
455 layer: u32,
456 token_idx: u32,
457 h: &crate::gguf_sharder::GgufHyperparams,
458 q_raw: &[u8],
459 k_raw: &[u8],
460 v_raw: &[u8],
461 out_info: Option<&GgufTensorInfo>,
462 mmap: &[u8],
463 tensor_data_start: u64,
464 scratch_a: &mut [f32],
465 scratch_b: &mut [f32],
466 ) -> bool {
467 let layout = etalon_inference::ordinary::cuda_attention::CudaAttentionLayout {
468 max_context: layout.max_context,
469 n_layer: layout.n_layer,
470 n_kv_head: layout.n_kv_head,
471 head_dim: layout.head_dim,
472 slot_kv_elems: layout.slot_kv_elems,
473 layer_stride: layout.layer_stride,
474 total_f32_elems: layout.total_f32_elems,
475 int8: layout.int8,
476 dict_k: layout.dict_k,
477 };
478 etalon_inference::ordinary::cuda_attention::try_cuda_soa_attention_layer(
479 self,
480 hidden_input,
481 n_embd,
482 q_dim,
483 kv_dim,
484 n_head,
485 n_kv,
486 head_dim,
487 &layout,
488 layer,
489 token_idx,
490 &(*h).into(),
491 q_raw,
492 k_raw,
493 v_raw,
494 out_info,
495 mmap,
496 tensor_data_start,
497 scratch_a,
498 scratch_b,
499 )
500 }
501
502 #[cfg(all(not(target_arch = "wasm32"), feature = "wgsl-forge"))]
505 pub fn dispatch_forge_causal_attention(
506 &mut self,
507 query: &[f32],
508 keys: &[f32],
509 values: &[f32],
510 scale: f32,
511 shape: crate::wgsl_forge::AttentionShape,
512 out: &mut [f32],
513 ) -> Result<(), crate::wgsl_forge::ForgeError> {
514 let schedule = crate::wgsl_forge::AttentionSchedule::a2000_decode();
515 schedule.validate(shape)?;
516 let q_len = (shape.head_count * shape.head_dim) as usize;
517 let kv_len = (shape.head_count * shape.context_tokens * shape.head_dim) as usize;
518 if query.len() < q_len || keys.len() < kv_len || values.len() < kv_len || out.len() < q_len
519 {
520 return Err(crate::wgsl_forge::ForgeError::InvalidKernel(
521 "dispatch_forge_causal_attention buffers smaller than required shape".into(),
522 ));
523 }
524
525 let shader = crate::wgsl_forge::synthesize_causal_attention(shape, schedule)?;
526 let params = crate::wgsl_forge::AttentionShaderParams {
527 context_tokens: shape.context_tokens,
528 head_dim: shape.head_dim,
529 head_count: shape.head_count,
530 _pad0: 0,
531 scale,
532 _pad1: 0,
533 _pad2: 0,
534 _pad3: 0,
535 };
536 if etalon_inference::gpu_attention::forge::dispatch_forge_attention(
537 self,
538 &shader.source,
539 bytemuck::bytes_of(¶ms),
540 q_len,
541 kv_len,
542 shape.head_count,
543 query,
544 keys,
545 values,
546 out,
547 ) {
548 return Ok(());
549 }
550
551 crate::wgsl_forge::causal_attention_oracle(shape, query, keys, values, scale, out)
552 }
553}