Skip to main content

qualia_core_db/gguf_bridge/
attention.rs

1//! Attention: GQA params, KV mask, attention pass, CPU SDPA ref, layer + q/ffn dispatch
2//! Split from gguf_bridge/mod.rs (structural refactor; no behaviour change).
3#![allow(dead_code, unused_variables, unused_mut)]
4use super::*;
5
6#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
7mod fused_tail;
8#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
9mod preproject;
10
11impl QTensorEngine {
12    /// `QTensorEngine::matmul_dims`.
13    pub(crate) fn matmul_dims(info: &GgufTensorInfo) -> (usize, usize) {
14        let n_in = info.dims[0] as usize;
15        let n_out = if info.n_dims > 1 && info.dims[1] > 0 {
16            info.dims[1] as usize
17        } else {
18            1
19        };
20        (n_in, n_out)
21    }
22
23    /// Builds WGSL attention params from the KV layout.
24    pub(crate) fn attention_gpu_params(
25        h: &crate::gguf_sharder::GgufHyperparams,
26        layout: &KvCacheLayout,
27        layer: u32,
28        token_idx: u32,
29        info: &GgufTensorInfo,
30        raw_len: usize,
31        proj_kind: u32,
32        num_tokens_in_batch: u32,
33        batch_start_token_idx: u32,
34        mask_active: u32,
35        mask_word_count: u32,
36        out_stride_elems: u32,
37    ) -> AttentionGpuParams {
38        AttentionGpuParams {
39            n_embd: h.n_embd,
40            n_head: h.n_head,
41            n_kv_head: h.effective_n_kv_head(),
42            head_dim: h.head_dim(),
43            q_heads_per_kv: h.q_heads_per_kv(),
44            token_idx,
45            max_context: layout.max_context,
46            layer_idx: layer,
47            layer_stride: layout.layer_stride,
48            slot_kv_elems: layout.slot_kv_elems,
49            weight_ggml_type: info.ggml_type,
50            weight_row_elems: info.dims[0] as u32,
51            weight_byte_len: raw_len as u32,
52            proj_kind,
53            rope_theta_base: h.effective_rope_freq_base(),
54            rope_scale: h.effective_rope_scale(),
55            num_tokens_in_batch,
56            batch_start_token_idx,
57            mask_active,
58            mask_word_count,
59            out_stride_elems,
60            proj_row_stride: 0, // default = legacy in-shader projection; mc8_stage overrides for B
61            kv_quant: if layout.int8 { 1 } else { 0 },
62            // W5b Phase 4b: dict_k (low 16) | n_atoms (high 16); 0 ⇒ f32/int8.
63            dict_pack: if layout.dict_k > 0 {
64                layout.dict_k | (layout.dict_n_atoms << 16)
65            } else {
66                0
67            },
68        }
69    }
70
71    #[inline]
72    /// `QTensorEngine::attention_kv_mask_for_dispatch`.
73    pub(crate) fn attention_kv_mask_for_dispatch(
74        layout: &KvCacheLayout,
75        token_idx: u32,
76        proj_kind: u32,
77    ) -> ([u32; KV_ATTENTION_MASK_WORDS], u32, u32) {
78        if proj_kind != 0 {
79            return ([0u32; KV_ATTENTION_MASK_WORDS], 0, 0);
80        }
81        let (words, active) =
82            crate::compute_universe::attention_kv_mask_u32(token_idx, layout.max_context);
83        (words, active, KV_ATTENTION_MASK_WORDS as u32)
84    }
85
86    /// Single fused-attention dispatch: K write, V write, or Q+online-softmax.
87    pub(crate) fn dispatch_attention_pass(
88        &self,
89        hidden: &[f32],
90        n_embd: usize,
91        num_tokens_in_batch: u32,
92        batch_start_token_idx: u32,
93        layout: &KvCacheLayout,
94        layer: u32,
95        token_idx: u32,
96        h: &crate::gguf_sharder::GgufHyperparams,
97        info: &GgufTensorInfo,
98        raw_weights: &[u8],
99        proj_kind: u32,
100        n_workgroups: u32,
101        norm_weight: Option<&[f32]>,
102        readback_out: Option<&mut [f32]>,
103    ) -> bool {
104        debug_assert!(token_idx < layout.max_context || layout.max_context == 0);
105        debug_assert!(n_workgroups > 0);
106        // Heterogeneous dispatch: ternary weights are strictly routed to adder paths (FFN),
107        // reserving the attention MAC units for high-precision projections.
108        if info.ggml_type == crate::ternary::GGML_TYPE_TERNARY_158 {
109            wlog("[attention] FAILED: ternary weights not supported for attention (MAC reserved)");
110            return false;
111        }
112
113        // #48 correctness path: route native attention through the CPU reference (the wasm-proven
114        // SDPA) when enabled — bypasses the GPU attention shader whose output is currently unbounded.
115        #[cfg(not(target_arch = "wasm32"))]
116        if crate::llm_bench::cpu_attention_enabled() {
117            return self.cpu_attention_pass(
118                hidden,
119                n_embd,
120                num_tokens_in_batch,
121                batch_start_token_idx,
122                layout,
123                layer,
124                h,
125                info,
126                raw_weights,
127                proj_kind,
128                norm_weight,
129                readback_out,
130            );
131        }
132        #[cfg(any(target_arch = "wasm32", not(feature = "gpu-runtime")))]
133        return self.cpu_attention_pass(
134            hidden,
135            n_embd,
136            num_tokens_in_batch,
137            batch_start_token_idx,
138            layout,
139            layer,
140            h,
141            info,
142            raw_weights,
143            proj_kind,
144            norm_weight,
145            readback_out,
146        );
147
148        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
149        if !ggml_gpu_attention_shader_supported(info.ggml_type) {
150            wlog(&format!(
151                "[attn_pass] GUARD unsupported quant kind={proj_kind}"
152            ));
153            return false;
154        }
155        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
156        let batch = num_tokens_in_batch.max(1) as usize;
157        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
158        let hidden_elems = n_embd.checked_mul(batch).unwrap_or(0);
159        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
160        if hidden_elems > hidden.len()
161            || hidden_elems > self.gemm_max_input_floats
162            || raw_weights.len() > self.max_tensor_bytes
163            || self.gemm_input_buf.is_none()
164            || self.kv_cache_gpu.is_none()
165            || self.attention_params_buf.is_none()
166            || self.attention_mask_buf.is_none()
167        {
168            wlog(&format!(
169                "[attn_pass] GUARD buffers kind={proj_kind} hidden_elems={hidden_elems} hidden={} gemm_in={} raw_w={} max_w={} gemm_in_buf={} kv_gpu={} params={} mask={}",
170                hidden.len(),
171                self.gemm_max_input_floats,
172                raw_weights.len(),
173                self.max_tensor_bytes,
174                self.gemm_input_buf.is_some(),
175                self.kv_cache_gpu.is_some(),
176                self.attention_params_buf.is_some(),
177                self.attention_mask_buf.is_some(),
178            ));
179            if crate::gguf_bridge::execution_env::var("QUALIA_LLM_DEBUG_DECODE").is_ok() {
180                eprintln!(
181                    "[attn_pass] GUARD kind={proj_kind} gemm_in_buf={} kv_gpu={} params={} mask={} hidden_elems={} gemm_max_in={} raw_w={} max_w={}",
182                    self.gemm_input_buf.is_some(),
183                    self.kv_cache_gpu.is_some(),
184                    self.attention_params_buf.is_some(),
185                    self.attention_mask_buf.is_some(),
186                    hidden_elems,
187                    self.gemm_max_input_floats,
188                    raw_weights.len(),
189                    self.max_tensor_bytes,
190                );
191            }
192            return false;
193        }
194
195        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
196        {
197            let (mask_words, mask_active, mask_word_count) =
198                Self::attention_kv_mask_for_dispatch(layout, token_idx, proj_kind);
199            let mut params = Self::attention_gpu_params(
200                h,
201                layout,
202                layer,
203                token_idx,
204                info,
205                raw_weights.len(),
206                proj_kind,
207                num_tokens_in_batch.max(1),
208                batch_start_token_idx,
209                mask_active,
210                mask_word_count,
211                0,
212            );
213            if n_embd != h.n_embd as usize {
214                params.proj_row_stride = n_embd as u32;
215            }
216            let input_buf = self.gemm_input_buf.as_ref().unwrap();
217            let weight_buf = self.gemm_weight_buf.as_ref().unwrap();
218            let output_buf = self.gemm_output_buf.as_ref().unwrap();
219            let params_buf = self.attention_params_buf.as_ref().unwrap();
220            let mask_buf = self.attention_mask_buf.as_ref().unwrap();
221            let kv_buf = self.kv_cache_gpu.as_ref().unwrap();
222            let staging = self.gemm_output_staging.as_ref().unwrap();
223
224            etalon_inference::gpu_attention::upload_input(
225                self.gpu_queue(),
226                input_buf,
227                hidden,
228                n_embd,
229                batch,
230                norm_weight,
231            );
232            // Phase 2 (attention): bind this projection's resident VRAM buffer (uploaded once, keyed by
233            // its mmap address) instead of re-uploading the Q/K/V weight into the shared buffer EVERY
234            // token. For a 3B that re-upload is ~30 MB/layer × n_layer ≈ ~0.8 GB/token of PCIe traffic
235            // the GEMM path already shed; attention was still paying it. Output is byte-identical (same
236            // bytes, same offsets). Falls back to the per-token upload when residency is off.
237            let resident = if crate::llm_bench::resident_weights_enabled() {
238                self.resident_weight_buffer(raw_weights.as_ptr() as u64, raw_weights)
239            } else {
240                None
241            };
242            let weight_binding: &etalon::Buffer = match resident.as_ref() {
243                Some(r) => r,
244                None => {
245                    self.write_weight_words(raw_weights, self.max_tensor_bytes);
246                    weight_buf
247                }
248            };
249
250            let resources = etalon_inference::gpu_attention::PassResources {
251                input: input_buf,
252                weight: weight_binding,
253                output: output_buf,
254                params: params_buf,
255                mask: mask_buf,
256                kv: kv_buf,
257                staging,
258                bind_layout: &self.attention_bind_layout,
259                pipeline: &self.attention_pipeline,
260            };
261            let readback_elems = readback_out.as_ref().map(|o| o.len()).unwrap_or(0);
262            let mut encoder = match etalon_inference::gpu_attention::encode_pass(
263                self.gpu_device(),
264                self.gpu_queue(),
265                &resources,
266                &params,
267                &mask_words,
268                n_workgroups,
269                readback_elems,
270                crate::llm_gpu_profiler::pass_writes_both(),
271            ) {
272                Some(encoder) => encoder,
273                None => return false,
274            };
275            crate::llm_gpu_profiler::resolve(&mut encoder);
276            self.submit_commands(Some(encoder.finish()));
277            crate::llm_gpu_profiler::accumulate(crate::llm_gpu_profiler::Phase::Attention);
278
279            if readback_elems == 0 {
280                return true;
281            }
282
283            let out_bytes = (readback_elems * 4) as etalon::BufferAddress;
284            let slice = staging.slice(..out_bytes);
285            let (tx, rx) = futures_channel::oneshot::channel();
286            slice.map_async(etalon::MapMode::Read, move |r| {
287                let _ = tx.send(r);
288            });
289            self.poll_wait();
290            #[cfg(not(target_arch = "wasm32"))]
291            if let Ok(handle) = tokio::runtime::Handle::try_current() {
292                if handle.block_on(rx).ok().map(|m| m.is_ok()).unwrap_or(false) {
293                    let data = slice
294                        .get_mapped_range()
295                        .expect("wgpu buffer map_range failed");
296                    let floats: &[f32] = bytemuck::cast_slice(&data);
297                    if let Some(out) = readback_out {
298                        out[..readback_elems].copy_from_slice(&floats[..readback_elems]);
299                    }
300                    drop(data);
301                    staging.unmap();
302                    return true;
303                }
304            }
305            let _ = staging.unmap();
306            false
307        }
308    }
309
310    /// WASM CPU attention fallback (Phase 2A). Projects one Q/K/V tensor for a batch of
311    /// tokens, applies RoPE (Q/K), writes K/V into `kv_cache_cpu`, and runs SDPA for Q.
312    /// `proj_kind`: 0=Q, 1=K, 2=V. Available on native too as the #48 correctness reference.
313    pub(crate) fn cpu_attention_pass(
314        &self,
315        hidden: &[f32],
316        n_embd: usize,
317        num_tokens_in_batch: u32,
318        batch_start_token_idx: u32,
319        layout: &KvCacheLayout,
320        layer: u32,
321        h: &crate::gguf_sharder::GgufHyperparams,
322        info: &GgufTensorInfo,
323        raw_weights: &[u8],
324        proj_kind: u32,
325        norm_weight: Option<&[f32]>,
326        mut readback_out: Option<&mut [f32]>,
327    ) -> bool {
328        let cache = match self.kv_cache_cpu.as_ref() {
329            Some(cache) => cache,
330            None => return false,
331        };
332        // Legacy engine cache is exclusively used by the admitted decode thread.
333        let kv =
334            unsafe { core::slice::from_raw_parts_mut(cache.as_ptr() as *mut f32, cache.len()) };
335        // W5b Phase 4b: dict-coded KV cache. When the layout stores sparse codes, clone this layer's
336        // dictionaries once for the call — the write path encodes K/V to codes, the read path (Q pass)
337        // reconstructs them. `dict_k = 0` ⇒ the untouched f32/int8 paths below.
338        let dict_mode = layout.dict_k > 0;
339        #[cfg(not(target_arch = "wasm32"))]
340        let (k_dict, v_dict) = if dict_mode {
341            match proj_kind {
342                1 => (
343                    crate::kv_dict_runtime::clone_layer_dict(layer as usize, true),
344                    None,
345                ),
346                2 => (
347                    None,
348                    crate::kv_dict_runtime::clone_layer_dict(layer as usize, false),
349                ),
350                _ => (
351                    crate::kv_dict_runtime::clone_layer_dict(layer as usize, true),
352                    crate::kv_dict_runtime::clone_layer_dict(layer as usize, false),
353                ),
354            }
355        } else {
356            (None, None)
357        };
358        #[cfg(target_arch = "wasm32")]
359        let (k_dict, v_dict): (
360            Option<crate::kv_dict::KvDictionary>,
361            Option<crate::kv_dict::KvDictionary>,
362        ) = (None, None);
363
364        let host = super::cpu_attention_adapter::CpuAttentionAdapter {
365            k: k_dict.as_ref(),
366            v: v_dict.as_ref(),
367        };
368        let layout = etalon_inference::ordinary::cpu_attention::CpuKvLayout {
369            max_context: layout.max_context,
370            layer_stride: layout.layer_stride,
371            slot_kv_elems: layout.slot_kv_elems,
372            n_kv_head: layout.n_kv_head,
373            head_dim: layout.head_dim,
374            dict_k: layout.dict_k,
375        };
376        etalon_inference::ordinary::cpu_attention::cpu_attention_pass(
377            &host,
378            kv,
379            hidden,
380            n_embd,
381            num_tokens_in_batch,
382            batch_start_token_idx,
383            &layout,
384            layer,
385            &(*h).into(),
386            info,
387            raw_weights,
388            proj_kind,
389            norm_weight,
390            readback_out,
391        )
392    }
393
394    /// GPU-fused Q/K/V projections, RoPE, ring-buffer KV write, and GQA online-softmax.
395    pub(crate) fn dispatch_attention_layer(
396        &mut self,
397        index: &crate::gguf_sharder::GgufTensorIndex,
398        layer: u32,
399        token_idx: u32,
400        hidden: &[f32],
401        emb_dim: usize,
402        tensors: &crate::gguf_sharder::LayerTensors,
403        scratch_a: &mut [f32],
404        scratch_b: &mut [f32],
405    ) -> Option<usize> {
406        if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
407            return None; // Gated attention executes only through its admitted owner.
408        }
409        if Self::is_qwen35_gated_attention_layer(index, tensors) {
410            return self.dispatch_qwen35_gated_attention_layer(
411                index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
412            );
413        }
414        etalon_inference::ordinary::attention::AttentionEngine { host: self }
415            .dispatch_attention_layer(
416                index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
417            )
418    }
419
420    /// Q+attn, output projection, and FFN for one token (K/V already in arena).
421    pub(crate) fn dispatch_attention_q_ffn_token(
422        &mut self,
423        index: &crate::gguf_sharder::GgufTensorIndex,
424        layer: u32,
425        token_idx: u32,
426        hidden: &mut [f32],
427        emb_dim: usize,
428        tensors: &crate::gguf_sharder::LayerTensors,
429        scratch_a: &mut [f32],
430        scratch_b: &mut [f32],
431    ) -> bool {
432        if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
433            return false;
434        }
435        etalon_inference::ordinary::attention::AttentionEngine { host: self }
436            .dispatch_attention_q_ffn_token(
437                index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
438            )
439    }
440
441    /// CUDA residual attention: prefer **P4 device SDPA/KV** (one D2H); else sticky
442    /// Q/K/V GEMV → host RoPE/KV/SDPA → CUDA O-proj.
443    /// Returns true when `scratch_a[..emb_dim]` holds the attention residual delta.
444    #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
445    pub(super) fn try_cuda_soa_attention_layer(
446        &self,
447        hidden_input: &[f32],
448        n_embd: usize,
449        q_dim: usize,
450        kv_dim: usize,
451        n_head: usize,
452        n_kv: usize,
453        head_dim: usize,
454        layout: &KvCacheLayout,
455        layer: u32,
456        token_idx: u32,
457        h: &crate::gguf_sharder::GgufHyperparams,
458        q_raw: &[u8],
459        k_raw: &[u8],
460        v_raw: &[u8],
461        out_info: Option<&GgufTensorInfo>,
462        mmap: &[u8],
463        tensor_data_start: u64,
464        scratch_a: &mut [f32],
465        scratch_b: &mut [f32],
466    ) -> bool {
467        let layout = etalon_inference::ordinary::cuda_attention::CudaAttentionLayout {
468            max_context: layout.max_context,
469            n_layer: layout.n_layer,
470            n_kv_head: layout.n_kv_head,
471            head_dim: layout.head_dim,
472            slot_kv_elems: layout.slot_kv_elems,
473            layer_stride: layout.layer_stride,
474            total_f32_elems: layout.total_f32_elems,
475            int8: layout.int8,
476            dict_k: layout.dict_k,
477        };
478        etalon_inference::ordinary::cuda_attention::try_cuda_soa_attention_layer(
479            self,
480            hidden_input,
481            n_embd,
482            q_dim,
483            kv_dim,
484            n_head,
485            n_kv,
486            head_dim,
487            &layout,
488            layer,
489            token_idx,
490            &(*h).into(),
491            q_raw,
492            k_raw,
493            v_raw,
494            out_info,
495            mmap,
496            tensor_data_start,
497            scratch_a,
498            scratch_b,
499        )
500    }
501
502    /// Certified Naga-validated WGSL Forge causal attention device dispatch during decode.
503    /// Native + `wgsl-forge` only — wasm profiles do not compile the Forge module.
504    #[cfg(all(not(target_arch = "wasm32"), feature = "wgsl-forge"))]
505    pub fn dispatch_forge_causal_attention(
506        &mut self,
507        query: &[f32],
508        keys: &[f32],
509        values: &[f32],
510        scale: f32,
511        shape: crate::wgsl_forge::AttentionShape,
512        out: &mut [f32],
513    ) -> Result<(), crate::wgsl_forge::ForgeError> {
514        let schedule = crate::wgsl_forge::AttentionSchedule::a2000_decode();
515        schedule.validate(shape)?;
516        let q_len = (shape.head_count * shape.head_dim) as usize;
517        let kv_len = (shape.head_count * shape.context_tokens * shape.head_dim) as usize;
518        if query.len() < q_len || keys.len() < kv_len || values.len() < kv_len || out.len() < q_len
519        {
520            return Err(crate::wgsl_forge::ForgeError::InvalidKernel(
521                "dispatch_forge_causal_attention buffers smaller than required shape".into(),
522            ));
523        }
524
525        let shader = crate::wgsl_forge::synthesize_causal_attention(shape, schedule)?;
526        let params = crate::wgsl_forge::AttentionShaderParams {
527            context_tokens: shape.context_tokens,
528            head_dim: shape.head_dim,
529            head_count: shape.head_count,
530            _pad0: 0,
531            scale,
532            _pad1: 0,
533            _pad2: 0,
534            _pad3: 0,
535        };
536        if etalon_inference::gpu_attention::forge::dispatch_forge_attention(
537            self,
538            &shader.source,
539            bytemuck::bytes_of(&params),
540            q_len,
541            kv_len,
542            shape.head_count,
543            query,
544            keys,
545            values,
546            out,
547        ) {
548            return Ok(());
549        }
550
551        crate::wgsl_forge::causal_attention_oracle(shape, query, keys, values, scale, out)
552    }
553}