Skip to main content

qualia_core_db/gguf_bridge/
attention_adapter.rs

1#![allow(unused_variables, unused_mut)]
2use super::*;
3use etalon_inference::ordinary::attention::AttentionProvider;
4use etalon_model::inference::ModelHyperparams;
5impl AttentionProvider for QTensorEngine {
6    type Index = crate::gguf_sharder::GgufTensorIndex;
7    type Layout = KvCacheLayout;
8    fn kv_layout(&self) -> Option<Self::Layout> {
9        self.kv_layout
10    }
11    fn hyperparams(index: &Self::Index) -> ModelHyperparams {
12        index.hyperparams.into()
13    }
14    fn tensor_data_start(index: &Self::Index) -> u64 {
15        index.tensor_data_start
16    }
17    fn model_bytes(&self) -> Option<&[u8]> {
18        QTensorEngine::model_bytes(self)
19    }
20    fn debug_decode(&self) -> bool {
21        super::execution_env::var("QUALIA_LLM_DEBUG_DECODE").is_ok()
22    }
23    fn prefer_cuda(&self) -> bool {
24        #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
25        {
26            return crate::prefer_tensor_core_gemm();
27        }
28        #[cfg(not(all(not(target_arch = "wasm32"), feature = "cuda")))]
29        {
30            false
31        }
32    }
33    fn attention_preproject(&self) -> bool {
34        cfg!(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))
35            && crate::llm_bench::attention_preproject_enabled()
36    }
37    fn attention_o_fuse(&self) -> bool {
38        cfg!(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))
39            && crate::llm_bench::attention_o_fuse_enabled()
40    }
41    fn dispatch_gemm_into(
42        &self,
43        index: &Self::Index,
44        info: &GgufTensorInfo,
45        input: &[f32],
46        out: &mut [f32],
47        n_in: usize,
48        n_out: usize,
49    ) -> bool {
50        QTensorEngine::dispatch_gemm_into(self, index, info, input, out, n_in, n_out)
51    }
52    fn dispatch_ffn_block_pre_norm(
53        &mut self,
54        index: &Self::Index,
55        hidden: &mut [f32],
56        emb_dim: usize,
57        tensors: &crate::gguf_sharder::LayerTensors,
58        scratch_a: &mut [f32],
59        scratch_b: &mut [f32],
60    ) -> bool {
61        QTensorEngine::dispatch_ffn_block_pre_norm(
62            self, index, hidden, emb_dim, tensors, scratch_a, scratch_b,
63        )
64    }
65    fn dispatch_attention_pass(
66        &self,
67        hidden: &[f32],
68        n_embd: usize,
69        num_tokens_in_batch: u32,
70        batch_start_token_idx: u32,
71        layout: &Self::Layout,
72        layer: u32,
73        token_idx: u32,
74        h: &ModelHyperparams,
75        info: &GgufTensorInfo,
76        raw_weights: &[u8],
77        proj_kind: u32,
78        n_workgroups: u32,
79        norm_weight: Option<&[f32]>,
80        readback_out: Option<&mut [f32]>,
81    ) -> bool {
82        QTensorEngine::dispatch_attention_pass(
83            self,
84            hidden,
85            n_embd,
86            num_tokens_in_batch,
87            batch_start_token_idx,
88            layout,
89            layer,
90            token_idx,
91            &(*h).into(),
92            info,
93            raw_weights,
94            proj_kind,
95            n_workgroups,
96            norm_weight,
97            readback_out,
98        )
99    }
100    fn try_cuda_soa_attention_layer(
101        &self,
102        hidden_input: &[f32],
103        n_embd: usize,
104        q_dim: usize,
105        kv_dim: usize,
106        n_head: usize,
107        n_kv: usize,
108        head_dim: usize,
109        layout: &Self::Layout,
110        layer: u32,
111        token_idx: u32,
112        h: &ModelHyperparams,
113        q_raw: &[u8],
114        k_raw: &[u8],
115        v_raw: &[u8],
116        out_info: Option<&GgufTensorInfo>,
117        mmap: &[u8],
118        tensor_data_start: u64,
119        scratch_a: &mut [f32],
120        scratch_b: &mut [f32],
121    ) -> bool {
122        #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
123        {
124            return QTensorEngine::try_cuda_soa_attention_layer(
125                self,
126                hidden_input,
127                n_embd,
128                q_dim,
129                kv_dim,
130                n_head,
131                n_kv,
132                head_dim,
133                layout,
134                layer,
135                token_idx,
136                &(*h).into(),
137                q_raw,
138                k_raw,
139                v_raw,
140                out_info,
141                mmap,
142                tensor_data_start,
143                scratch_a,
144                scratch_b,
145            );
146        }
147        #[cfg(not(all(not(target_arch = "wasm32"), feature = "cuda")))]
148        {
149            false
150        }
151    }
152    fn dispatch_attention_kv_preproject_fused(
153        &self,
154        hidden: &[f32],
155        n_embd: usize,
156        layout: &Self::Layout,
157        layer: u32,
158        token_idx: u32,
159        h: &ModelHyperparams,
160        k_info: &GgufTensorInfo,
161        k_raw: &[u8],
162        v_info: &GgufTensorInfo,
163        v_raw: &[u8],
164        kv_dim: usize,
165    ) -> bool {
166        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
167        {
168            return QTensorEngine::dispatch_attention_kv_preproject_fused(
169                self,
170                hidden,
171                n_embd,
172                layout,
173                layer,
174                token_idx,
175                &(*h).into(),
176                k_info,
177                k_raw,
178                v_info,
179                v_raw,
180                kv_dim,
181            );
182        }
183        #[cfg(not(all(not(target_arch = "wasm32"), feature = "gpu-runtime")))]
184        {
185            false
186        }
187    }
188    fn dispatch_attention_q_o_fused(
189        &self,
190        hidden: &[f32],
191        n_embd: usize,
192        layout: &Self::Layout,
193        layer: u32,
194        token_idx: u32,
195        h: &ModelHyperparams,
196        q_info: &GgufTensorInfo,
197        q_raw: &[u8],
198        o_info: &GgufTensorInfo,
199        o_raw: &[u8],
200        out: &mut [f32],
201        o_in: usize,
202        o_out: usize,
203    ) -> bool {
204        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
205        {
206            return QTensorEngine::dispatch_attention_q_o_fused(
207                self,
208                hidden,
209                n_embd,
210                layout,
211                layer,
212                token_idx,
213                &(*h).into(),
214                q_info,
215                q_raw,
216                o_info,
217                o_raw,
218                out,
219                o_in,
220                o_out,
221            );
222        }
223        #[cfg(not(all(not(target_arch = "wasm32"), feature = "gpu-runtime")))]
224        {
225            false
226        }
227    }
228}
229
230#[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
231impl etalon_inference::ordinary::cuda_attention::CudaAttentionHost for QTensorEngine {
232    fn with_cpu_kv<R>(&self, run: impl FnOnce(Option<&mut [f32]>) -> R) -> R {
233        let kv = self.kv_cache_cpu.as_ref().map(|kv| {
234            // SAFETY: the caller retains exclusive decode-thread custody.
235            unsafe { core::slice::from_raw_parts_mut(kv.as_ptr() as *mut f32, kv.len()) }
236        });
237        run(kv)
238    }
239    fn gemm(
240        &self,
241        raw: &[u8],
242        info: &GgufTensorInfo,
243        input: &[f32],
244        out: &mut [f32],
245        n_in: usize,
246        n_out: usize,
247    ) -> bool {
248        stack_gemm_quant(raw, info, input, out, n_in, n_out)
249    }
250    fn device_attempt(
251        &self,
252        hit: bool,
253        layer: u32,
254        token_idx: u32,
255        o_in: usize,
256        o_out: usize,
257        q_dim: usize,
258        n_embd: usize,
259    ) {
260        if hit {
261            static LOGGED: std::sync::atomic::AtomicBool =
262                std::sync::atomic::AtomicBool::new(false);
263            if !LOGGED.swap(true, std::sync::atomic::Ordering::Relaxed) {
264                log::info!(
265                    "cuda_attn|device_sdpa|first_hit|layer={layer}|tok={token_idx}|kv_ready={}",
266                    crate::device_kv_ready()
267                );
268            }
269        } else {
270            static MISSES: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
271            if MISSES.fetch_add(1, std::sync::atomic::Ordering::Relaxed) < 3 {
272                log::warn!("cuda_attn|device_sdpa|miss|layer={layer}|tok={token_idx}|kv_ready={}|o_in={o_in}|o_out={o_out}|q_dim={q_dim}|n_embd={n_embd}", crate::device_kv_ready());
273            }
274        }
275    }
276}
277
278#[cfg(all(not(target_arch = "wasm32"), feature = "wgsl-forge"))]
279impl etalon_inference::gpu_attention::forge::ForgeAttentionHost for QTensorEngine {
280    fn device(&self) -> &etalon::Device {
281        QTensorEngine::device(self)
282    }
283    fn submit(&self, command: etalon::CommandBuffer) {
284        self.submit_commands(Some(command));
285    }
286    fn poll_wait(&self) {
287        QTensorEngine::poll_wait(self);
288    }
289    fn wait_for_map(
290        &self,
291        receiver: futures_channel::oneshot::Receiver<Result<(), etalon::BufferAsyncError>>,
292    ) -> bool {
293        if let Ok(handle) = tokio::runtime::Handle::try_current() {
294            return handle
295                .block_on(receiver)
296                .ok()
297                .map(|r| r.is_ok())
298                .unwrap_or(false);
299        }
300        false
301    }
302}