1#![allow(unused_variables, unused_mut)]
2use super::*;
3use etalon_inference::ordinary::attention::AttentionProvider;
4use etalon_model::inference::ModelHyperparams;
5impl AttentionProvider for QTensorEngine {
6 type Index = crate::gguf_sharder::GgufTensorIndex;
7 type Layout = KvCacheLayout;
8 fn kv_layout(&self) -> Option<Self::Layout> {
9 self.kv_layout
10 }
11 fn hyperparams(index: &Self::Index) -> ModelHyperparams {
12 index.hyperparams.into()
13 }
14 fn tensor_data_start(index: &Self::Index) -> u64 {
15 index.tensor_data_start
16 }
17 fn model_bytes(&self) -> Option<&[u8]> {
18 QTensorEngine::model_bytes(self)
19 }
20 fn debug_decode(&self) -> bool {
21 super::execution_env::var("QUALIA_LLM_DEBUG_DECODE").is_ok()
22 }
23 fn prefer_cuda(&self) -> bool {
24 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
25 {
26 return crate::prefer_tensor_core_gemm();
27 }
28 #[cfg(not(all(not(target_arch = "wasm32"), feature = "cuda")))]
29 {
30 false
31 }
32 }
33 fn attention_preproject(&self) -> bool {
34 cfg!(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))
35 && crate::llm_bench::attention_preproject_enabled()
36 }
37 fn attention_o_fuse(&self) -> bool {
38 cfg!(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))
39 && crate::llm_bench::attention_o_fuse_enabled()
40 }
41 fn dispatch_gemm_into(
42 &self,
43 index: &Self::Index,
44 info: &GgufTensorInfo,
45 input: &[f32],
46 out: &mut [f32],
47 n_in: usize,
48 n_out: usize,
49 ) -> bool {
50 QTensorEngine::dispatch_gemm_into(self, index, info, input, out, n_in, n_out)
51 }
52 fn dispatch_ffn_block_pre_norm(
53 &mut self,
54 index: &Self::Index,
55 hidden: &mut [f32],
56 emb_dim: usize,
57 tensors: &crate::gguf_sharder::LayerTensors,
58 scratch_a: &mut [f32],
59 scratch_b: &mut [f32],
60 ) -> bool {
61 QTensorEngine::dispatch_ffn_block_pre_norm(
62 self, index, hidden, emb_dim, tensors, scratch_a, scratch_b,
63 )
64 }
65 fn dispatch_attention_pass(
66 &self,
67 hidden: &[f32],
68 n_embd: usize,
69 num_tokens_in_batch: u32,
70 batch_start_token_idx: u32,
71 layout: &Self::Layout,
72 layer: u32,
73 token_idx: u32,
74 h: &ModelHyperparams,
75 info: &GgufTensorInfo,
76 raw_weights: &[u8],
77 proj_kind: u32,
78 n_workgroups: u32,
79 norm_weight: Option<&[f32]>,
80 readback_out: Option<&mut [f32]>,
81 ) -> bool {
82 QTensorEngine::dispatch_attention_pass(
83 self,
84 hidden,
85 n_embd,
86 num_tokens_in_batch,
87 batch_start_token_idx,
88 layout,
89 layer,
90 token_idx,
91 &(*h).into(),
92 info,
93 raw_weights,
94 proj_kind,
95 n_workgroups,
96 norm_weight,
97 readback_out,
98 )
99 }
100 fn try_cuda_soa_attention_layer(
101 &self,
102 hidden_input: &[f32],
103 n_embd: usize,
104 q_dim: usize,
105 kv_dim: usize,
106 n_head: usize,
107 n_kv: usize,
108 head_dim: usize,
109 layout: &Self::Layout,
110 layer: u32,
111 token_idx: u32,
112 h: &ModelHyperparams,
113 q_raw: &[u8],
114 k_raw: &[u8],
115 v_raw: &[u8],
116 out_info: Option<&GgufTensorInfo>,
117 mmap: &[u8],
118 tensor_data_start: u64,
119 scratch_a: &mut [f32],
120 scratch_b: &mut [f32],
121 ) -> bool {
122 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
123 {
124 return QTensorEngine::try_cuda_soa_attention_layer(
125 self,
126 hidden_input,
127 n_embd,
128 q_dim,
129 kv_dim,
130 n_head,
131 n_kv,
132 head_dim,
133 layout,
134 layer,
135 token_idx,
136 &(*h).into(),
137 q_raw,
138 k_raw,
139 v_raw,
140 out_info,
141 mmap,
142 tensor_data_start,
143 scratch_a,
144 scratch_b,
145 );
146 }
147 #[cfg(not(all(not(target_arch = "wasm32"), feature = "cuda")))]
148 {
149 false
150 }
151 }
152 fn dispatch_attention_kv_preproject_fused(
153 &self,
154 hidden: &[f32],
155 n_embd: usize,
156 layout: &Self::Layout,
157 layer: u32,
158 token_idx: u32,
159 h: &ModelHyperparams,
160 k_info: &GgufTensorInfo,
161 k_raw: &[u8],
162 v_info: &GgufTensorInfo,
163 v_raw: &[u8],
164 kv_dim: usize,
165 ) -> bool {
166 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
167 {
168 return QTensorEngine::dispatch_attention_kv_preproject_fused(
169 self,
170 hidden,
171 n_embd,
172 layout,
173 layer,
174 token_idx,
175 &(*h).into(),
176 k_info,
177 k_raw,
178 v_info,
179 v_raw,
180 kv_dim,
181 );
182 }
183 #[cfg(not(all(not(target_arch = "wasm32"), feature = "gpu-runtime")))]
184 {
185 false
186 }
187 }
188 fn dispatch_attention_q_o_fused(
189 &self,
190 hidden: &[f32],
191 n_embd: usize,
192 layout: &Self::Layout,
193 layer: u32,
194 token_idx: u32,
195 h: &ModelHyperparams,
196 q_info: &GgufTensorInfo,
197 q_raw: &[u8],
198 o_info: &GgufTensorInfo,
199 o_raw: &[u8],
200 out: &mut [f32],
201 o_in: usize,
202 o_out: usize,
203 ) -> bool {
204 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
205 {
206 return QTensorEngine::dispatch_attention_q_o_fused(
207 self,
208 hidden,
209 n_embd,
210 layout,
211 layer,
212 token_idx,
213 &(*h).into(),
214 q_info,
215 q_raw,
216 o_info,
217 o_raw,
218 out,
219 o_in,
220 o_out,
221 );
222 }
223 #[cfg(not(all(not(target_arch = "wasm32"), feature = "gpu-runtime")))]
224 {
225 false
226 }
227 }
228}
229
230#[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
231impl etalon_inference::ordinary::cuda_attention::CudaAttentionHost for QTensorEngine {
232 fn with_cpu_kv<R>(&self, run: impl FnOnce(Option<&mut [f32]>) -> R) -> R {
233 let kv = self.kv_cache_cpu.as_ref().map(|kv| {
234 unsafe { core::slice::from_raw_parts_mut(kv.as_ptr() as *mut f32, kv.len()) }
236 });
237 run(kv)
238 }
239 fn gemm(
240 &self,
241 raw: &[u8],
242 info: &GgufTensorInfo,
243 input: &[f32],
244 out: &mut [f32],
245 n_in: usize,
246 n_out: usize,
247 ) -> bool {
248 stack_gemm_quant(raw, info, input, out, n_in, n_out)
249 }
250 fn device_attempt(
251 &self,
252 hit: bool,
253 layer: u32,
254 token_idx: u32,
255 o_in: usize,
256 o_out: usize,
257 q_dim: usize,
258 n_embd: usize,
259 ) {
260 if hit {
261 static LOGGED: std::sync::atomic::AtomicBool =
262 std::sync::atomic::AtomicBool::new(false);
263 if !LOGGED.swap(true, std::sync::atomic::Ordering::Relaxed) {
264 log::info!(
265 "cuda_attn|device_sdpa|first_hit|layer={layer}|tok={token_idx}|kv_ready={}",
266 crate::device_kv_ready()
267 );
268 }
269 } else {
270 static MISSES: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
271 if MISSES.fetch_add(1, std::sync::atomic::Ordering::Relaxed) < 3 {
272 log::warn!("cuda_attn|device_sdpa|miss|layer={layer}|tok={token_idx}|kv_ready={}|o_in={o_in}|o_out={o_out}|q_dim={q_dim}|n_embd={n_embd}", crate::device_kv_ready());
273 }
274 }
275 }
276}
277
278#[cfg(all(not(target_arch = "wasm32"), feature = "wgsl-forge"))]
279impl etalon_inference::gpu_attention::forge::ForgeAttentionHost for QTensorEngine {
280 fn device(&self) -> &etalon::Device {
281 QTensorEngine::device(self)
282 }
283 fn submit(&self, command: etalon::CommandBuffer) {
284 self.submit_commands(Some(command));
285 }
286 fn poll_wait(&self) {
287 QTensorEngine::poll_wait(self);
288 }
289 fn wait_for_map(
290 &self,
291 receiver: futures_channel::oneshot::Receiver<Result<(), etalon::BufferAsyncError>>,
292 ) -> bool {
293 if let Ok(handle) = tokio::runtime::Handle::try_current() {
294 return handle
295 .block_on(receiver)
296 .ok()
297 .map(|r| r.is_ok())
298 .unwrap_or(false);
299 }
300 false
301 }
302}