1#![allow(dead_code)]
4use super::*;
5
6impl QTensorEngine {
7 pub fn dispatch_output_logits_into(
9 &self,
10 index: &crate::gguf_sharder::GgufTensorIndex,
11 hidden: &[f32],
12 emb_dim: usize,
13 logits_out: &mut [f32],
14 ) -> usize {
15 #[cfg(not(target_arch = "wasm32"))]
16 if let Some(ref pkg) = self.ftw_package {
17 if let (Some(packed), Some(scales), Some(globals)) = (
18 pkg.fetch_tensor_bytes("lm_head.weight"),
19 pkg.fetch_tensor_bytes("lm_head.weight_scale"),
20 pkg.fetch_tensor_bytes("lm_head.weight_global"),
21 ) {
22 let vocab_size = globals.len() / 2;
23 let fill = vocab_size.min(logits_out.len());
24 let row_bytes = emb_dim / 2;
25 let scale_bytes = emb_dim / 16;
26 if emb_dim == 0 || hidden.len() < emb_dim || fill == 0 {
27 return 0;
28 }
29 if packed.len() < vocab_size * row_bytes || scales.len() < vocab_size * scale_bytes
30 {
31 return 0;
32 }
33 for token_id in 0..fill {
34 let g_val = half::f16::from_le_bytes([
35 globals[token_id * 2],
36 globals[token_id * 2 + 1],
37 ])
38 .to_f32();
39 let p_row = &packed[token_id * row_bytes..(token_id + 1) * row_bytes];
40 let s_row = &scales[token_id * scale_bytes..(token_id + 1) * scale_bytes];
41 logits_out[token_id] = etalon_inference::ordinary::output::nvfp4_output_row(
42 p_row, s_row, g_val, hidden, emb_dim,
43 );
44 }
45 return fill;
46 }
47 }
48
49 let Some(info) = index.logits_projection_info() else {
50 let n = emb_dim.min(logits_out.len());
51 logits_out[..n].copy_from_slice(&hidden[..n]);
52 return n;
53 };
54 let (n_in, vocab_size) = Self::matmul_dims(info);
55 let fill = vocab_size.min(logits_out.len());
56 if n_in > emb_dim || fill == 0 {
57 let n = emb_dim.min(logits_out.len());
58 logits_out[..n].copy_from_slice(&hidden[..n]);
59 return n;
60 }
61 let mmap = match self.model_bytes() {
62 Some(m) => m,
63 None => {
64 let n = emb_dim.min(logits_out.len());
65 logits_out[..n].copy_from_slice(&hidden[..n]);
66 return n;
67 }
68 };
69 let mut written = 0usize;
70 let n_chunks = vocab_size.div_ceil(VOCAB_CHUNK_ROWS);
71 for chunk_idx in 0..n_chunks {
72 if written >= fill {
73 break;
74 }
75 let row_start = chunk_idx * VOCAB_CHUNK_ROWS;
76 let chunk_rows = VOCAB_CHUNK_ROWS.min(vocab_size - row_start);
77 let raw = match crate::ggml_quants::fetch_tensor_row_range_bytes(
78 mmap,
79 index.tensor_data_start,
80 info,
81 row_start,
82 chunk_rows,
83 ) {
84 Ok(s) => s,
85 Err(_) => break,
86 };
87 let out_rows = chunk_rows.min(fill - written);
88 if !self.dispatch_gemm_raw_into(
89 info,
90 raw,
91 &hidden[..n_in],
92 &mut logits_out[written..written + out_rows],
93 n_in,
94 out_rows,
95 ) {
96 break;
97 }
98 written += out_rows;
99 }
100 if written > 0 {
101 written
102 } else {
103 let n = emb_dim.min(logits_out.len());
104 logits_out[..n].copy_from_slice(&hidden[..n]);
105 n
106 }
107 }
108
109 pub fn decode_lexicon_bound(&self, _logits: &[f32], valid_lexicon_ids: &[u64]) -> u64 {
111 if valid_lexicon_ids.is_empty() {
112 0
113 } else {
114 valid_lexicon_ids[0]
115 }
116 }
117
118 #[cfg(target_arch = "wasm32")]
119 pub(crate) async fn dispatch_gemm_raw_into_async(
121 &self,
122 info: &GgufTensorInfo,
123 raw: &[u8],
124 input: &[f32],
125 out: &mut [f32],
126 n_in: usize,
127 n_out: usize,
128 ) -> bool {
129 etalon_inference::browser_projection::BrowserProjectionEngine { host: self }
130 .dispatch_gemm_raw_into_async(info, raw, input, out, n_in, n_out)
131 .await
132 }
133
134 #[cfg(target_arch = "wasm32")]
139 pub(crate) async fn dispatch_gemm_resident_chunk_async(
140 &self,
141 weight_ggml_type: u32,
142 weight_row_elems: u32,
143 weight: etalon::BindingResource<'_>,
144 weight_byte_len: u32,
145 input: &[f32],
146 out: &mut [f32],
147 n_in: usize,
148 n_out: usize,
149 ) -> bool {
150 etalon_inference::browser_projection::BrowserProjectionEngine { host: self }
151 .dispatch_gemm_resident_chunk_async(
152 weight_ggml_type,
153 weight_row_elems,
154 weight,
155 weight_byte_len,
156 input,
157 out,
158 n_in,
159 n_out,
160 )
161 .await
162 }
163 #[cfg(target_arch = "wasm32")]
164 pub async fn dispatch_gemm_into_async(
166 &self,
167 index: &crate::gguf_sharder::GgufTensorIndex,
168 info: &GgufTensorInfo,
169 input: &[f32],
170 out: &mut [f32],
171 n_in: usize,
172 n_out: usize,
173 ) -> bool {
174 etalon_inference::browser_projection::BrowserProjectionEngine { host: self }
175 .dispatch_gemm_into_async(index, info, input, out, n_in, n_out)
176 .await
177 }
178 #[cfg(target_arch = "wasm32")]
179 pub(crate) async fn dispatch_attention_layer_async(
181 &mut self,
182 index: &crate::gguf_sharder::GgufTensorIndex,
183 layer: u32,
184 token_idx: u32,
185 hidden: &[f32],
186 emb_dim: usize,
187 tensors: &crate::gguf_sharder::LayerTensors,
188 scratch_a: &mut [f32],
189 scratch_b: &mut [f32],
190 ) -> Option<usize> {
191 etalon_inference::ordinary::async_attention::dispatch_attention_layer_async(
192 self, index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
193 )
194 .await
195 }
196
197 #[cfg(target_arch = "wasm32")]
198 pub(crate) async fn dispatch_attention_pass_async(
200 &self,
201 hidden: &[f32],
202 n_embd: usize,
203 num_tokens_in_batch: u32,
204 batch_start_token_idx: u32,
205 layout: &KvCacheLayout,
206 layer: u32,
207 token_idx: u32,
208 h: &crate::gguf_sharder::GgufHyperparams,
209 info: &GgufTensorInfo,
210 raw_weights: &[u8],
211 proj_kind: u32,
212 n_workgroups: u32,
213 readback_out: Option<&mut [f32]>,
214 ) -> bool {
215 if !ggml_gpu_quant_supported(info.ggml_type) {
216 wlog(&format!(
217 "[attn_pass_async] GUARD unsupported quant kind={proj_kind}"
218 ));
219 return false;
220 }
221 if !ggml_gpu_attention_shader_supported(info.ggml_type) {
222 return self.cpu_attention_pass(
223 hidden,
224 n_embd,
225 num_tokens_in_batch,
226 batch_start_token_idx,
227 layout,
228 layer,
229 h,
230 info,
231 raw_weights,
232 proj_kind,
233 None,
234 readback_out,
235 );
236 }
237 let batch = num_tokens_in_batch.max(1) as usize;
238 let hidden_elems = n_embd.checked_mul(batch).unwrap_or(0);
239 if hidden_elems > hidden.len()
240 || hidden_elems > self.gemm_max_input_floats
241 || raw_weights.len() > self.max_tensor_bytes
242 || self.gemm_input_buf.is_none()
243 || self.kv_cache_gpu.is_none()
244 || self.attention_params_buf.is_none()
245 || self.attention_mask_buf.is_none()
246 {
247 wlog(&format!(
248 "[attn_pass_async] GUARD buffers kind={proj_kind} hidden_elems={hidden_elems} hidden={} gemm_in={} raw_w={} max_w={}",
249 hidden.len(),
250 self.gemm_max_input_floats,
251 raw_weights.len(),
252 self.max_tensor_bytes,
253 ));
254 return false;
255 }
256
257 let (mask_words, mask_active, mask_word_count) =
258 Self::attention_kv_mask_for_dispatch(layout, token_idx, proj_kind);
259 let params = Self::attention_gpu_params(
260 h,
261 layout,
262 layer,
263 token_idx,
264 info,
265 raw_weights.len(),
266 proj_kind,
267 num_tokens_in_batch.max(1),
268 batch_start_token_idx,
269 mask_active,
270 mask_word_count,
271 0,
272 );
273 let input_buf = self.gemm_input_buf.as_ref().unwrap();
274 let weight_buf = self.gemm_weight_buf.as_ref().unwrap();
275 let output_buf = self.gemm_output_buf.as_ref().unwrap();
276 let params_buf = self.attention_params_buf.as_ref().unwrap();
277 let mask_buf = self.attention_mask_buf.as_ref().unwrap();
278 let kv_buf = self.kv_cache_gpu.as_ref().unwrap();
279 let staging = self.gemm_output_staging.as_ref().unwrap();
280
281 self.gpu_queue()
282 .write_buffer(input_buf, 0, bytemuck::cast_slice(&hidden[..hidden_elems]));
283 self.write_weight_words(raw_weights, self.max_tensor_bytes);
284 self.gpu_queue()
285 .write_buffer(params_buf, 0, bytemuck::bytes_of(¶ms));
286 self.gpu_queue()
287 .write_buffer(mask_buf, 0, bytemuck::cast_slice(&mask_words));
288
289 let resources = etalon_inference::gpu_attention::PassResources {
290 input: input_buf,
291 weight: weight_buf,
292 output: output_buf,
293 params: params_buf,
294 mask: mask_buf,
295 kv: kv_buf,
296 staging,
297 bind_layout: &self.attention_pipeline.get_bind_group_layout(0),
298 pipeline: &self.attention_pipeline,
299 };
300 let readback_elems = readback_out.as_ref().map(|o| o.len()).unwrap_or(0);
301 let mut encoder = match etalon_inference::gpu_attention::encode_pass(
302 self.gpu_device(),
303 self.gpu_queue(),
304 &resources,
305 ¶ms,
306 &mask_words,
307 n_workgroups,
308 readback_elems,
309 crate::llm_gpu_profiler::pass_writes_both(),
310 ) {
311 Some(encoder) => encoder,
312 None => return false,
313 };
314 crate::llm_gpu_profiler::resolve(&mut encoder);
315 self.submit_commands(Some(encoder.finish()));
316 crate::llm_gpu_profiler::accumulate(crate::llm_gpu_profiler::Phase::Attention);
317
318 if readback_elems == 0 {
319 return true;
320 }
321
322 let out_bytes = (readback_elems * 4) as etalon::BufferAddress;
323 let slice = staging.slice(..out_bytes);
324 if await_wgpu_map(slice).await {
325 let data = slice
326 .get_mapped_range()
327 .expect("wgpu buffer map_range failed");
328 let floats: &[f32] = bytemuck::cast_slice(&data);
329 if let Some(out) = readback_out {
330 out[..readback_elems].copy_from_slice(&floats[..readback_elems]);
331 }
332 drop(data);
333 staging.unmap();
334 return true;
335 }
336 let _ = staging.unmap();
337 false
338 }
339 #[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
340 pub(crate) async fn dispatch_attention_q_ffn_token_async(
342 &mut self,
343 index: &crate::gguf_sharder::GgufTensorIndex,
344 layer: u32,
345 token_idx: u32,
346 hidden: &mut [f32],
347 emb_dim: usize,
348 tensors: &crate::gguf_sharder::LayerTensors,
349 scratch_a: &mut [f32],
350 scratch_b: &mut [f32],
351 ) -> bool {
352 etalon_inference::ordinary::async_attention_token::dispatch_attention_q_ffn_token_async(
353 self, index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
354 )
355 .await
356 }
357}