1use super::*;
5
6#[derive(Clone, Copy, Debug, PartialEq, Eq)]
10pub enum PrefillDispatchFailure {
11 EmptyInput,
12 Layer {
13 layer: u32,
14 stage: PrefillLayerStage,
15 },
16}
17
18#[derive(Clone, Copy, Debug, PartialEq, Eq)]
19pub enum PrefillLayerStage {
20 EmptyBatch,
21 MissingKvLayout,
22 MissingAttentionK,
23 MissingAttentionV,
24 MissingAttentionQ,
25 BatchHiddenTooSmall,
26 MissingModelMap,
27 FetchAttentionK,
28 FetchAttentionV,
29 DispatchAttentionK,
30 DispatchAttentionV,
31 DispatchQueryFfn,
32}
33
34impl QTensorEngine {
35 pub(crate) fn dispatch_prefill_layer_batch(
37 &mut self,
38 index: &crate::gguf_sharder::GgufTensorIndex,
39 layer: u32,
40 batch_hidden: &mut [f32],
41 emb_dim: usize,
42 n_tokens: u32,
43 batch_start_token_idx: u32,
44 scratch_a: &mut [f32],
45 scratch_b: &mut [f32],
46 ) -> Result<(), PrefillLayerStage> {
47 if n_tokens == 0 {
48 wlog("[prefill_layer] FAILED n_tokens=0");
49 return Err(PrefillLayerStage::EmptyBatch);
50 }
51 let layout = match self.kv_layout {
52 Some(l) => l,
53 None => {
54 wlog("[prefill_layer] FAILED kv_layout is None");
55 return Err(PrefillLayerStage::MissingKvLayout);
56 }
57 };
58 let tensors = index.get_layer_tensors(layer);
59 if index.hyperparams.architecture == etalon_model::inference::ARCH_GEMMA4
60 || Self::is_qwen35_gated_attention_layer(index, &tensors)
61 {
62 let batch_elems = emb_dim * n_tokens as usize;
63 if batch_elems > batch_hidden.len() {
64 return Err(PrefillLayerStage::BatchHiddenTooSmall);
65 }
66 for t in 0..n_tokens {
67 let off = t as usize * emb_dim;
68 let t_hidden = &mut batch_hidden[off..off + emb_dim];
69 if !self.dispatch_transformer_layer(
70 index,
71 layer,
72 batch_start_token_idx + t,
73 t_hidden,
74 emb_dim,
75 scratch_a,
76 scratch_b,
77 ) {
78 return Err(PrefillLayerStage::DispatchQueryFfn);
79 }
80 }
81 return Ok(());
82 }
83 if tensors.is_hybrid_ssm_layer() {
84 let batch_elems = emb_dim * n_tokens as usize;
85 if batch_elems > batch_hidden.len() {
86 return Err(PrefillLayerStage::BatchHiddenTooSmall);
87 }
88 for t in 0..n_tokens {
89 let off = t as usize * emb_dim;
90 let t_hidden = &mut batch_hidden[off..off + emb_dim];
91 if !self.dispatch_hybrid_ssm_layer(
92 index, layer, t_hidden, emb_dim, &tensors, scratch_a, scratch_b,
93 ) {
94 return Err(PrefillLayerStage::DispatchQueryFfn);
95 }
96 if !self.dispatch_ffn_block_pre_norm(
97 index, t_hidden, emb_dim, &tensors, scratch_a, scratch_b,
98 ) {
99 return Err(PrefillLayerStage::DispatchQueryFfn);
100 }
101 }
102 return Ok(());
103 }
104
105 let k_info = tensors
106 .attn_k
107 .as_ref()
108 .ok_or(PrefillLayerStage::MissingAttentionK)?;
109 let v_info = tensors
110 .attn_v
111 .as_ref()
112 .ok_or(PrefillLayerStage::MissingAttentionV)?;
113 if tensors.attn_q.is_none() {
114 return Err(PrefillLayerStage::MissingAttentionQ);
115 }
116 let h = index.hyperparams;
117 let n_kv = h.effective_n_kv_head();
118 let n_embd = h.n_embd as usize;
119 let batch_elems = n_embd * n_tokens as usize;
120 if batch_elems > batch_hidden.len() {
121 wlog(&format!(
122 "[prefill_layer] FAILED batch_elems OOB elems={batch_elems} hidden={}",
123 batch_hidden.len()
124 ));
125 return Err(PrefillLayerStage::BatchHiddenTooSmall);
126 }
127 let mmap = match self.model_bytes() {
128 Some(m) => m,
129 None => {
130 wlog("[prefill_layer] FAILED gguf_mmap is None");
131 return Err(PrefillLayerStage::MissingModelMap);
132 }
133 };
134 let k_raw =
135 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info) {
136 Ok(s) => s,
137 Err(e) => {
138 wlog(&format!("[prefill_layer] FAILED fetch attn_k bytes: {e:?}"));
139 return Err(PrefillLayerStage::FetchAttentionK);
140 }
141 };
142 let v_raw =
143 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info) {
144 Ok(s) => s,
145 Err(e) => {
146 wlog(&format!("[prefill_layer] FAILED fetch attn_v bytes: {e:?}"));
147 return Err(PrefillLayerStage::FetchAttentionV);
148 }
149 };
150 let n_kv_wg = n_tokens.saturating_mul(n_kv);
151 let mut norm_w_attn = [0f32; MAX_HIDDEN_DIM];
155 let norm_weight_attn: Option<&[f32]> = tensors.attn_norm.as_ref().and_then(|info| {
156 let n = dequant_norm_row_into(mmap, index.tensor_data_start, info, &mut norm_w_attn);
157 if n >= n_embd {
158 Some(&norm_w_attn[..n_embd])
159 } else {
160 None
161 }
162 });
163 if !self.dispatch_attention_pass(
164 &batch_hidden[..batch_elems],
165 n_embd,
166 n_tokens,
167 batch_start_token_idx,
168 &layout,
169 layer,
170 batch_start_token_idx,
171 &h,
172 k_info,
173 k_raw,
174 1,
175 n_kv_wg,
176 norm_weight_attn,
177 None,
178 ) {
179 wlog(&format!("[prefill_layer] K pass FAILED layer={layer}"));
180 return Err(PrefillLayerStage::DispatchAttentionK);
181 }
182 if !self.dispatch_attention_pass(
183 &batch_hidden[..batch_elems],
184 n_embd,
185 n_tokens,
186 batch_start_token_idx,
187 &layout,
188 layer,
189 batch_start_token_idx,
190 &h,
191 v_info,
192 v_raw,
193 2,
194 n_kv_wg,
195 norm_weight_attn,
196 None,
197 ) {
198 wlog(&format!("[prefill_layer] V pass FAILED layer={layer}"));
199 return Err(PrefillLayerStage::DispatchAttentionV);
200 }
201 for t in 0..n_tokens {
202 let abs = batch_start_token_idx + t;
203 let off = t as usize * emb_dim;
204 if !self.dispatch_attention_q_ffn_token(
205 index,
206 layer,
207 abs,
208 &mut batch_hidden[off..off + emb_dim],
209 emb_dim,
210 &tensors,
211 scratch_a,
212 scratch_b,
213 ) {
214 wlog(&format!(
215 "[prefill_layer] q_ffn FAILED layer={layer} t={t} abs={abs}"
216 ));
217 return Err(PrefillLayerStage::DispatchQueryFfn);
218 }
219 }
220 Ok(())
221 }
222
223 #[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
225 pub(crate) async fn dispatch_prefill_layer_batch_async(
226 &mut self,
227 index: &crate::gguf_sharder::GgufTensorIndex,
228 layer: u32,
229 batch_hidden: &mut [f32],
230 emb_dim: usize,
231 n_tokens: u32,
232 batch_start_token_idx: u32,
233 scratch_a: &mut [f32],
234 scratch_b: &mut [f32],
235 ) -> bool {
236 if n_tokens == 0 {
237 wlog("[prefill_layer] FAILED n_tokens=0");
238 return false;
239 }
240 let layout = match self.kv_layout {
241 Some(l) => l,
242 None => {
243 wlog("[prefill_layer] FAILED kv_layout is None");
244 return false;
245 }
246 };
247 let tensors = index.get_layer_tensors(layer);
248 let k_info = match tensors.attn_k.as_ref() {
249 Some(i) => i,
250 None => {
251 wlog(&format!(
252 "[prefill_layer] FAILED missing attn_k layer={layer}"
253 ));
254 return false;
255 }
256 };
257 let v_info = match tensors.attn_v.as_ref() {
258 Some(i) => i,
259 None => {
260 wlog(&format!(
261 "[prefill_layer] FAILED missing attn_v layer={layer}"
262 ));
263 return false;
264 }
265 };
266 if tensors.attn_q.is_none() {
267 wlog(&format!(
268 "[prefill_layer] FAILED missing attn_q layer={layer}"
269 ));
270 return false;
271 }
272 let h = index.hyperparams;
273 let n_kv = h.effective_n_kv_head();
274 let n_embd = h.n_embd as usize;
275 let batch_elems = n_embd * n_tokens as usize;
276 if batch_elems > batch_hidden.len() {
277 wlog(&format!(
278 "[prefill_layer] FAILED batch_elems OOB elems={batch_elems} hidden={}",
279 batch_hidden.len()
280 ));
281 return false;
282 }
283 let mmap = match self.model_bytes() {
284 Some(m) => m,
285 None => {
286 wlog("[prefill_layer] FAILED gguf_mmap is None");
287 return false;
288 }
289 };
290 let k_raw =
291 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info) {
292 Ok(s) => s,
293 Err(e) => {
294 wlog(&format!("[prefill_layer] FAILED fetch attn_k bytes: {e:?}"));
295 return false;
296 }
297 };
298 let v_raw =
299 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info) {
300 Ok(s) => s,
301 Err(e) => {
302 wlog(&format!("[prefill_layer] FAILED fetch attn_v bytes: {e:?}"));
303 return false;
304 }
305 };
306 let mut norm_w_attn = [0f32; MAX_HIDDEN_DIM];
307 let mut norm_scratch = [0f32; PREFILL_CHUNK_STACK_FLOATS];
308 let attn_input: &mut [f32] = if let Some(norm_info) = tensors.attn_norm.as_ref() {
309 let n =
310 dequant_norm_row_into(mmap, index.tensor_data_start, norm_info, &mut norm_w_attn);
311 if n >= n_embd {
312 for t in 0..n_tokens as usize {
313 let off = t * n_embd;
314 norm_scratch[off..off + n_embd]
315 .copy_from_slice(&batch_hidden[off..off + n_embd]);
316 rms_norm_inplace(
317 &mut norm_scratch[off..off + n_embd],
318 &norm_w_attn[..n_embd],
319 RMS_NORM_EPS,
320 );
321 }
322 &mut norm_scratch[..batch_elems]
323 } else {
324 batch_hidden
325 }
326 } else {
327 batch_hidden
328 };
329 let n_kv_wg = n_tokens.saturating_mul(n_kv);
330 if !self
331 .dispatch_attention_pass_async(
332 attn_input,
333 n_embd,
334 n_tokens,
335 batch_start_token_idx,
336 &layout,
337 layer,
338 batch_start_token_idx,
339 &h,
340 k_info,
341 k_raw,
342 1,
343 n_kv_wg,
344 None,
345 )
346 .await
347 {
348 wlog(&format!("[prefill_layer] K pass FAILED layer={layer}"));
349 return false;
350 }
351 if !self
352 .dispatch_attention_pass_async(
353 attn_input,
354 n_embd,
355 n_tokens,
356 batch_start_token_idx,
357 &layout,
358 layer,
359 batch_start_token_idx,
360 &h,
361 v_info,
362 v_raw,
363 2,
364 n_kv_wg,
365 None,
366 )
367 .await
368 {
369 wlog(&format!("[prefill_layer] V pass FAILED layer={layer}"));
370 return false;
371 }
372 for t in 0..n_tokens {
373 let abs = batch_start_token_idx + t;
374 let off = t as usize * emb_dim;
375 if !self
376 .dispatch_attention_q_ffn_token_async(
377 index,
378 layer,
379 abs,
380 &mut batch_hidden[off..off + emb_dim],
381 emb_dim,
382 &tensors,
383 scratch_a,
384 scratch_b,
385 )
386 .await
387 {
388 wlog(&format!(
389 "[prefill_layer] q_ffn FAILED layer={layer} t={t} abs={abs}"
390 ));
391 return false;
392 }
393 }
394 true
395 }
396
397 pub fn dispatch_prefill_chunk(
399 &mut self,
400 index: &crate::gguf_sharder::GgufTensorIndex,
401 batch_hidden: &mut [f32],
402 emb_dim: usize,
403 n_tokens: u32,
404 batch_start_token_idx: u32,
405 scratch_a: &mut [f32],
406 scratch_b: &mut [f32],
407 max_layers: u32,
408 ) -> Result<(), PrefillDispatchFailure> {
409 #[cfg(not(target_arch = "wasm32"))]
410 if self.qwen35_gguf_required || self.gemma4_required {
411 return Err(PrefillDispatchFailure::Layer {
412 layer: 0,
413 stage: PrefillLayerStage::DispatchQueryFfn,
414 });
415 }
416 if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
417 return Err(PrefillDispatchFailure::Layer {
420 layer: 0,
421 stage: PrefillLayerStage::DispatchQueryFfn,
422 });
423 }
424 let n_layer = index.hyperparams.n_layer;
425 if n_layer == 0 || n_tokens == 0 {
426 return Err(PrefillDispatchFailure::EmptyInput);
427 }
428 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
431 if crate::llm_bench::resident_prefill_enabled() {
432 if self
433 .dispatch_prefill_chunk_resident(
434 index,
435 &batch_hidden[..],
436 emb_dim,
437 n_tokens,
438 batch_start_token_idx,
439 max_layers,
440 )
441 .is_some()
442 {
443 crate::llm_bench::record_resident_prefill_hit();
444 return Ok(());
445 }
446 crate::llm_bench::record_resident_prefill_fallback();
447 }
448 let limit = if max_layers == 0 {
449 n_layer
450 } else {
451 max_layers.min(n_layer)
452 };
453 for layer in 0..limit {
454 if let Err(stage) = self.dispatch_prefill_layer_batch(
455 index,
456 layer,
457 batch_hidden,
458 emb_dim,
459 n_tokens,
460 batch_start_token_idx,
461 scratch_a,
462 scratch_b,
463 ) {
464 return Err(PrefillDispatchFailure::Layer { layer, stage });
465 }
466 }
467 Ok(())
468 }
469
470 pub fn dispatch_transformer_layer(
472 &mut self,
473 index: &crate::gguf_sharder::GgufTensorIndex,
474 layer: u32,
475 token_idx: u32,
476 hidden: &mut [f32],
477 emb_dim: usize,
478 scratch_a: &mut [f32],
479 scratch_b: &mut [f32],
480 ) -> bool {
481 #[cfg(not(target_arch = "wasm32"))]
482 if self.gemma4_required {
483 return false;
485 }
486 #[cfg(not(target_arch = "wasm32"))]
487 if self.qwen35_gguf_required {
488 return self.dense_qwen_layer(
489 index, layer, token_idx, hidden, emb_dim, scratch_a, scratch_b,
490 );
491 }
492 if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
493 #[cfg(not(target_arch = "wasm32"))]
494 {
495 let Some(plan) = self.qwen35_plan(index) else {
496 return false;
497 };
498 if emb_dim != plan.profile.hidden
499 || hidden.len() != emb_dim
500 || layer as usize >= plan.layers.len()
501 {
502 return false;
503 }
504 let Some(session) = &mut self.qwen35_ftw else {
505 return false;
506 };
507 return session.position() == Ok(token_idx as usize)
508 && session.forward(layer as usize..layer as usize + 1, hidden) == Ok(1);
509 }
510 #[cfg(target_arch = "wasm32")]
511 return false;
512 }
513 if index.hyperparams.architecture == etalon_model::inference::ARCH_GEMMA4 {
514 return false;
515 }
516 etalon_inference::ordinary::HybridEngine { host: self }.layer(
517 index, layer, token_idx, hidden, emb_dim, scratch_a, scratch_b,
518 )
519 }
520
521 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
524 pub fn try_cuda_mega_pass_decode(
525 &mut self,
526 index: &crate::gguf_sharder::GgufTensorIndex,
527 hidden: &mut [f32],
528 emb_dim: usize,
529 token_idx: u32,
530 ) -> Option<u32> {
531 if self.qwen35_gguf_required || self.gemma4_required {
532 return None;
533 }
534 self.try_prepared_cuda_decode(index, hidden, emb_dim, token_idx)
535 }
536
537 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
541 pub fn try_cuda_mega_pass_decode_token(
542 &mut self,
543 index: &crate::gguf_sharder::GgufTensorIndex,
544 token_id: u32,
545 hidden: &mut [f32],
546 emb_dim: usize,
547 token_idx: u32,
548 ) -> Option<u32> {
549 if self.qwen35_gguf_required || self.gemma4_required {
550 return None;
551 }
552 self.try_prepared_cuda_decode_token(index, token_id, hidden, emb_dim, token_idx)
553 }
554
555 #[cfg(any())]
559 pub fn try_cuda_mega_pass_decode_unprepared_reference(
560 &self,
561 index: &crate::gguf_sharder::GgufTensorIndex,
562 hidden: &mut [f32],
563 emb_dim: usize,
564 token_idx: u32,
565 ) -> Option<u32> {
566 use super::RMS_NORM_EPS;
567 use crate::ggml_quants::{GGML_TYPE_Q4_K_SOA, fetch_tensor_bytes};
568 use crate::gguf_bridge::cpu_ops::dequant_norm_row_into;
569 use crate::inference::cuda_lane::{MegaPassLayerDims, MegaPassLayerWeights};
570
571 let h = index.hyperparams;
572 let n_layer = h.n_layer;
573 let n_embd = h.n_embd as usize;
574 let n_head = h.n_head as usize;
575 let n_kv = h.effective_n_kv_head() as usize;
576 let head_dim = h.head_dim() as usize;
577 if n_layer == 0 || n_embd == 0 || n_embd > 4096 || n_embd != emb_dim {
578 return None;
579 }
580 let layout = self.kv_layout?;
581 if layout.int8 || layout.dict_k > 0 {
582 return None;
583 }
584 let mmap = self.model_bytes()?;
585 let tds = index.tensor_data_start;
586
587 struct LayerRaw<'a> {
589 q_raw: &'a [u8],
590 k_raw: &'a [u8],
591 v_raw: &'a [u8],
592 o_raw: &'a [u8],
593 g_raw: &'a [u8],
594 u_raw: &'a [u8],
595 d_raw: &'a [u8],
596 }
597 let mut layer_raws: Vec<LayerRaw<'_>> = Vec::with_capacity(n_layer as usize);
598 let mut layer_dims: Vec<MegaPassLayerDims> = Vec::with_capacity(n_layer as usize);
599 let mut all_attn_norms: Vec<Vec<f32>> = Vec::with_capacity(n_layer as usize);
600 let mut all_ffn_norms: Vec<Vec<f32>> = Vec::with_capacity(n_layer as usize);
601
602 for l in 0..n_layer {
603 let t = index.get_layer_tensors(l);
604
605 let q_info = t.attn_q.as_ref()?;
606 let k_info = t.attn_k.as_ref()?;
607 let v_info = t.attn_v.as_ref()?;
608 let o_info = t.attn_output.as_ref()?;
609 let g_info = t.ffn_gate.as_ref()?;
610 let u_info = t.ffn_up.as_ref()?;
611 let d_info = t.ffn_down.as_ref()?;
612
613 if q_info.ggml_type != GGML_TYPE_Q4_K_SOA
614 || k_info.ggml_type != GGML_TYPE_Q4_K_SOA
615 || v_info.ggml_type != GGML_TYPE_Q4_K_SOA
616 || o_info.ggml_type != GGML_TYPE_Q4_K_SOA
617 || g_info.ggml_type != GGML_TYPE_Q4_K_SOA
618 || u_info.ggml_type != GGML_TYPE_Q4_K_SOA
619 || d_info.ggml_type != GGML_TYPE_Q4_K_SOA
620 {
621 log::debug!(
622 "mega_pass_decode|skip|layer{l}|not_soa|q={} k={} v={} o={} g={} u={} d={}",
623 q_info.ggml_type,
624 k_info.ggml_type,
625 v_info.ggml_type,
626 o_info.ggml_type,
627 g_info.ggml_type,
628 u_info.ggml_type,
629 d_info.ggml_type
630 );
631 return None;
632 }
633
634 let (q_in, q_out) = Self::matmul_dims(q_info);
635 let (k_in, k_out) = Self::matmul_dims(k_info);
636 let (_v_in, _v_out) = Self::matmul_dims(v_info);
637 let (o_in, o_out) = Self::matmul_dims(o_info);
638 let (g_in, g_out) = Self::matmul_dims(g_info);
639 let (u_in, u_out) = Self::matmul_dims(u_info);
640 let (d_in, d_out) = Self::matmul_dims(d_info);
641
642 let q_raw = fetch_tensor_bytes(mmap, tds, q_info).ok()?;
643 let k_raw = fetch_tensor_bytes(mmap, tds, k_info).ok()?;
644 let v_raw = fetch_tensor_bytes(mmap, tds, v_info).ok()?;
645 let o_raw = fetch_tensor_bytes(mmap, tds, o_info).ok()?;
646 let g_raw = fetch_tensor_bytes(mmap, tds, g_info).ok()?;
647 let u_raw = fetch_tensor_bytes(mmap, tds, u_info).ok()?;
648 let d_raw = fetch_tensor_bytes(mmap, tds, d_info).ok()?;
649
650 let mut attn_norm = vec![0.0f32; n_embd];
651 let mut ffn_norm = vec![0.0f32; n_embd];
652 if let Some(an_info) = t.attn_norm.as_ref() {
653 if dequant_norm_row_into(mmap, tds, an_info, &mut attn_norm) < n_embd {
654 return None;
655 }
656 }
657 if let Some(fn_info) = t.ffn_norm.as_ref() {
658 if dequant_norm_row_into(mmap, tds, fn_info, &mut ffn_norm) < n_embd {
659 return None;
660 }
661 }
662 all_attn_norms.push(attn_norm);
663 all_ffn_norms.push(ffn_norm);
664 layer_raws.push(LayerRaw {
665 q_raw,
666 k_raw,
667 v_raw,
668 o_raw,
669 g_raw,
670 u_raw,
671 d_raw,
672 });
673 layer_dims.push(MegaPassLayerDims {
674 q_in,
675 q_out,
676 kv_in: k_in,
677 kv_out: k_out,
678 o_in,
679 o_out,
680 gate_in: g_in,
681 gate_out: g_out,
682 up_in: u_in,
683 up_out: u_out,
684 down_in: d_in,
685 down_out: d_out,
686 });
687 }
688
689 let layer_weights: Vec<MegaPassLayerWeights<'_>> = layer_raws
691 .iter()
692 .zip(all_attn_norms.iter().zip(all_ffn_norms.iter()))
693 .map(|(r, (an, fn_))| MegaPassLayerWeights {
694 attn_norm: &an[..n_embd],
695 q_raw: r.q_raw,
696 k_raw: r.k_raw,
697 v_raw: r.v_raw,
698 o_raw: r.o_raw,
699 ffn_norm: &fn_[..n_embd],
700 gate_raw: r.g_raw,
701 up_raw: r.u_raw,
702 down_raw: r.d_raw,
703 })
704 .collect();
705
706 let mut output_norm_buf = vec![0.0f32; n_embd];
708 let output_norm: Option<&[f32]> = if let Some(on_info) = index.output_norm_info() {
709 if dequant_norm_row_into(mmap, tds, on_info, &mut output_norm_buf) >= n_embd {
710 Some(&output_norm_buf[..n_embd])
711 } else {
712 None
713 }
714 } else {
715 None
716 };
717
718 let lm_info = index.logits_projection_info()?;
720 let (lm_in, lm_out) = Self::matmul_dims(lm_info);
721 let lm_raw = fetch_tensor_bytes(mmap, tds, lm_info).ok()?;
722 let lm_head_raw = if lm_info.ggml_type == GGML_TYPE_Q4_K_SOA {
723 Some(lm_raw)
724 } else {
725 None
726 };
727
728 let rope_base = h.effective_rope_freq_base();
729 let rope_scale = h.effective_rope_scale();
730 let rms_eps = RMS_NORM_EPS;
731
732 crate::try_cuda_mega_pass(
733 n_embd,
734 n_head,
735 n_kv,
736 head_dim,
737 n_layer,
738 token_idx,
739 layout.max_context,
740 layout.layer_stride,
741 layout.slot_kv_elems,
742 rope_base,
743 rope_scale,
744 rms_eps,
745 &mut hidden[..n_embd],
746 &layer_weights,
747 &layer_dims,
748 output_norm,
749 lm_head_raw,
750 lm_in,
751 lm_out,
752 )
753 }
754
755 pub fn dispatch_transformer_forward(
758 &mut self,
759 index: &crate::gguf_sharder::GgufTensorIndex,
760 hidden: &mut [f32],
761 emb_dim: usize,
762 scratch_a: &mut [f32],
763 scratch_b: &mut [f32],
764 token_idx: u32,
765 max_layers: u32,
766 ) -> u32 {
767 #[cfg(not(target_arch = "wasm32"))]
768 if self.gemma4_required {
769 return 0;
770 }
771 #[cfg(not(target_arch = "wasm32"))]
772 if self.qwen35_gguf_required {
773 return self.dense_qwen_forward(
774 index, hidden, emb_dim, token_idx, max_layers, scratch_a, scratch_b,
775 );
776 }
777 if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
778 #[cfg(not(target_arch = "wasm32"))]
779 return self.qwen35_forward(index, hidden, emb_dim, token_idx, max_layers);
780 #[cfg(target_arch = "wasm32")]
781 return 0;
782 }
783 let n_layer = index.hyperparams.n_layer;
784 if n_layer == 0 {
785 return 0;
786 }
787 let limit = if max_layers == 0 {
788 n_layer
789 } else {
790 max_layers.min(n_layer)
791 };
792 use std::sync::atomic::{AtomicBool, Ordering as DbgOrdering};
794 static FWD_DBG_DONE: AtomicBool = AtomicBool::new(false);
795 let dbg = crate::gguf_bridge::execution_env::var("QUALIA_LLM_DEBUG_DECODE").is_ok()
796 && !FWD_DBG_DONE.swap(true, DbgOrdering::Relaxed);
797 if dbg {
798 let nf = hidden[..emb_dim].iter().filter(|v| !v.is_finite()).count();
799 eprintln!(
800 "[fwd-dbg] post-embed nonfinite={}/{} sample={:?}",
801 nf,
802 emb_dim,
803 &hidden[..emb_dim.min(4)]
804 );
805 }
806 let mut ran = 0u32;
807 for layer in 0..limit {
808 if self.dispatch_transformer_layer(
809 index, layer, token_idx, hidden, emb_dim, scratch_a, scratch_b,
810 ) {
811 ran += 1;
812 }
813 if dbg {
814 let nf = hidden[..emb_dim].iter().filter(|v| !v.is_finite()).count();
815 eprintln!(
816 "[fwd-dbg] after layer {} nonfinite={}/{} ran={} sample={:?}",
817 layer,
818 nf,
819 emb_dim,
820 ran,
821 &hidden[..emb_dim.min(4)]
822 );
823 if nf > 0 {
824 break;
825 }
826 }
827 }
828 ran
829 }
830
831 #[cfg(target_arch = "wasm32")]
833 pub async fn dispatch_transformer_layer_async(
834 &mut self,
835 index: &crate::gguf_sharder::GgufTensorIndex,
836 layer: u32,
837 token_idx: u32,
838 hidden: &mut [f32],
839 emb_dim: usize,
840 scratch_a: &mut [f32],
841 scratch_b: &mut [f32],
842 ) -> bool {
843 let tensors = index.get_layer_tensors(layer);
844 let mut attn_ok = false;
845
846 if tensors.attn_q.is_some() && tensors.attn_k.is_some() && tensors.attn_v.is_some() {
847 if let Some(n) = self
848 .dispatch_attention_layer_async(
849 index,
850 layer,
851 token_idx,
852 &hidden[..emb_dim],
853 emb_dim,
854 &tensors,
855 scratch_a,
856 scratch_b,
857 )
858 .await
859 {
860 add_residual_inplace(&mut hidden[..emb_dim], &scratch_a[..n], n);
861 attn_ok = true;
862 }
863 } else if let Some(info) = tensors.attn_output {
864 let (n_in, n_out) = Self::matmul_dims(&info);
865 if n_in <= emb_dim
866 && self
867 .dispatch_gemm_into_async(index, &info, &hidden[..n_in], scratch_a, n_in, n_out)
868 .await
869 {
870 add_residual_inplace(
871 &mut hidden[..emb_dim],
872 &scratch_a[..n_out],
873 emb_dim.min(n_out),
874 );
875 attn_ok = true;
876 }
877 }
878
879 if !attn_ok && tensors.attn_output.is_none() && tensors.ffn_gate.is_none() {
880 return false;
881 }
882
883 self.dispatch_ffn_block_pre_norm_async(
884 index, hidden, emb_dim, &tensors, scratch_a, scratch_b,
885 )
886 .await
887 }
888
889 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
891 pub(crate) fn encode_attn_ffn_tail_gpu(
892 &self,
893 pipeline: &mut WasmGpuPipeline,
894 index: &crate::gguf_sharder::GgufTensorIndex,
895 layer: u32,
896 token_idx: u32,
897 emb_dim: usize,
898 tensors: &crate::gguf_sharder::LayerTensors,
899 token_hidden: &etalon::Buffer,
900 attn_input: Option<&etalon::Buffer>,
901 work_aliases_hidden: bool,
902 ) -> bool {
903 let mmap = match self.model_bytes() {
904 Some(m) => m,
905 None => return false,
906 };
907 let h = index.hyperparams;
908 let n_embd = h.n_embd as usize;
909 let layout = match self.kv_layout {
910 Some(l) => l,
911 None => return false,
912 };
913 let work_buf = self.gemm_output_buf.as_ref().unwrap();
914 let aux_buf = self.gemm_aux_buf.as_ref().unwrap();
915 let norm_buf = self.norm_weight_buf.as_ref().unwrap();
916 let q_info = match tensors.attn_q.as_ref() {
917 Some(i) => i,
918 None => return false,
919 };
920 let q_raw =
921 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, q_info) {
922 Ok(s) => s,
923 Err(_) => return false,
924 };
925 let q_in_buf = if let Some(pre) = attn_input {
926 pre
927 } else if let Some(norm) = tensors.attn_norm.as_ref() {
928 if !self.upload_norm_weights(mmap, index.tensor_data_start, norm, n_embd) {
929 return false;
930 }
931 self.encode_elem(
932 pipeline,
933 ELEM_OP_RMS_NORM,
934 n_embd as u32,
935 1,
936 token_hidden,
937 norm_buf,
938 aux_buf,
939 );
940 aux_buf
941 } else {
942 token_hidden
943 };
944 let ffn_buf = self.gemm_ffn_buf.as_ref().unwrap();
945 let emb_bytes = (emb_dim * 4) as etalon::BufferAddress;
946 let q_dim = (h.n_head * h.head_dim()) as usize;
947 let (mask_words, mask_active, mask_word_count) =
948 Self::attention_kv_mask_for_dispatch(&layout, token_idx, 0);
949 let q_params = Self::attention_gpu_params(
950 &h,
951 &layout,
952 layer,
953 token_idx,
954 q_info,
955 q_raw.len(),
956 0,
957 1,
958 token_idx,
959 mask_active,
960 mask_word_count,
961 0,
962 );
963 let q_off = self.mc8_upload_attn_param(&q_params);
964 if mask_active != 0 {
965 self.gpu_queue().write_buffer(
966 self.attention_mask_buf.as_ref().unwrap(),
967 0,
968 bytemuck::cast_slice(&mask_words),
969 );
970 }
971 if !self.encode_attention_pass_gpu(
972 pipeline,
973 q_in_buf,
974 ffn_buf,
975 n_embd,
976 1,
977 token_idx,
978 &layout,
979 layer,
980 token_idx,
981 &h,
982 q_info,
983 q_raw,
984 0,
985 h.n_head,
986 q_off,
987 Mc8WeightRole::AttnQ,
988 ) {
989 return false;
990 }
991 self.mc8_flush(pipeline);
992 if let Some(out_info) = tensors.attn_output.as_ref() {
993 let (o_in, o_out) = Self::matmul_dims(out_info);
994 let o_raw = match crate::ggml_quants::fetch_tensor_bytes(
995 mmap,
996 index.tensor_data_start,
997 out_info,
998 ) {
999 Ok(s) => s,
1000 Err(_) => return false,
1001 };
1002 if work_aliases_hidden {
1003 pipeline
1004 .encoder
1005 .copy_buffer_to_buffer(token_hidden, 0, aux_buf, 0, emb_bytes);
1006 self.mc8_flush(pipeline);
1007 }
1008 if o_in > q_dim
1009 || !self.encode_gemm_bufs(pipeline, out_info, o_raw, o_in, o_out, ffn_buf, work_buf)
1010 {
1011 return false;
1012 }
1013 self.mc8_flush(pipeline);
1014 let attn_residual_base: &etalon::Buffer = if work_aliases_hidden {
1015 aux_buf
1016 } else {
1017 token_hidden
1018 };
1019 self.encode_residual_add_gpu(
1021 pipeline,
1022 attn_residual_base,
1023 work_buf,
1024 token_hidden,
1025 ffn_buf,
1026 emb_dim as u32,
1027 );
1028 } else {
1029 self.encode_residual_add_gpu(
1030 pipeline,
1031 token_hidden,
1032 ffn_buf,
1033 token_hidden,
1034 aux_buf,
1035 emb_dim as u32,
1036 );
1037 }
1038 self.mc8_flush(pipeline);
1039 let gate_info = match tensors.ffn_gate.as_ref() {
1040 Some(i) => i,
1041 None => return false,
1042 };
1043 let up_info = match tensors.ffn_up.as_ref() {
1044 Some(i) => i,
1045 None => return false,
1046 };
1047 let down_info = match tensors.ffn_down.as_ref() {
1048 Some(i) => i,
1049 None => return false,
1050 };
1051 let (gate_in, n_ffn) = Self::matmul_dims(gate_info);
1052 let (up_in, up_out) = Self::matmul_dims(up_info);
1053 let (dn_in, dn_out) = Self::matmul_dims(down_info);
1054 if gate_in > n_embd
1055 || up_in != gate_in
1056 || up_out != n_ffn
1057 || dn_in != n_ffn
1058 || dn_out < n_embd
1059 {
1060 return false;
1061 }
1062 let gate_raw = match crate::ggml_quants::fetch_tensor_bytes(
1063 mmap,
1064 index.tensor_data_start,
1065 gate_info,
1066 ) {
1067 Ok(s) => s,
1068 Err(_) => return false,
1069 };
1070 let up_raw =
1071 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, up_info) {
1072 Ok(s) => s,
1073 Err(_) => return false,
1074 };
1075 let down_raw = match crate::ggml_quants::fetch_tensor_bytes(
1076 mmap,
1077 index.tensor_data_start,
1078 down_info,
1079 ) {
1080 Ok(s) => s,
1081 Err(_) => return false,
1082 };
1083 let base_save = match self.prefill_scratch_buf.as_ref() {
1084 Some(b) => b,
1085 None => return false,
1086 };
1087 pipeline
1088 .encoder
1089 .copy_buffer_to_buffer(token_hidden, 0, base_save, 0, emb_bytes);
1090 self.mc8_flush(pipeline);
1091 if let Some(norm) = tensors.ffn_norm.as_ref() {
1092 if !self.upload_norm_weights(mmap, index.tensor_data_start, norm, n_embd) {
1093 return false;
1094 }
1095 self.encode_elem(
1096 pipeline,
1097 ELEM_OP_RMS_NORM,
1098 n_embd as u32,
1099 1,
1100 token_hidden,
1101 norm_buf,
1102 aux_buf,
1103 );
1104 } else {
1105 pipeline
1106 .encoder
1107 .copy_buffer_to_buffer(token_hidden, 0, aux_buf, 0, emb_bytes);
1108 }
1109 self.mc8_flush(pipeline);
1110 if !self.encode_gemm_bufs(
1111 pipeline, gate_info, gate_raw, gate_in, n_ffn, aux_buf, work_buf,
1112 ) {
1113 return false;
1114 }
1115 self.mc8_flush(pipeline);
1116 if !self.encode_gemm_bufs(pipeline, up_info, up_raw, up_in, n_ffn, aux_buf, ffn_buf) {
1117 return false;
1118 }
1119 self.mc8_flush(pipeline);
1120 self.encode_elem(
1121 pipeline,
1122 ELEM_OP_SILU_MUL,
1123 n_ffn as u32,
1124 1,
1125 work_buf,
1126 ffn_buf,
1127 aux_buf,
1128 );
1129 self.mc8_flush(pipeline);
1130 if !self.encode_gemm_bufs(
1131 pipeline, down_info, down_raw, dn_in, dn_out, aux_buf, work_buf,
1132 ) {
1133 return false;
1134 }
1135 self.mc8_flush(pipeline);
1136 self.encode_residual_add_gpu(
1139 pipeline,
1140 base_save,
1141 work_buf,
1142 token_hidden,
1143 aux_buf,
1144 emb_dim as u32,
1145 );
1146 self.mc8_flush(pipeline);
1147 true
1148 }
1149
1150 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
1153 #[allow(dead_code)]
1154 pub(crate) fn encode_transformer_layer_gpu(
1155 &self,
1156 pipeline: &mut WasmGpuPipeline,
1157 index: &crate::gguf_sharder::GgufTensorIndex,
1158 layer: u32,
1159 token_idx: u32,
1160 emb_dim: usize,
1161 ) -> bool {
1162 let tensors = index.get_layer_tensors(layer);
1163 let layout = match self.kv_layout {
1164 Some(l) => l,
1165 None => return false,
1166 };
1167 let mmap = match self.model_bytes() {
1168 Some(m) => m,
1169 None => return false,
1170 };
1171 let h = index.hyperparams;
1172 let n_embd = h.n_embd as usize;
1173 if emb_dim < n_embd {
1174 return false;
1175 }
1176 let hidden_buf = self.gemm_input_buf.as_ref().unwrap();
1177 let work_buf = self.gemm_output_buf.as_ref().unwrap();
1178 let aux_buf = self.gemm_aux_buf.as_ref().unwrap();
1179 let norm_buf = self.norm_weight_buf.as_ref().unwrap();
1180
1181 let (k_info, v_info) = match (tensors.attn_k.as_ref(), tensors.attn_v.as_ref()) {
1182 (Some(k), Some(v)) => (k, v),
1183 _ => return false,
1184 };
1185 if tensors.attn_q.is_none() {
1186 return false;
1187 }
1188 let k_raw =
1189 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info) {
1190 Ok(s) => s,
1191 Err(_) => return false,
1192 };
1193 let v_raw =
1194 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info) {
1195 Ok(s) => s,
1196 Err(_) => return false,
1197 };
1198
1199 let attn_input = if let Some(norm) = tensors.attn_norm.as_ref() {
1200 if !self.upload_norm_weights(mmap, index.tensor_data_start, norm, n_embd) {
1201 return false;
1202 }
1203 self.encode_elem(
1204 pipeline,
1205 ELEM_OP_RMS_NORM,
1206 n_embd as u32,
1207 1,
1208 hidden_buf,
1209 norm_buf,
1210 aux_buf,
1211 );
1212 self.mc8_flush(pipeline);
1213 aux_buf
1214 } else {
1215 hidden_buf
1216 };
1217
1218 let n_kv = h.effective_n_kv_head();
1219 let mut attn_arena = Mc8UniformArena {
1220 bytes: [0u8; MC8_MAX_GEMM_UNIFORM_SLOTS * MC8_UNIFORM_ALIGN],
1221 slots: 0,
1222 };
1223 let k_params = Self::attention_gpu_params(
1224 &h,
1225 &layout,
1226 layer,
1227 token_idx,
1228 k_info,
1229 k_raw.len(),
1230 1,
1231 1,
1232 token_idx,
1233 0,
1234 0,
1235 0,
1236 );
1237 let v_params = Self::attention_gpu_params(
1238 &h,
1239 &layout,
1240 layer,
1241 token_idx,
1242 v_info,
1243 v_raw.len(),
1244 2,
1245 1,
1246 token_idx,
1247 0,
1248 0,
1249 0,
1250 );
1251 let k_off = attn_arena.push(&k_params);
1252 let v_off = attn_arena.push(&v_params);
1253 attn_arena.upload(
1254 self.gpu_queue(),
1255 self.attention_params_buf.as_ref().unwrap(),
1256 );
1257 if !self.encode_attention_pass_gpu(
1258 pipeline,
1259 attn_input,
1260 work_buf,
1261 n_embd,
1262 1,
1263 token_idx,
1264 &layout,
1265 layer,
1266 token_idx,
1267 &h,
1268 k_info,
1269 k_raw,
1270 1,
1271 n_kv,
1272 k_off,
1273 Mc8WeightRole::AttnK,
1274 ) {
1275 return false;
1276 }
1277 if !self.encode_attention_pass_gpu(
1278 pipeline,
1279 attn_input,
1280 work_buf,
1281 n_embd,
1282 1,
1283 token_idx,
1284 &layout,
1285 layer,
1286 token_idx,
1287 &h,
1288 v_info,
1289 v_raw,
1290 2,
1291 n_kv,
1292 v_off,
1293 Mc8WeightRole::AttnV,
1294 ) {
1295 return false;
1296 }
1297 self.mc8_flush(pipeline);
1298 self.encode_attn_ffn_tail_gpu(
1299 pipeline,
1300 index,
1301 layer,
1302 token_idx,
1303 emb_dim,
1304 &tensors,
1305 hidden_buf,
1306 Some(attn_input),
1307 false,
1308 )
1309 }
1310
1311 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
1318 pub async fn dispatch_transformer_forward_async(
1319 &mut self,
1320 index: &crate::gguf_sharder::GgufTensorIndex,
1321 hidden: &mut [f32],
1322 emb_dim: usize,
1323 _scratch_a: &mut [f32],
1324 _scratch_b: &mut [f32],
1325 token_idx: u32,
1326 max_layers: u32,
1327 ) -> u32 {
1328 let n_layer = index.hyperparams.n_layer;
1329 if n_layer == 0 || !self.mc8_buffers_ready() {
1330 return 0;
1331 }
1332 if self.prefill_work_buf_a.is_none() || self.prefill_work_buf_b.is_none() {
1333 wlog("[MC8] decode forward: prefill work buffers missing — cannot run super-arena");
1334 return 0;
1335 }
1336 if !self.mc8_weights_resident {
1338 let _ = self.mc8_upload_all_resident_weights(index);
1339 }
1340 let limit = if max_layers == 0 {
1341 n_layer
1342 } else {
1343 max_layers.min(n_layer)
1344 };
1345 let n_embd = index.hyperparams.n_embd as usize;
1346 if emb_dim < n_embd || n_embd > hidden.len() || n_embd > self.gemm_max_input_floats {
1347 return 0;
1348 }
1349 let prefill_scratch = match self.prefill_scratch_buf.as_ref() {
1350 Some(b) => b,
1351 None => return 0,
1352 };
1353 let batch_buf = self.gemm_input_buf.as_ref().unwrap();
1354 let token_buf = self.gemm_output_buf.as_ref().unwrap();
1355 if self.norm_weight_buf.is_none() {
1356 return 0;
1357 }
1358 self.gpu_queue()
1359 .write_buffer(batch_buf, 0, bytemuck::cast_slice(&hidden[..n_embd]));
1360 let mmap = match self.model_bytes() {
1361 Some(m) => m,
1362 None => return 0,
1363 };
1364 let layout = match self.kv_layout {
1365 Some(l) => l,
1366 None => return 0,
1367 };
1368 let n_tokens = 1u32;
1369 let mut ran = 0u32;
1370 let mut layer_uniform_cursors = Mc8ChunkUniformCursors {
1375 attn: 0,
1376 elem: 0,
1377 gemm: 0,
1378 };
1379 let mut enc = WasmGpuPipeline::begin(self);
1380 for layer in 0..limit {
1381 if layer > 0 && (layer % MC8_LAYERS_PER_ENCODER) == 0 {
1382 self.mc8_flush(&mut enc);
1383 layer_uniform_cursors.reset();
1384 }
1385 let tensors = index.get_layer_tensors(layer);
1386 let k_info = match tensors.attn_k.as_ref() {
1387 Some(i) => i,
1388 None => break,
1389 };
1390 let v_info = match tensors.attn_v.as_ref() {
1391 Some(i) => i,
1392 None => break,
1393 };
1394 let k_raw =
1395 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info)
1396 {
1397 Ok(s) => s,
1398 Err(_) => break,
1399 };
1400 let v_raw =
1401 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info)
1402 {
1403 Ok(s) => s,
1404 Err(_) => break,
1405 };
1406 let h = index.hyperparams;
1407 let n_kv = h.effective_n_kv_head();
1408 let used_attn_norm = tensors.attn_norm.is_some();
1409 let (uniforms, geom) = match self.mc8_stage_prefill_layer_super_arena(
1410 index,
1411 layer,
1412 &tensors,
1413 token_idx,
1414 n_tokens,
1415 emb_dim,
1416 used_attn_norm,
1417 k_info,
1418 &k_raw,
1419 v_info,
1420 &v_raw,
1421 &mut layer_uniform_cursors,
1422 ) {
1423 Some(v) => v,
1424 None => break,
1425 };
1426 let attn_src = if used_attn_norm {
1427 if let (Some(norm), Some(off)) =
1428 (tensors.attn_norm.as_ref(), uniforms.attn_norm_elem_off)
1429 {
1430 let (norm_b, norm_b_off) = match self.mc8_norm_source(
1431 mmap,
1432 index.tensor_data_start,
1433 norm,
1434 n_embd,
1435 layer,
1436 false,
1437 ) {
1438 Some(v) => v,
1439 None => break,
1440 };
1441 self.encode_elem_offset(
1442 &mut enc,
1443 ELEM_OP_RMS_NORM,
1444 n_embd as u32,
1445 n_tokens,
1446 batch_buf,
1447 0,
1448 geom.batch_in_bytes,
1449 norm_b,
1450 norm_b_off,
1451 geom.n_embd_bytes,
1452 prefill_scratch,
1453 0,
1454 geom.batch_in_bytes,
1455 0,
1456 0,
1457 0,
1458 0,
1459 0,
1460 0,
1461 off,
1462 );
1463 }
1464 prefill_scratch
1465 } else {
1466 batch_buf
1467 };
1468 let n_kv_wg = n_tokens.saturating_mul(n_kv);
1469 let kv_dim = (n_kv * h.head_dim()) as usize;
1472 let kv_proj_bytes = (kv_dim * n_tokens as usize * 4) as etalon::BufferAddress;
1473 let k_proj = self.mc8_k_proj_buf.as_ref().unwrap();
1474 let v_proj = self.mc8_v_proj_buf.as_ref().unwrap();
1475 if !self.encode_gemm_bufs_offset(
1476 &mut enc,
1477 k_info,
1478 k_raw,
1479 n_embd,
1480 kv_dim,
1481 attn_src,
1482 0,
1483 geom.batch_in_bytes,
1484 k_proj,
1485 0,
1486 kv_proj_bytes,
1487 n_tokens,
1488 n_embd as u32,
1489 kv_dim as u32,
1490 uniforms.off_k_gemm,
1491 layer,
1492 Mc8WeightRole::AttnK,
1493 ) {
1494 break;
1495 }
1496 if !self.encode_gemm_bufs_offset(
1497 &mut enc,
1498 v_info,
1499 v_raw,
1500 n_embd,
1501 kv_dim,
1502 attn_src,
1503 0,
1504 geom.batch_in_bytes,
1505 v_proj,
1506 0,
1507 kv_proj_bytes,
1508 n_tokens,
1509 n_embd as u32,
1510 kv_dim as u32,
1511 uniforms.off_v_gemm,
1512 layer,
1513 Mc8WeightRole::AttnV,
1514 ) {
1515 break;
1516 }
1517 if !self.encode_attention_pass_gpu(
1518 &mut enc,
1519 k_proj,
1520 token_buf,
1521 n_embd,
1522 n_tokens,
1523 token_idx,
1524 &layout,
1525 layer,
1526 token_idx,
1527 &h,
1528 k_info,
1529 k_raw,
1530 1,
1531 n_kv_wg,
1532 uniforms.k_off,
1533 Mc8WeightRole::AttnK,
1534 ) {
1535 break;
1536 }
1537 if !self.encode_attention_pass_gpu(
1538 &mut enc,
1539 v_proj,
1540 token_buf,
1541 n_embd,
1542 n_tokens,
1543 token_idx,
1544 &layout,
1545 layer,
1546 token_idx,
1547 &h,
1548 v_info,
1549 v_raw,
1550 2,
1551 n_kv_wg,
1552 uniforms.v_off,
1553 Mc8WeightRole::AttnV,
1554 ) {
1555 break;
1556 }
1557 let work_a = self.prefill_work_buf_a.as_ref().unwrap();
1561 let work_b = self.prefill_work_buf_b.as_ref().unwrap();
1562 if !self.encode_prefill_q_ffn_tail_fused(
1563 &mut enc,
1564 index,
1565 layer,
1566 &tensors,
1567 batch_buf,
1568 attn_src,
1569 work_a,
1570 work_b,
1571 n_tokens,
1572 token_idx,
1573 emb_dim,
1574 used_attn_norm,
1575 &uniforms,
1576 &geom,
1577 ) {
1578 break;
1579 }
1580 ran += 1;
1581 }
1582 self.mc8_flush(&mut enc);
1584 if ran > 0 && !self.pipeline_read_hidden(emb_dim, hidden).await {
1585 return 0;
1586 }
1587 ran
1588 }
1589
1590 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
1593 pub async fn dispatch_forward_and_argmax_fused_async(
1594 &mut self,
1595 index: &crate::gguf_sharder::GgufTensorIndex,
1596 hidden: &mut [f32],
1597 emb_dim: usize,
1598 token_idx: u32,
1599 max_layers: u32,
1600 _chunk_logits: &mut [f32],
1601 max_chunks: u32,
1602 ) -> Option<StreamingArgmaxResult> {
1603 let n_layer = index.hyperparams.n_layer;
1604 if n_layer == 0 || !self.mc8_buffers_ready() {
1605 return None;
1606 }
1607 if self.prefill_work_buf_a.is_none() || self.prefill_work_buf_b.is_none() {
1608 return None;
1609 }
1610 if !self.mc8_weights_resident {
1611 let _ = self.mc8_upload_all_resident_weights(index);
1612 }
1613 let limit = if max_layers == 0 {
1614 n_layer
1615 } else {
1616 max_layers.min(n_layer)
1617 };
1618 let n_embd = index.hyperparams.n_embd as usize;
1619 if emb_dim < n_embd || n_embd > hidden.len() || n_embd > self.gemm_max_input_floats {
1620 return None;
1621 }
1622 let prefill_scratch = self.prefill_scratch_buf.as_ref()?;
1623 let batch_buf = self.gemm_input_buf.as_ref().unwrap();
1624 let norm_buf = self.norm_weight_buf.as_ref().unwrap();
1625 let mmap = self.model_bytes()?;
1626 let layout = self.kv_layout?;
1627
1628 self.gpu_queue()
1630 .write_buffer(batch_buf, 0, bytemuck::cast_slice(&hidden[..n_embd]));
1631
1632 let n_tokens = 1u32;
1634 let mut ran = 0u32;
1635 let mut layer_uniform_cursors = Mc8ChunkUniformCursors {
1636 attn: 0,
1637 elem: 0,
1638 gemm: 0,
1639 };
1640 let mut enc = WasmGpuPipeline::begin(self);
1641 for layer in 0..limit {
1642 if layer > 0 && (layer % MC8_LAYERS_PER_ENCODER) == 0 {
1643 self.mc8_flush(&mut enc);
1644 layer_uniform_cursors.reset();
1645 }
1646 let tensors = index.get_layer_tensors(layer);
1647 let k_info = match tensors.attn_k.as_ref() {
1648 Some(i) => i,
1649 None => break,
1650 };
1651 let v_info = match tensors.attn_v.as_ref() {
1652 Some(i) => i,
1653 None => break,
1654 };
1655 let k_raw =
1656 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info)
1657 {
1658 Ok(s) => s,
1659 Err(_) => break,
1660 };
1661 let v_raw =
1662 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info)
1663 {
1664 Ok(s) => s,
1665 Err(_) => break,
1666 };
1667 let h = index.hyperparams;
1668 let n_kv = h.effective_n_kv_head();
1669 let used_attn_norm = tensors.attn_norm.is_some();
1670 let (uniforms, geom) = match self.mc8_stage_prefill_layer_super_arena(
1671 index,
1672 layer,
1673 &tensors,
1674 token_idx,
1675 n_tokens,
1676 emb_dim,
1677 used_attn_norm,
1678 k_info,
1679 &k_raw,
1680 v_info,
1681 &v_raw,
1682 &mut layer_uniform_cursors,
1683 ) {
1684 Some(v) => v,
1685 None => break,
1686 };
1687 let attn_src = if used_attn_norm {
1688 if let (Some(norm), Some(off)) =
1689 (tensors.attn_norm.as_ref(), uniforms.attn_norm_elem_off)
1690 {
1691 let (norm_b, norm_b_off) = match self.mc8_norm_source(
1692 mmap,
1693 index.tensor_data_start,
1694 norm,
1695 n_embd,
1696 layer,
1697 false,
1698 ) {
1699 Some(v) => v,
1700 None => break,
1701 };
1702 self.encode_elem_offset(
1703 &mut enc,
1704 ELEM_OP_RMS_NORM,
1705 n_embd as u32,
1706 n_tokens,
1707 batch_buf,
1708 0,
1709 geom.batch_in_bytes,
1710 norm_b,
1711 norm_b_off,
1712 geom.n_embd_bytes,
1713 prefill_scratch,
1714 0,
1715 geom.batch_in_bytes,
1716 0,
1717 0,
1718 0,
1719 0,
1720 0,
1721 0,
1722 off,
1723 );
1724 }
1725 prefill_scratch
1726 } else {
1727 batch_buf
1728 };
1729 let n_kv_wg = n_tokens.saturating_mul(n_kv);
1730 let kv_dim = (n_kv * h.head_dim()) as usize;
1731 let kv_proj_bytes = (kv_dim * n_tokens as usize * 4) as etalon::BufferAddress;
1732 let k_proj = self.mc8_k_proj_buf.as_ref().unwrap();
1733 let v_proj = self.mc8_v_proj_buf.as_ref().unwrap();
1734 if !self.encode_gemm_bufs_offset(
1735 &mut enc,
1736 k_info,
1737 k_raw,
1738 n_embd,
1739 kv_dim,
1740 attn_src,
1741 0,
1742 geom.batch_in_bytes,
1743 k_proj,
1744 0,
1745 kv_proj_bytes,
1746 n_tokens,
1747 n_embd as u32,
1748 kv_dim as u32,
1749 uniforms.off_k_gemm,
1750 layer,
1751 Mc8WeightRole::AttnK,
1752 ) {
1753 break;
1754 }
1755 if !self.encode_gemm_bufs_offset(
1756 &mut enc,
1757 v_info,
1758 v_raw,
1759 n_embd,
1760 kv_dim,
1761 attn_src,
1762 0,
1763 geom.batch_in_bytes,
1764 v_proj,
1765 0,
1766 kv_proj_bytes,
1767 n_tokens,
1768 n_embd as u32,
1769 kv_dim as u32,
1770 uniforms.off_v_gemm,
1771 layer,
1772 Mc8WeightRole::AttnV,
1773 ) {
1774 break;
1775 }
1776 if !self.encode_attention_pass_gpu(
1777 &mut enc,
1778 k_proj,
1779 self.gemm_output_buf.as_ref().unwrap(),
1780 n_embd,
1781 n_tokens,
1782 token_idx,
1783 &layout,
1784 layer,
1785 token_idx,
1786 &h,
1787 k_info,
1788 k_raw,
1789 1,
1790 n_kv_wg,
1791 uniforms.k_off,
1792 Mc8WeightRole::AttnK,
1793 ) {
1794 break;
1795 }
1796 if !self.encode_attention_pass_gpu(
1797 &mut enc,
1798 v_proj,
1799 self.gemm_output_buf.as_ref().unwrap(),
1800 n_embd,
1801 n_tokens,
1802 token_idx,
1803 &layout,
1804 layer,
1805 token_idx,
1806 &h,
1807 v_info,
1808 v_raw,
1809 2,
1810 n_kv_wg,
1811 uniforms.v_off,
1812 Mc8WeightRole::AttnV,
1813 ) {
1814 break;
1815 }
1816 let work_a = self.prefill_work_buf_a.as_ref().unwrap();
1817 let work_b = self.prefill_work_buf_b.as_ref().unwrap();
1818 if !self.encode_prefill_q_ffn_tail_fused(
1819 &mut enc,
1820 index,
1821 layer,
1822 &tensors,
1823 batch_buf,
1824 attn_src,
1825 work_a,
1826 work_b,
1827 n_tokens,
1828 token_idx,
1829 emb_dim,
1830 used_attn_norm,
1831 &uniforms,
1832 &geom,
1833 ) {
1834 break;
1835 }
1836 ran += 1;
1837 }
1838 if ran == 0 {
1839 self.mc8_flush(&mut enc);
1840 return None;
1841 }
1842
1843 self.mc8_flush(&mut enc);
1847
1848 let output_norm_info = match index.output_norm_info() {
1850 Some(i) => i,
1851 None => {
1852 if !self.pipeline_read_hidden(emb_dim, hidden).await {
1854 return None;
1855 }
1856 return None; }
1858 };
1859 let mut norm_w = [0f32; MAX_HIDDEN_DIM];
1861 if dequant_norm_row_into(mmap, index.tensor_data_start, output_norm_info, &mut norm_w)
1862 < n_embd
1863 {
1864 return None;
1865 }
1866 self.gpu_queue()
1867 .write_buffer(norm_buf, 0, bytemuck::cast_slice(&norm_w[..n_embd]));
1868 let n_embd_bytes = (n_embd * 4) as etalon::BufferAddress;
1870 self.encode_elem_offset(
1871 &mut enc,
1872 ELEM_OP_RMS_NORM,
1873 n_embd as u32,
1874 1,
1875 batch_buf,
1876 0,
1877 n_embd_bytes,
1878 norm_buf,
1879 0,
1880 n_embd_bytes,
1881 prefill_scratch,
1882 0,
1883 n_embd_bytes,
1884 0,
1885 0,
1886 0,
1887 0,
1888 0,
1889 0,
1890 0,
1891 );
1892 enc.encoder
1894 .copy_buffer_to_buffer(prefill_scratch, 0, batch_buf, 0, n_embd_bytes);
1895 self.mc8_flush(&mut enc);
1898
1899 let resident_buf = self.mc8_logits_resident_buf.as_ref()?;
1901 let row_bytes = self.mc8_logits_row_bytes as u64;
1902 let output_buf = self.gemm_output_buf.as_ref().unwrap();
1903 let params_buf = self.gemm_params_buf.as_ref().unwrap();
1904 let staging = self.gemm_output_staging.as_ref().unwrap();
1905 let logits_info = index.logits_projection_info()?;
1906 let (n_in, vocab_size) = Self::matmul_dims(logits_info);
1907 if n_in == 0 || vocab_size == 0 || n_in > emb_dim {
1908 self.mc8_flush(&mut enc);
1909 return None;
1910 }
1911 let full_chunks = vocab_size.div_ceil(VOCAB_CHUNK_ROWS);
1912 let n_chunks = if max_chunks == 0 {
1913 full_chunks
1914 } else {
1915 (max_chunks as usize).min(full_chunks)
1916 };
1917 let top1_plan =
1918 crate::gguf_bridge::browser::webgpu::BrowserTop1Plan::new(vocab_size, n_chunks as u32)?;
1919 if !self.prepare_browser_top1(top1_plan) {
1920 self.mc8_flush(&mut enc);
1921 return None;
1922 }
1923 let params = GemmGpuParams {
1924 n_in: n_in as u32,
1925 n_out: VOCAB_CHUNK_ROWS as u32,
1926 weight_ggml_type: logits_info.ggml_type,
1927 weight_row_elems: logits_info.dims[0] as u32,
1928 weight_byte_len: (VOCAB_CHUNK_ROWS as u64 * row_bytes) as u32,
1929 n_batch: 1,
1930 in_row_stride: 0,
1931 out_row_stride: 0,
1932 };
1933 self.gpu_queue()
1934 .write_buffer(params_buf, 0, bytemuck::bytes_of(¶ms));
1935 #[cfg(target_arch = "wasm32")]
1936 let use_mmv_q8_0 =
1937 logits_info.ggml_type == crate::ggml_quants::GGML_TYPE_Q8_0 && (n_in % 32 == 0);
1938 #[cfg(target_arch = "wasm32")]
1939 let logits_pipeline: &etalon::ComputePipeline = if use_mmv_q8_0 {
1940 &self.mmv_q8_0_pipeline
1941 } else {
1942 &self.pipeline
1943 };
1944 #[cfg(not(target_arch = "wasm32"))]
1945 let logits_pipeline: &etalon::ComputePipeline = &self.pipeline;
1946 let bind_layout = logits_pipeline.get_bind_group_layout(0);
1947 for chunk_idx in 0..n_chunks {
1948 let row_start = chunk_idx * VOCAB_CHUNK_ROWS;
1949 let chunk_rows = VOCAB_CHUNK_ROWS.min(vocab_size - row_start);
1950 let weight = etalon::BindingResource::Buffer(etalon::BufferBinding {
1951 buffer: resident_buf,
1952 offset: row_start as u64 * row_bytes,
1953 size: std::num::NonZeroU64::new(chunk_rows as u64 * row_bytes),
1954 });
1955 let key = mc8_bg_hash(&[
1956 7,
1957 chunk_idx as u64,
1958 row_start as u64,
1959 row_bytes as u64,
1960 chunk_rows as u64,
1961 resident_buf as *const _ as u64,
1962 batch_buf as *const _ as u64,
1963 output_buf as *const _ as u64,
1964 ]);
1965 let bind_group = {
1966 let cached = self
1967 .mc8_bg_cache
1968 .lock()
1969 .ok()
1970 .and_then(|c| c.get(&key).cloned());
1971 match cached {
1972 Some(bg) => bg,
1973 None => {
1974 let bg =
1975 self.gpu_device()
1976 .create_bind_group(&etalon::BindGroupDescriptor {
1977 label: Some("FusedArgmaxBind"),
1978 layout: &bind_layout,
1979 entries: &[
1980 etalon::BindGroupEntry {
1981 binding: 0,
1982 resource: batch_buf.as_entire_binding(),
1983 },
1984 etalon::BindGroupEntry {
1985 binding: 1,
1986 resource: weight,
1987 },
1988 etalon::BindGroupEntry {
1989 binding: 2,
1990 resource: Self::mc8_dynamic_uniform_binding(params_buf),
1991 },
1992 etalon::BindGroupEntry {
1993 binding: 3,
1994 resource: output_buf.as_entire_binding(),
1995 },
1996 ],
1997 });
1998 if let Ok(mut c) = self.mc8_bg_cache.lock() {
1999 c.insert(key, bg.clone());
2000 }
2001 bg
2002 }
2003 }
2004 };
2005 {
2006 let mut cpass = enc
2007 .encoder
2008 .begin_compute_pass(&etalon::ComputePassDescriptor {
2009 label: None,
2010 timestamp_writes: None,
2011 });
2012 cpass.set_pipeline(logits_pipeline);
2013 cpass.set_bind_group(0, &bind_group, &[0]);
2014 #[cfg(target_arch = "wasm32")]
2015 if use_mmv_q8_0 {
2016 cpass.dispatch_workgroups((chunk_rows as u32 + 3) / 4, 1, 1);
2017 } else {
2018 cpass.dispatch_workgroups((chunk_rows as u32 + 63) / 64, 1, 1);
2019 }
2020 #[cfg(not(target_arch = "wasm32"))]
2021 cpass.dispatch_workgroups((chunk_rows as u32 + 63) / 64, 1, 1);
2022 }
2023 if !self.encode_browser_top1_chunk(
2024 &mut enc.encoder,
2025 output_buf,
2026 staging,
2027 top1_plan,
2028 chunk_idx,
2029 ) {
2030 self.mc8_flush(&mut enc);
2031 return None;
2032 }
2033 }
2034
2035 self.submit_commands(Some(enc.encoder.finish()));
2038 self.read_browser_top1(staging, top1_plan).await
2039 }
2040
2041 pub fn verify_topology_draft_batch(
2043 &mut self,
2044 index: &crate::gguf_sharder::GgufTensorIndex,
2045 ctx: &mut Vec<u32>,
2046 draft: &crate::compute_universe::TopologyDraftBatch,
2047 emb_dim: usize,
2048 emb_buf: &mut [f32],
2049 scratch_a: &mut [f32],
2050 scratch_b: &mut [f32],
2051 max_layers: u32,
2052 max_vocab_chunks: u32,
2053 ) -> u32 {
2054 #[cfg(not(target_arch = "wasm32"))]
2055 if self.gemma4_required || self.qwen35_gguf_required {
2056 return 0;
2058 }
2059 let mmap = match self.clone_model_bytes() {
2060 Some(m) => m,
2061 None => return 0,
2062 };
2063 let gamma = draft.draft_len as usize;
2064 if gamma == 0 || ctx.is_empty() {
2065 return 0;
2066 }
2067 let mut accepted = 0u32;
2068 for i in 0..gamma {
2069 let cur = *ctx.last().unwrap();
2070 let token_idx = ctx.len().saturating_sub(1) as u32;
2071 let hidden_ok =
2072 index.dequantize_token_embedding_into(mmap.as_ref(), cur, &mut emb_buf[..emb_dim]);
2073 if hidden_ok == 0 {
2074 break;
2075 }
2076 let _ = self.dispatch_transformer_forward(
2077 index,
2078 &mut emb_buf[..emb_dim],
2079 emb_dim,
2080 scratch_a,
2081 scratch_b,
2082 token_idx,
2083 max_layers,
2084 );
2085 let pred = if let Some(argmax) = self.dispatch_output_argmax_chunked(
2086 index,
2087 &emb_buf[..emb_dim],
2088 emb_dim,
2089 scratch_a,
2090 max_vocab_chunks,
2091 None,
2092 ) {
2093 if argmax.max_logit > f32::NEG_INFINITY {
2094 argmax.best_token_id
2095 } else {
2096 break;
2097 }
2098 } else {
2099 break;
2100 };
2101 if pred != draft.draft_ids[i] {
2102 break;
2103 }
2104 ctx.push(pred);
2105 accepted += 1;
2106 }
2107 accepted
2108 }
2109}