Skip to main content

qualia_core_db/gguf_bridge/
forward.rs

1//! Forward-pass orchestration: prefill (batch/chunk, sync + async + MC8-fused), transformer
2//! layer/forward (sync + async), encode helpers, topology-draft verify, and async lifecycle.
3//! Split from gguf_bridge/mod.rs (structural refactor; no behaviour change).
4use super::*;
5
6/// Exact failure point for native chunked prefill.  Returning this to the
7/// caller keeps a failed model forward from being misreported as a zero-rate
8/// benchmark.
9#[derive(Clone, Copy, Debug, PartialEq, Eq)]
10pub enum PrefillDispatchFailure {
11    EmptyInput,
12    Layer {
13        layer: u32,
14        stage: PrefillLayerStage,
15    },
16}
17
18#[derive(Clone, Copy, Debug, PartialEq, Eq)]
19pub enum PrefillLayerStage {
20    EmptyBatch,
21    MissingKvLayout,
22    MissingAttentionK,
23    MissingAttentionV,
24    MissingAttentionQ,
25    BatchHiddenTooSmall,
26    MissingModelMap,
27    FetchAttentionK,
28    FetchAttentionV,
29    DispatchAttentionK,
30    DispatchAttentionV,
31    DispatchQueryFfn,
32}
33
34impl QTensorEngine {
35    /// One transformer layer for a batched prefill chunk: batched K/V then per-token Q+FFN.
36    pub(crate) fn dispatch_prefill_layer_batch(
37        &mut self,
38        index: &crate::gguf_sharder::GgufTensorIndex,
39        layer: u32,
40        batch_hidden: &mut [f32],
41        emb_dim: usize,
42        n_tokens: u32,
43        batch_start_token_idx: u32,
44        scratch_a: &mut [f32],
45        scratch_b: &mut [f32],
46    ) -> Result<(), PrefillLayerStage> {
47        if n_tokens == 0 {
48            wlog("[prefill_layer] FAILED n_tokens=0");
49            return Err(PrefillLayerStage::EmptyBatch);
50        }
51        let layout = match self.kv_layout {
52            Some(l) => l,
53            None => {
54                wlog("[prefill_layer] FAILED kv_layout is None");
55                return Err(PrefillLayerStage::MissingKvLayout);
56            }
57        };
58        let tensors = index.get_layer_tensors(layer);
59        if index.hyperparams.architecture == etalon_model::inference::ARCH_GEMMA4
60            || Self::is_qwen35_gated_attention_layer(index, &tensors)
61        {
62            let batch_elems = emb_dim * n_tokens as usize;
63            if batch_elems > batch_hidden.len() {
64                return Err(PrefillLayerStage::BatchHiddenTooSmall);
65            }
66            for t in 0..n_tokens {
67                let off = t as usize * emb_dim;
68                let t_hidden = &mut batch_hidden[off..off + emb_dim];
69                if !self.dispatch_transformer_layer(
70                    index,
71                    layer,
72                    batch_start_token_idx + t,
73                    t_hidden,
74                    emb_dim,
75                    scratch_a,
76                    scratch_b,
77                ) {
78                    return Err(PrefillLayerStage::DispatchQueryFfn);
79                }
80            }
81            return Ok(());
82        }
83        if tensors.is_hybrid_ssm_layer() {
84            let batch_elems = emb_dim * n_tokens as usize;
85            if batch_elems > batch_hidden.len() {
86                return Err(PrefillLayerStage::BatchHiddenTooSmall);
87            }
88            for t in 0..n_tokens {
89                let off = t as usize * emb_dim;
90                let t_hidden = &mut batch_hidden[off..off + emb_dim];
91                if !self.dispatch_hybrid_ssm_layer(
92                    index, layer, t_hidden, emb_dim, &tensors, scratch_a, scratch_b,
93                ) {
94                    return Err(PrefillLayerStage::DispatchQueryFfn);
95                }
96                if !self.dispatch_ffn_block_pre_norm(
97                    index, t_hidden, emb_dim, &tensors, scratch_a, scratch_b,
98                ) {
99                    return Err(PrefillLayerStage::DispatchQueryFfn);
100                }
101            }
102            return Ok(());
103        }
104
105        let k_info = tensors
106            .attn_k
107            .as_ref()
108            .ok_or(PrefillLayerStage::MissingAttentionK)?;
109        let v_info = tensors
110            .attn_v
111            .as_ref()
112            .ok_or(PrefillLayerStage::MissingAttentionV)?;
113        if tensors.attn_q.is_none() {
114            return Err(PrefillLayerStage::MissingAttentionQ);
115        }
116        let h = index.hyperparams;
117        let n_kv = h.effective_n_kv_head();
118        let n_embd = h.n_embd as usize;
119        let batch_elems = n_embd * n_tokens as usize;
120        if batch_elems > batch_hidden.len() {
121            wlog(&format!(
122                "[prefill_layer] FAILED batch_elems OOB elems={batch_elems} hidden={}",
123                batch_hidden.len()
124            ));
125            return Err(PrefillLayerStage::BatchHiddenTooSmall);
126        }
127        let mmap = match self.model_bytes() {
128            Some(m) => m,
129            None => {
130                wlog("[prefill_layer] FAILED gguf_mmap is None");
131                return Err(PrefillLayerStage::MissingModelMap);
132            }
133        };
134        let k_raw =
135            match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info) {
136                Ok(s) => s,
137                Err(e) => {
138                    wlog(&format!("[prefill_layer] FAILED fetch attn_k bytes: {e:?}"));
139                    return Err(PrefillLayerStage::FetchAttentionK);
140                }
141            };
142        let v_raw =
143            match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info) {
144                Ok(s) => s,
145                Err(e) => {
146                    wlog(&format!("[prefill_layer] FAILED fetch attn_v bytes: {e:?}"));
147                    return Err(PrefillLayerStage::FetchAttentionV);
148                }
149            };
150        let n_kv_wg = n_tokens.saturating_mul(n_kv);
151        // attn_norm MUST be applied to the K/V projection input on ALL targets — native previously
152        // passed None here → prefill wrote K/V from the RAW residual → the KV cache exploded across
153        // layers → the whole forward (and decode reading it) blew up. (#48)
154        let mut norm_w_attn = [0f32; MAX_HIDDEN_DIM];
155        let norm_weight_attn: Option<&[f32]> = tensors.attn_norm.as_ref().and_then(|info| {
156            let n = dequant_norm_row_into(mmap, index.tensor_data_start, info, &mut norm_w_attn);
157            if n >= n_embd {
158                Some(&norm_w_attn[..n_embd])
159            } else {
160                None
161            }
162        });
163        if !self.dispatch_attention_pass(
164            &batch_hidden[..batch_elems],
165            n_embd,
166            n_tokens,
167            batch_start_token_idx,
168            &layout,
169            layer,
170            batch_start_token_idx,
171            &h,
172            k_info,
173            k_raw,
174            1,
175            n_kv_wg,
176            norm_weight_attn,
177            None,
178        ) {
179            wlog(&format!("[prefill_layer] K pass FAILED layer={layer}"));
180            return Err(PrefillLayerStage::DispatchAttentionK);
181        }
182        if !self.dispatch_attention_pass(
183            &batch_hidden[..batch_elems],
184            n_embd,
185            n_tokens,
186            batch_start_token_idx,
187            &layout,
188            layer,
189            batch_start_token_idx,
190            &h,
191            v_info,
192            v_raw,
193            2,
194            n_kv_wg,
195            norm_weight_attn,
196            None,
197        ) {
198            wlog(&format!("[prefill_layer] V pass FAILED layer={layer}"));
199            return Err(PrefillLayerStage::DispatchAttentionV);
200        }
201        for t in 0..n_tokens {
202            let abs = batch_start_token_idx + t;
203            let off = t as usize * emb_dim;
204            if !self.dispatch_attention_q_ffn_token(
205                index,
206                layer,
207                abs,
208                &mut batch_hidden[off..off + emb_dim],
209                emb_dim,
210                &tensors,
211                scratch_a,
212                scratch_b,
213            ) {
214                wlog(&format!(
215                    "[prefill_layer] q_ffn FAILED layer={layer} t={t} abs={abs}"
216                ));
217                return Err(PrefillLayerStage::DispatchQueryFfn);
218            }
219        }
220        Ok(())
221    }
222
223    /// Phase 2B: batched prefill layer via async GPU attention (K/V GPU; Q+FFN per token).
224    #[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
225    pub(crate) async fn dispatch_prefill_layer_batch_async(
226        &mut self,
227        index: &crate::gguf_sharder::GgufTensorIndex,
228        layer: u32,
229        batch_hidden: &mut [f32],
230        emb_dim: usize,
231        n_tokens: u32,
232        batch_start_token_idx: u32,
233        scratch_a: &mut [f32],
234        scratch_b: &mut [f32],
235    ) -> bool {
236        if n_tokens == 0 {
237            wlog("[prefill_layer] FAILED n_tokens=0");
238            return false;
239        }
240        let layout = match self.kv_layout {
241            Some(l) => l,
242            None => {
243                wlog("[prefill_layer] FAILED kv_layout is None");
244                return false;
245            }
246        };
247        let tensors = index.get_layer_tensors(layer);
248        let k_info = match tensors.attn_k.as_ref() {
249            Some(i) => i,
250            None => {
251                wlog(&format!(
252                    "[prefill_layer] FAILED missing attn_k layer={layer}"
253                ));
254                return false;
255            }
256        };
257        let v_info = match tensors.attn_v.as_ref() {
258            Some(i) => i,
259            None => {
260                wlog(&format!(
261                    "[prefill_layer] FAILED missing attn_v layer={layer}"
262                ));
263                return false;
264            }
265        };
266        if tensors.attn_q.is_none() {
267            wlog(&format!(
268                "[prefill_layer] FAILED missing attn_q layer={layer}"
269            ));
270            return false;
271        }
272        let h = index.hyperparams;
273        let n_kv = h.effective_n_kv_head();
274        let n_embd = h.n_embd as usize;
275        let batch_elems = n_embd * n_tokens as usize;
276        if batch_elems > batch_hidden.len() {
277            wlog(&format!(
278                "[prefill_layer] FAILED batch_elems OOB elems={batch_elems} hidden={}",
279                batch_hidden.len()
280            ));
281            return false;
282        }
283        let mmap = match self.model_bytes() {
284            Some(m) => m,
285            None => {
286                wlog("[prefill_layer] FAILED gguf_mmap is None");
287                return false;
288            }
289        };
290        let k_raw =
291            match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info) {
292                Ok(s) => s,
293                Err(e) => {
294                    wlog(&format!("[prefill_layer] FAILED fetch attn_k bytes: {e:?}"));
295                    return false;
296                }
297            };
298        let v_raw =
299            match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info) {
300                Ok(s) => s,
301                Err(e) => {
302                    wlog(&format!("[prefill_layer] FAILED fetch attn_v bytes: {e:?}"));
303                    return false;
304                }
305            };
306        let mut norm_w_attn = [0f32; MAX_HIDDEN_DIM];
307        let mut norm_scratch = [0f32; PREFILL_CHUNK_STACK_FLOATS];
308        let attn_input: &mut [f32] = if let Some(norm_info) = tensors.attn_norm.as_ref() {
309            let n =
310                dequant_norm_row_into(mmap, index.tensor_data_start, norm_info, &mut norm_w_attn);
311            if n >= n_embd {
312                for t in 0..n_tokens as usize {
313                    let off = t * n_embd;
314                    norm_scratch[off..off + n_embd]
315                        .copy_from_slice(&batch_hidden[off..off + n_embd]);
316                    rms_norm_inplace(
317                        &mut norm_scratch[off..off + n_embd],
318                        &norm_w_attn[..n_embd],
319                        RMS_NORM_EPS,
320                    );
321                }
322                &mut norm_scratch[..batch_elems]
323            } else {
324                batch_hidden
325            }
326        } else {
327            batch_hidden
328        };
329        let n_kv_wg = n_tokens.saturating_mul(n_kv);
330        if !self
331            .dispatch_attention_pass_async(
332                attn_input,
333                n_embd,
334                n_tokens,
335                batch_start_token_idx,
336                &layout,
337                layer,
338                batch_start_token_idx,
339                &h,
340                k_info,
341                k_raw,
342                1,
343                n_kv_wg,
344                None,
345            )
346            .await
347        {
348            wlog(&format!("[prefill_layer] K pass FAILED layer={layer}"));
349            return false;
350        }
351        if !self
352            .dispatch_attention_pass_async(
353                attn_input,
354                n_embd,
355                n_tokens,
356                batch_start_token_idx,
357                &layout,
358                layer,
359                batch_start_token_idx,
360                &h,
361                v_info,
362                v_raw,
363                2,
364                n_kv_wg,
365                None,
366            )
367            .await
368        {
369            wlog(&format!("[prefill_layer] V pass FAILED layer={layer}"));
370            return false;
371        }
372        for t in 0..n_tokens {
373            let abs = batch_start_token_idx + t;
374            let off = t as usize * emb_dim;
375            if !self
376                .dispatch_attention_q_ffn_token_async(
377                    index,
378                    layer,
379                    abs,
380                    &mut batch_hidden[off..off + emb_dim],
381                    emb_dim,
382                    &tensors,
383                    scratch_a,
384                    scratch_b,
385                )
386                .await
387            {
388                wlog(&format!(
389                    "[prefill_layer] q_ffn FAILED layer={layer} t={t} abs={abs}"
390                ));
391                return false;
392            }
393        }
394        true
395    }
396
397    /// Chunked prefill: populate KV arena for `n_tokens` prompt positions starting at `batch_start`.
398    pub fn dispatch_prefill_chunk(
399        &mut self,
400        index: &crate::gguf_sharder::GgufTensorIndex,
401        batch_hidden: &mut [f32],
402        emb_dim: usize,
403        n_tokens: u32,
404        batch_start_token_idx: u32,
405        scratch_a: &mut [f32],
406        scratch_b: &mut [f32],
407        max_layers: u32,
408    ) -> Result<(), PrefillDispatchFailure> {
409        #[cfg(not(target_arch = "wasm32"))]
410        if self.qwen35_gguf_required || self.gemma4_required {
411            return Err(PrefillDispatchFailure::Layer {
412                layer: 0,
413                stage: PrefillLayerStage::DispatchQueryFfn,
414            });
415        }
416        if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
417            // Qwen prompt execution is token-major in its admitted Session;
418            // the generic layer-major KV prefill has incompatible state rules.
419            return Err(PrefillDispatchFailure::Layer {
420                layer: 0,
421                stage: PrefillLayerStage::DispatchQueryFfn,
422            });
423        }
424        let n_layer = index.hyperparams.n_layer;
425        if n_layer == 0 || n_tokens == 0 {
426            return Err(PrefillDispatchFailure::EmptyInput);
427        }
428        // W3: resident single-fence-per-chunk arena (toggle-gated, default OFF). Populates the KV
429        // cache for the whole chunk in ONE submit; any ineligibility falls back to the legacy loop.
430        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
431        if crate::llm_bench::resident_prefill_enabled() {
432            if self
433                .dispatch_prefill_chunk_resident(
434                    index,
435                    &batch_hidden[..],
436                    emb_dim,
437                    n_tokens,
438                    batch_start_token_idx,
439                    max_layers,
440                )
441                .is_some()
442            {
443                crate::llm_bench::record_resident_prefill_hit();
444                return Ok(());
445            }
446            crate::llm_bench::record_resident_prefill_fallback();
447        }
448        let limit = if max_layers == 0 {
449            n_layer
450        } else {
451            max_layers.min(n_layer)
452        };
453        for layer in 0..limit {
454            if let Err(stage) = self.dispatch_prefill_layer_batch(
455                index,
456                layer,
457                batch_hidden,
458                emb_dim,
459                n_tokens,
460                batch_start_token_idx,
461                scratch_a,
462                scratch_b,
463            ) {
464                return Err(PrefillDispatchFailure::Layer { layer, stage });
465            }
466        }
467        Ok(())
468    }
469
470    /// One transformer block using real mmap tensor offsets (stack buffers only).
471    pub fn dispatch_transformer_layer(
472        &mut self,
473        index: &crate::gguf_sharder::GgufTensorIndex,
474        layer: u32,
475        token_idx: u32,
476        hidden: &mut [f32],
477        emb_dim: usize,
478        scratch_a: &mut [f32],
479        scratch_b: &mut [f32],
480    ) -> bool {
481        #[cfg(not(target_arch = "wasm32"))]
482        if self.gemma4_required {
483            // Hidden-only execution lacks the token required by Gemma PLE.
484            return false;
485        }
486        #[cfg(not(target_arch = "wasm32"))]
487        if self.qwen35_gguf_required {
488            return self.dense_qwen_layer(
489                index, layer, token_idx, hidden, emb_dim, scratch_a, scratch_b,
490            );
491        }
492        if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
493            #[cfg(not(target_arch = "wasm32"))]
494            {
495                let Some(plan) = self.qwen35_plan(index) else {
496                    return false;
497                };
498                if emb_dim != plan.profile.hidden
499                    || hidden.len() != emb_dim
500                    || layer as usize >= plan.layers.len()
501                {
502                    return false;
503                }
504                let Some(session) = &mut self.qwen35_ftw else {
505                    return false;
506                };
507                return session.position() == Ok(token_idx as usize)
508                    && session.forward(layer as usize..layer as usize + 1, hidden) == Ok(1);
509            }
510            #[cfg(target_arch = "wasm32")]
511            return false;
512        }
513        if index.hyperparams.architecture == etalon_model::inference::ARCH_GEMMA4 {
514            return false;
515        }
516        etalon_inference::ordinary::HybridEngine { host: self }.layer(
517            index, layer, token_idx, hidden, emb_dim, scratch_a, scratch_b,
518        )
519    }
520
521    /// Attempt the CUDA mega-pass: all layers in one fenced CUDA stream.
522    /// Returns `Some(token_id)` on success, `None` to fall back to per-layer path.
523    #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
524    pub fn try_cuda_mega_pass_decode(
525        &mut self,
526        index: &crate::gguf_sharder::GgufTensorIndex,
527        hidden: &mut [f32],
528        emb_dim: usize,
529        token_idx: u32,
530    ) -> Option<u32> {
531        if self.qwen35_gguf_required || self.gemma4_required {
532            return None;
533        }
534        self.try_prepared_cuda_decode(index, hidden, emb_dim, token_idx)
535    }
536
537    /// Decode directly from a token id using the resident Q8 embedding table.
538    ///
539    /// This path avoids CPU embedding dequantization and the full hidden-state H2D upload.
540    #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
541    pub fn try_cuda_mega_pass_decode_token(
542        &mut self,
543        index: &crate::gguf_sharder::GgufTensorIndex,
544        token_id: u32,
545        hidden: &mut [f32],
546        emb_dim: usize,
547        token_idx: u32,
548    ) -> Option<u32> {
549        if self.qwen35_gguf_required || self.gemma4_required {
550            return None;
551        }
552        self.try_prepared_cuda_decode_token(index, token_id, hidden, emb_dim, token_idx)
553    }
554
555    /// Superseded unprepared implementation retained temporarily for line-by-line parity review
556    /// during R9.4 decomposition. It is excluded from compilation and will be removed after the
557    /// prepared-plan differential test is certified.
558    #[cfg(any())]
559    pub fn try_cuda_mega_pass_decode_unprepared_reference(
560        &self,
561        index: &crate::gguf_sharder::GgufTensorIndex,
562        hidden: &mut [f32],
563        emb_dim: usize,
564        token_idx: u32,
565    ) -> Option<u32> {
566        use super::RMS_NORM_EPS;
567        use crate::ggml_quants::{GGML_TYPE_Q4_K_SOA, fetch_tensor_bytes};
568        use crate::gguf_bridge::cpu_ops::dequant_norm_row_into;
569        use crate::inference::cuda_lane::{MegaPassLayerDims, MegaPassLayerWeights};
570
571        let h = index.hyperparams;
572        let n_layer = h.n_layer;
573        let n_embd = h.n_embd as usize;
574        let n_head = h.n_head as usize;
575        let n_kv = h.effective_n_kv_head() as usize;
576        let head_dim = h.head_dim() as usize;
577        if n_layer == 0 || n_embd == 0 || n_embd > 4096 || n_embd != emb_dim {
578            return None;
579        }
580        let layout = self.kv_layout?;
581        if layout.int8 || layout.dict_k > 0 {
582            return None;
583        }
584        let mmap = self.model_bytes()?;
585        let tds = index.tensor_data_start;
586
587        // Phase 1: Collect raw weights, dims, and norm weights.
588        struct LayerRaw<'a> {
589            q_raw: &'a [u8],
590            k_raw: &'a [u8],
591            v_raw: &'a [u8],
592            o_raw: &'a [u8],
593            g_raw: &'a [u8],
594            u_raw: &'a [u8],
595            d_raw: &'a [u8],
596        }
597        let mut layer_raws: Vec<LayerRaw<'_>> = Vec::with_capacity(n_layer as usize);
598        let mut layer_dims: Vec<MegaPassLayerDims> = Vec::with_capacity(n_layer as usize);
599        let mut all_attn_norms: Vec<Vec<f32>> = Vec::with_capacity(n_layer as usize);
600        let mut all_ffn_norms: Vec<Vec<f32>> = Vec::with_capacity(n_layer as usize);
601
602        for l in 0..n_layer {
603            let t = index.get_layer_tensors(l);
604
605            let q_info = t.attn_q.as_ref()?;
606            let k_info = t.attn_k.as_ref()?;
607            let v_info = t.attn_v.as_ref()?;
608            let o_info = t.attn_output.as_ref()?;
609            let g_info = t.ffn_gate.as_ref()?;
610            let u_info = t.ffn_up.as_ref()?;
611            let d_info = t.ffn_down.as_ref()?;
612
613            if q_info.ggml_type != GGML_TYPE_Q4_K_SOA
614                || k_info.ggml_type != GGML_TYPE_Q4_K_SOA
615                || v_info.ggml_type != GGML_TYPE_Q4_K_SOA
616                || o_info.ggml_type != GGML_TYPE_Q4_K_SOA
617                || g_info.ggml_type != GGML_TYPE_Q4_K_SOA
618                || u_info.ggml_type != GGML_TYPE_Q4_K_SOA
619                || d_info.ggml_type != GGML_TYPE_Q4_K_SOA
620            {
621                log::debug!(
622                    "mega_pass_decode|skip|layer{l}|not_soa|q={} k={} v={} o={} g={} u={} d={}",
623                    q_info.ggml_type,
624                    k_info.ggml_type,
625                    v_info.ggml_type,
626                    o_info.ggml_type,
627                    g_info.ggml_type,
628                    u_info.ggml_type,
629                    d_info.ggml_type
630                );
631                return None;
632            }
633
634            let (q_in, q_out) = Self::matmul_dims(q_info);
635            let (k_in, k_out) = Self::matmul_dims(k_info);
636            let (_v_in, _v_out) = Self::matmul_dims(v_info);
637            let (o_in, o_out) = Self::matmul_dims(o_info);
638            let (g_in, g_out) = Self::matmul_dims(g_info);
639            let (u_in, u_out) = Self::matmul_dims(u_info);
640            let (d_in, d_out) = Self::matmul_dims(d_info);
641
642            let q_raw = fetch_tensor_bytes(mmap, tds, q_info).ok()?;
643            let k_raw = fetch_tensor_bytes(mmap, tds, k_info).ok()?;
644            let v_raw = fetch_tensor_bytes(mmap, tds, v_info).ok()?;
645            let o_raw = fetch_tensor_bytes(mmap, tds, o_info).ok()?;
646            let g_raw = fetch_tensor_bytes(mmap, tds, g_info).ok()?;
647            let u_raw = fetch_tensor_bytes(mmap, tds, u_info).ok()?;
648            let d_raw = fetch_tensor_bytes(mmap, tds, d_info).ok()?;
649
650            let mut attn_norm = vec![0.0f32; n_embd];
651            let mut ffn_norm = vec![0.0f32; n_embd];
652            if let Some(an_info) = t.attn_norm.as_ref() {
653                if dequant_norm_row_into(mmap, tds, an_info, &mut attn_norm) < n_embd {
654                    return None;
655                }
656            }
657            if let Some(fn_info) = t.ffn_norm.as_ref() {
658                if dequant_norm_row_into(mmap, tds, fn_info, &mut ffn_norm) < n_embd {
659                    return None;
660                }
661            }
662            all_attn_norms.push(attn_norm);
663            all_ffn_norms.push(ffn_norm);
664            layer_raws.push(LayerRaw {
665                q_raw,
666                k_raw,
667                v_raw,
668                o_raw,
669                g_raw,
670                u_raw,
671                d_raw,
672            });
673            layer_dims.push(MegaPassLayerDims {
674                q_in,
675                q_out,
676                kv_in: k_in,
677                kv_out: k_out,
678                o_in,
679                o_out,
680                gate_in: g_in,
681                gate_out: g_out,
682                up_in: u_in,
683                up_out: u_out,
684                down_in: d_in,
685                down_out: d_out,
686            });
687        }
688
689        // Phase 2: Build layer_weights with references into the now-stable norm vectors.
690        let layer_weights: Vec<MegaPassLayerWeights<'_>> = layer_raws
691            .iter()
692            .zip(all_attn_norms.iter().zip(all_ffn_norms.iter()))
693            .map(|(r, (an, fn_))| MegaPassLayerWeights {
694                attn_norm: &an[..n_embd],
695                q_raw: r.q_raw,
696                k_raw: r.k_raw,
697                v_raw: r.v_raw,
698                o_raw: r.o_raw,
699                ffn_norm: &fn_[..n_embd],
700                gate_raw: r.g_raw,
701                up_raw: r.u_raw,
702                down_raw: r.d_raw,
703            })
704            .collect();
705
706        // Output norm.
707        let mut output_norm_buf = vec![0.0f32; n_embd];
708        let output_norm: Option<&[f32]> = if let Some(on_info) = index.output_norm_info() {
709            if dequant_norm_row_into(mmap, tds, on_info, &mut output_norm_buf) >= n_embd {
710                Some(&output_norm_buf[..n_embd])
711            } else {
712                None
713            }
714        } else {
715            None
716        };
717
718        // LM head.
719        let lm_info = index.logits_projection_info()?;
720        let (lm_in, lm_out) = Self::matmul_dims(lm_info);
721        let lm_raw = fetch_tensor_bytes(mmap, tds, lm_info).ok()?;
722        let lm_head_raw = if lm_info.ggml_type == GGML_TYPE_Q4_K_SOA {
723            Some(lm_raw)
724        } else {
725            None
726        };
727
728        let rope_base = h.effective_rope_freq_base();
729        let rope_scale = h.effective_rope_scale();
730        let rms_eps = RMS_NORM_EPS;
731
732        crate::try_cuda_mega_pass(
733            n_embd,
734            n_head,
735            n_kv,
736            head_dim,
737            n_layer,
738            token_idx,
739            layout.max_context,
740            layout.layer_stride,
741            layout.slot_kv_elems,
742            rope_base,
743            rope_scale,
744            rms_eps,
745            &mut hidden[..n_embd],
746            &layer_weights,
747            &layer_dims,
748            output_norm,
749            lm_head_raw,
750            lm_in,
751            lm_out,
752        )
753    }
754
755    /// Sequential layer-by-layer forward (one tensor payload in VRAM at a time).
756    /// `max_layers`: `0` runs all blocks; otherwise caps how many layers execute.
757    pub fn dispatch_transformer_forward(
758        &mut self,
759        index: &crate::gguf_sharder::GgufTensorIndex,
760        hidden: &mut [f32],
761        emb_dim: usize,
762        scratch_a: &mut [f32],
763        scratch_b: &mut [f32],
764        token_idx: u32,
765        max_layers: u32,
766    ) -> u32 {
767        #[cfg(not(target_arch = "wasm32"))]
768        if self.gemma4_required {
769            return 0;
770        }
771        #[cfg(not(target_arch = "wasm32"))]
772        if self.qwen35_gguf_required {
773            return self.dense_qwen_forward(
774                index, hidden, emb_dim, token_idx, max_layers, scratch_a, scratch_b,
775            );
776        }
777        if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
778            #[cfg(not(target_arch = "wasm32"))]
779            return self.qwen35_forward(index, hidden, emb_dim, token_idx, max_layers);
780            #[cfg(target_arch = "wasm32")]
781            return 0;
782        }
783        let n_layer = index.hyperparams.n_layer;
784        if n_layer == 0 {
785            return 0;
786        }
787        let limit = if max_layers == 0 {
788            n_layer
789        } else {
790            max_layers.min(n_layer)
791        };
792        // #48 diagnostic: localize where the hidden state turns non-finite (gated; runs once).
793        use std::sync::atomic::{AtomicBool, Ordering as DbgOrdering};
794        static FWD_DBG_DONE: AtomicBool = AtomicBool::new(false);
795        let dbg = crate::gguf_bridge::execution_env::var("QUALIA_LLM_DEBUG_DECODE").is_ok()
796            && !FWD_DBG_DONE.swap(true, DbgOrdering::Relaxed);
797        if dbg {
798            let nf = hidden[..emb_dim].iter().filter(|v| !v.is_finite()).count();
799            eprintln!(
800                "[fwd-dbg] post-embed nonfinite={}/{} sample={:?}",
801                nf,
802                emb_dim,
803                &hidden[..emb_dim.min(4)]
804            );
805        }
806        let mut ran = 0u32;
807        for layer in 0..limit {
808            if self.dispatch_transformer_layer(
809                index, layer, token_idx, hidden, emb_dim, scratch_a, scratch_b,
810            ) {
811                ran += 1;
812            }
813            if dbg {
814                let nf = hidden[..emb_dim].iter().filter(|v| !v.is_finite()).count();
815                eprintln!(
816                    "[fwd-dbg] after layer {} nonfinite={}/{} ran={} sample={:?}",
817                    layer,
818                    nf,
819                    emb_dim,
820                    ran,
821                    &hidden[..emb_dim.min(4)]
822                );
823                if nf > 0 {
824                    break;
825                }
826            }
827        }
828        ran
829    }
830
831    /// Phase 2B: async single-layer forward (GPU `map_async`; CPU path unchanged in sync API).
832    #[cfg(target_arch = "wasm32")]
833    pub async fn dispatch_transformer_layer_async(
834        &mut self,
835        index: &crate::gguf_sharder::GgufTensorIndex,
836        layer: u32,
837        token_idx: u32,
838        hidden: &mut [f32],
839        emb_dim: usize,
840        scratch_a: &mut [f32],
841        scratch_b: &mut [f32],
842    ) -> bool {
843        let tensors = index.get_layer_tensors(layer);
844        let mut attn_ok = false;
845
846        if tensors.attn_q.is_some() && tensors.attn_k.is_some() && tensors.attn_v.is_some() {
847            if let Some(n) = self
848                .dispatch_attention_layer_async(
849                    index,
850                    layer,
851                    token_idx,
852                    &hidden[..emb_dim],
853                    emb_dim,
854                    &tensors,
855                    scratch_a,
856                    scratch_b,
857                )
858                .await
859            {
860                add_residual_inplace(&mut hidden[..emb_dim], &scratch_a[..n], n);
861                attn_ok = true;
862            }
863        } else if let Some(info) = tensors.attn_output {
864            let (n_in, n_out) = Self::matmul_dims(&info);
865            if n_in <= emb_dim
866                && self
867                    .dispatch_gemm_into_async(index, &info, &hidden[..n_in], scratch_a, n_in, n_out)
868                    .await
869            {
870                add_residual_inplace(
871                    &mut hidden[..emb_dim],
872                    &scratch_a[..n_out],
873                    emb_dim.min(n_out),
874                );
875                attn_ok = true;
876            }
877        }
878
879        if !attn_ok && tensors.attn_output.is_none() && tensors.ffn_gate.is_none() {
880            return false;
881        }
882
883        self.dispatch_ffn_block_pre_norm_async(
884            index, hidden, emb_dim, &tensors, scratch_a, scratch_b,
885        )
886        .await
887    }
888
889    /// MC8: Q + o_proj + FFN tail (K/V already written for this token).
890    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
891    pub(crate) fn encode_attn_ffn_tail_gpu(
892        &self,
893        pipeline: &mut WasmGpuPipeline,
894        index: &crate::gguf_sharder::GgufTensorIndex,
895        layer: u32,
896        token_idx: u32,
897        emb_dim: usize,
898        tensors: &crate::gguf_sharder::LayerTensors,
899        token_hidden: &etalon::Buffer,
900        attn_input: Option<&etalon::Buffer>,
901        work_aliases_hidden: bool,
902    ) -> bool {
903        let mmap = match self.model_bytes() {
904            Some(m) => m,
905            None => return false,
906        };
907        let h = index.hyperparams;
908        let n_embd = h.n_embd as usize;
909        let layout = match self.kv_layout {
910            Some(l) => l,
911            None => return false,
912        };
913        let work_buf = self.gemm_output_buf.as_ref().unwrap();
914        let aux_buf = self.gemm_aux_buf.as_ref().unwrap();
915        let norm_buf = self.norm_weight_buf.as_ref().unwrap();
916        let q_info = match tensors.attn_q.as_ref() {
917            Some(i) => i,
918            None => return false,
919        };
920        let q_raw =
921            match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, q_info) {
922                Ok(s) => s,
923                Err(_) => return false,
924            };
925        let q_in_buf = if let Some(pre) = attn_input {
926            pre
927        } else if let Some(norm) = tensors.attn_norm.as_ref() {
928            if !self.upload_norm_weights(mmap, index.tensor_data_start, norm, n_embd) {
929                return false;
930            }
931            self.encode_elem(
932                pipeline,
933                ELEM_OP_RMS_NORM,
934                n_embd as u32,
935                1,
936                token_hidden,
937                norm_buf,
938                aux_buf,
939            );
940            aux_buf
941        } else {
942            token_hidden
943        };
944        let ffn_buf = self.gemm_ffn_buf.as_ref().unwrap();
945        let emb_bytes = (emb_dim * 4) as etalon::BufferAddress;
946        let q_dim = (h.n_head * h.head_dim()) as usize;
947        let (mask_words, mask_active, mask_word_count) =
948            Self::attention_kv_mask_for_dispatch(&layout, token_idx, 0);
949        let q_params = Self::attention_gpu_params(
950            &h,
951            &layout,
952            layer,
953            token_idx,
954            q_info,
955            q_raw.len(),
956            0,
957            1,
958            token_idx,
959            mask_active,
960            mask_word_count,
961            0,
962        );
963        let q_off = self.mc8_upload_attn_param(&q_params);
964        if mask_active != 0 {
965            self.gpu_queue().write_buffer(
966                self.attention_mask_buf.as_ref().unwrap(),
967                0,
968                bytemuck::cast_slice(&mask_words),
969            );
970        }
971        if !self.encode_attention_pass_gpu(
972            pipeline,
973            q_in_buf,
974            ffn_buf,
975            n_embd,
976            1,
977            token_idx,
978            &layout,
979            layer,
980            token_idx,
981            &h,
982            q_info,
983            q_raw,
984            0,
985            h.n_head,
986            q_off,
987            Mc8WeightRole::AttnQ,
988        ) {
989            return false;
990        }
991        self.mc8_flush(pipeline);
992        if let Some(out_info) = tensors.attn_output.as_ref() {
993            let (o_in, o_out) = Self::matmul_dims(out_info);
994            let o_raw = match crate::ggml_quants::fetch_tensor_bytes(
995                mmap,
996                index.tensor_data_start,
997                out_info,
998            ) {
999                Ok(s) => s,
1000                Err(_) => return false,
1001            };
1002            if work_aliases_hidden {
1003                pipeline
1004                    .encoder
1005                    .copy_buffer_to_buffer(token_hidden, 0, aux_buf, 0, emb_bytes);
1006                self.mc8_flush(pipeline);
1007            }
1008            if o_in > q_dim
1009                || !self.encode_gemm_bufs(pipeline, out_info, o_raw, o_in, o_out, ffn_buf, work_buf)
1010            {
1011                return false;
1012            }
1013            self.mc8_flush(pipeline);
1014            let attn_residual_base: &etalon::Buffer = if work_aliases_hidden {
1015                aux_buf
1016            } else {
1017                token_hidden
1018            };
1019            // Never use `prefill_scratch_buf` here — it holds batched attn RMSNorm rows.
1020            self.encode_residual_add_gpu(
1021                pipeline,
1022                attn_residual_base,
1023                work_buf,
1024                token_hidden,
1025                ffn_buf,
1026                emb_dim as u32,
1027            );
1028        } else {
1029            self.encode_residual_add_gpu(
1030                pipeline,
1031                token_hidden,
1032                ffn_buf,
1033                token_hidden,
1034                aux_buf,
1035                emb_dim as u32,
1036            );
1037        }
1038        self.mc8_flush(pipeline);
1039        let gate_info = match tensors.ffn_gate.as_ref() {
1040            Some(i) => i,
1041            None => return false,
1042        };
1043        let up_info = match tensors.ffn_up.as_ref() {
1044            Some(i) => i,
1045            None => return false,
1046        };
1047        let down_info = match tensors.ffn_down.as_ref() {
1048            Some(i) => i,
1049            None => return false,
1050        };
1051        let (gate_in, n_ffn) = Self::matmul_dims(gate_info);
1052        let (up_in, up_out) = Self::matmul_dims(up_info);
1053        let (dn_in, dn_out) = Self::matmul_dims(down_info);
1054        if gate_in > n_embd
1055            || up_in != gate_in
1056            || up_out != n_ffn
1057            || dn_in != n_ffn
1058            || dn_out < n_embd
1059        {
1060            return false;
1061        }
1062        let gate_raw = match crate::ggml_quants::fetch_tensor_bytes(
1063            mmap,
1064            index.tensor_data_start,
1065            gate_info,
1066        ) {
1067            Ok(s) => s,
1068            Err(_) => return false,
1069        };
1070        let up_raw =
1071            match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, up_info) {
1072                Ok(s) => s,
1073                Err(_) => return false,
1074            };
1075        let down_raw = match crate::ggml_quants::fetch_tensor_bytes(
1076            mmap,
1077            index.tensor_data_start,
1078            down_info,
1079        ) {
1080            Ok(s) => s,
1081            Err(_) => return false,
1082        };
1083        let base_save = match self.prefill_scratch_buf.as_ref() {
1084            Some(b) => b,
1085            None => return false,
1086        };
1087        pipeline
1088            .encoder
1089            .copy_buffer_to_buffer(token_hidden, 0, base_save, 0, emb_bytes);
1090        self.mc8_flush(pipeline);
1091        if let Some(norm) = tensors.ffn_norm.as_ref() {
1092            if !self.upload_norm_weights(mmap, index.tensor_data_start, norm, n_embd) {
1093                return false;
1094            }
1095            self.encode_elem(
1096                pipeline,
1097                ELEM_OP_RMS_NORM,
1098                n_embd as u32,
1099                1,
1100                token_hidden,
1101                norm_buf,
1102                aux_buf,
1103            );
1104        } else {
1105            pipeline
1106                .encoder
1107                .copy_buffer_to_buffer(token_hidden, 0, aux_buf, 0, emb_bytes);
1108        }
1109        self.mc8_flush(pipeline);
1110        if !self.encode_gemm_bufs(
1111            pipeline, gate_info, gate_raw, gate_in, n_ffn, aux_buf, work_buf,
1112        ) {
1113            return false;
1114        }
1115        self.mc8_flush(pipeline);
1116        if !self.encode_gemm_bufs(pipeline, up_info, up_raw, up_in, n_ffn, aux_buf, ffn_buf) {
1117            return false;
1118        }
1119        self.mc8_flush(pipeline);
1120        self.encode_elem(
1121            pipeline,
1122            ELEM_OP_SILU_MUL,
1123            n_ffn as u32,
1124            1,
1125            work_buf,
1126            ffn_buf,
1127            aux_buf,
1128        );
1129        self.mc8_flush(pipeline);
1130        if !self.encode_gemm_bufs(
1131            pipeline, down_info, down_raw, dn_in, dn_out, aux_buf, work_buf,
1132        ) {
1133            return false;
1134        }
1135        self.mc8_flush(pipeline);
1136        // FFN residual: down output is in work_buf; pre-FFN skip is in base_save.
1137        // Use aux_buf as scratch (SiLU output consumed; down GEMM flushed above).
1138        self.encode_residual_add_gpu(
1139            pipeline,
1140            base_save,
1141            work_buf,
1142            token_hidden,
1143            aux_buf,
1144            emb_dim as u32,
1145        );
1146        self.mc8_flush(pipeline);
1147        true
1148    }
1149
1150    /// MC8: encode one decode layer entirely on GPU (no map_async).
1151    /// Superseded by the Part 3w super-arena decode forward (kept for reference/fallback).
1152    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
1153    #[allow(dead_code)]
1154    pub(crate) fn encode_transformer_layer_gpu(
1155        &self,
1156        pipeline: &mut WasmGpuPipeline,
1157        index: &crate::gguf_sharder::GgufTensorIndex,
1158        layer: u32,
1159        token_idx: u32,
1160        emb_dim: usize,
1161    ) -> bool {
1162        let tensors = index.get_layer_tensors(layer);
1163        let layout = match self.kv_layout {
1164            Some(l) => l,
1165            None => return false,
1166        };
1167        let mmap = match self.model_bytes() {
1168            Some(m) => m,
1169            None => return false,
1170        };
1171        let h = index.hyperparams;
1172        let n_embd = h.n_embd as usize;
1173        if emb_dim < n_embd {
1174            return false;
1175        }
1176        let hidden_buf = self.gemm_input_buf.as_ref().unwrap();
1177        let work_buf = self.gemm_output_buf.as_ref().unwrap();
1178        let aux_buf = self.gemm_aux_buf.as_ref().unwrap();
1179        let norm_buf = self.norm_weight_buf.as_ref().unwrap();
1180
1181        let (k_info, v_info) = match (tensors.attn_k.as_ref(), tensors.attn_v.as_ref()) {
1182            (Some(k), Some(v)) => (k, v),
1183            _ => return false,
1184        };
1185        if tensors.attn_q.is_none() {
1186            return false;
1187        }
1188        let k_raw =
1189            match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info) {
1190                Ok(s) => s,
1191                Err(_) => return false,
1192            };
1193        let v_raw =
1194            match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info) {
1195                Ok(s) => s,
1196                Err(_) => return false,
1197            };
1198
1199        let attn_input = if let Some(norm) = tensors.attn_norm.as_ref() {
1200            if !self.upload_norm_weights(mmap, index.tensor_data_start, norm, n_embd) {
1201                return false;
1202            }
1203            self.encode_elem(
1204                pipeline,
1205                ELEM_OP_RMS_NORM,
1206                n_embd as u32,
1207                1,
1208                hidden_buf,
1209                norm_buf,
1210                aux_buf,
1211            );
1212            self.mc8_flush(pipeline);
1213            aux_buf
1214        } else {
1215            hidden_buf
1216        };
1217
1218        let n_kv = h.effective_n_kv_head();
1219        let mut attn_arena = Mc8UniformArena {
1220            bytes: [0u8; MC8_MAX_GEMM_UNIFORM_SLOTS * MC8_UNIFORM_ALIGN],
1221            slots: 0,
1222        };
1223        let k_params = Self::attention_gpu_params(
1224            &h,
1225            &layout,
1226            layer,
1227            token_idx,
1228            k_info,
1229            k_raw.len(),
1230            1,
1231            1,
1232            token_idx,
1233            0,
1234            0,
1235            0,
1236        );
1237        let v_params = Self::attention_gpu_params(
1238            &h,
1239            &layout,
1240            layer,
1241            token_idx,
1242            v_info,
1243            v_raw.len(),
1244            2,
1245            1,
1246            token_idx,
1247            0,
1248            0,
1249            0,
1250        );
1251        let k_off = attn_arena.push(&k_params);
1252        let v_off = attn_arena.push(&v_params);
1253        attn_arena.upload(
1254            self.gpu_queue(),
1255            self.attention_params_buf.as_ref().unwrap(),
1256        );
1257        if !self.encode_attention_pass_gpu(
1258            pipeline,
1259            attn_input,
1260            work_buf,
1261            n_embd,
1262            1,
1263            token_idx,
1264            &layout,
1265            layer,
1266            token_idx,
1267            &h,
1268            k_info,
1269            k_raw,
1270            1,
1271            n_kv,
1272            k_off,
1273            Mc8WeightRole::AttnK,
1274        ) {
1275            return false;
1276        }
1277        if !self.encode_attention_pass_gpu(
1278            pipeline,
1279            attn_input,
1280            work_buf,
1281            n_embd,
1282            1,
1283            token_idx,
1284            &layout,
1285            layer,
1286            token_idx,
1287            &h,
1288            v_info,
1289            v_raw,
1290            2,
1291            n_kv,
1292            v_off,
1293            Mc8WeightRole::AttnV,
1294        ) {
1295            return false;
1296        }
1297        self.mc8_flush(pipeline);
1298        self.encode_attn_ffn_tail_gpu(
1299            pipeline,
1300            index,
1301            layer,
1302            token_idx,
1303            emb_dim,
1304            &tensors,
1305            hidden_buf,
1306            Some(attn_input),
1307            false,
1308        )
1309    }
1310
1311    /// MC8 Part 3w: decode forward via the prefill super-arena (n_tokens=1).
1312    /// Reuses `mc8_stage_prefill_layer_super_arena` + `encode_prefill_q_ffn_tail_fused`
1313    /// (dynamic-offset uniforms + 7 disjoint weight buffers) → **2 submits/layer**
1314    /// (KV-visibility flush + layer-end), down from the legacy 13 flushes/layer.
1315    /// A single decode token at absolute position `token_idx` is a 1-row prefill chunk:
1316    /// dense causal Q (`mask_active=0`, `logical <= abs_pos`) is correct for decode.
1317    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
1318    pub async fn dispatch_transformer_forward_async(
1319        &mut self,
1320        index: &crate::gguf_sharder::GgufTensorIndex,
1321        hidden: &mut [f32],
1322        emb_dim: usize,
1323        _scratch_a: &mut [f32],
1324        _scratch_b: &mut [f32],
1325        token_idx: u32,
1326        max_layers: u32,
1327    ) -> u32 {
1328        let n_layer = index.hyperparams.n_layer;
1329        if n_layer == 0 || !self.mc8_buffers_ready() {
1330            return 0;
1331        }
1332        if self.prefill_work_buf_a.is_none() || self.prefill_work_buf_b.is_none() {
1333            wlog("[MC8] decode forward: prefill work buffers missing — cannot run super-arena");
1334            return 0;
1335        }
1336        // Part 3x: upload all layer weights to GPU once (idempotent; falls back if it fails).
1337        if !self.mc8_weights_resident {
1338            let _ = self.mc8_upload_all_resident_weights(index);
1339        }
1340        let limit = if max_layers == 0 {
1341            n_layer
1342        } else {
1343            max_layers.min(n_layer)
1344        };
1345        let n_embd = index.hyperparams.n_embd as usize;
1346        if emb_dim < n_embd || n_embd > hidden.len() || n_embd > self.gemm_max_input_floats {
1347            return 0;
1348        }
1349        let prefill_scratch = match self.prefill_scratch_buf.as_ref() {
1350            Some(b) => b,
1351            None => return 0,
1352        };
1353        let batch_buf = self.gemm_input_buf.as_ref().unwrap();
1354        let token_buf = self.gemm_output_buf.as_ref().unwrap();
1355        if self.norm_weight_buf.is_none() {
1356            return 0;
1357        }
1358        self.gpu_queue()
1359            .write_buffer(batch_buf, 0, bytemuck::cast_slice(&hidden[..n_embd]));
1360        let mmap = match self.model_bytes() {
1361            Some(m) => m,
1362            None => return 0,
1363        };
1364        let layout = match self.kv_layout {
1365            Some(l) => l,
1366            None => return 0,
1367        };
1368        let n_tokens = 1u32;
1369        let mut ran = 0u32;
1370        // Phase 5.4 single-submit: one encoder + monotonic uniform cursors across the whole forward;
1371        // flush only at MC8_LAYERS_PER_ENCODER chunk boundaries → 1 submit for ≤64-layer models (vs
1372        // the old 2/layer). Per-layer write_buffer races are gone (resident norms + accumulating
1373        // cursors), so KV/work-buffer visibility relies on WebGPU intra-encoder barriers.
1374        let mut layer_uniform_cursors = Mc8ChunkUniformCursors {
1375            attn: 0,
1376            elem: 0,
1377            gemm: 0,
1378        };
1379        let mut enc = WasmGpuPipeline::begin(self);
1380        for layer in 0..limit {
1381            if layer > 0 && (layer % MC8_LAYERS_PER_ENCODER) == 0 {
1382                self.mc8_flush(&mut enc);
1383                layer_uniform_cursors.reset();
1384            }
1385            let tensors = index.get_layer_tensors(layer);
1386            let k_info = match tensors.attn_k.as_ref() {
1387                Some(i) => i,
1388                None => break,
1389            };
1390            let v_info = match tensors.attn_v.as_ref() {
1391                Some(i) => i,
1392                None => break,
1393            };
1394            let k_raw =
1395                match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info)
1396                {
1397                    Ok(s) => s,
1398                    Err(_) => break,
1399                };
1400            let v_raw =
1401                match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info)
1402                {
1403                    Ok(s) => s,
1404                    Err(_) => break,
1405                };
1406            let h = index.hyperparams;
1407            let n_kv = h.effective_n_kv_head();
1408            let used_attn_norm = tensors.attn_norm.is_some();
1409            let (uniforms, geom) = match self.mc8_stage_prefill_layer_super_arena(
1410                index,
1411                layer,
1412                &tensors,
1413                token_idx,
1414                n_tokens,
1415                emb_dim,
1416                used_attn_norm,
1417                k_info,
1418                &k_raw,
1419                v_info,
1420                &v_raw,
1421                &mut layer_uniform_cursors,
1422            ) {
1423                Some(v) => v,
1424                None => break,
1425            };
1426            let attn_src = if used_attn_norm {
1427                if let (Some(norm), Some(off)) =
1428                    (tensors.attn_norm.as_ref(), uniforms.attn_norm_elem_off)
1429                {
1430                    let (norm_b, norm_b_off) = match self.mc8_norm_source(
1431                        mmap,
1432                        index.tensor_data_start,
1433                        norm,
1434                        n_embd,
1435                        layer,
1436                        false,
1437                    ) {
1438                        Some(v) => v,
1439                        None => break,
1440                    };
1441                    self.encode_elem_offset(
1442                        &mut enc,
1443                        ELEM_OP_RMS_NORM,
1444                        n_embd as u32,
1445                        n_tokens,
1446                        batch_buf,
1447                        0,
1448                        geom.batch_in_bytes,
1449                        norm_b,
1450                        norm_b_off,
1451                        geom.n_embd_bytes,
1452                        prefill_scratch,
1453                        0,
1454                        geom.batch_in_bytes,
1455                        0,
1456                        0,
1457                        0,
1458                        0,
1459                        0,
1460                        0,
1461                        off,
1462                    );
1463                }
1464                prefill_scratch
1465            } else {
1466                batch_buf
1467            };
1468            let n_kv_wg = n_tokens.saturating_mul(n_kv);
1469            // Phase 5.5: K/V projection on the parallel GEMM → proj buffers; the (now lightweight)
1470            // attention shader reads the pre-computed projection instead of matmul-ing on 1 thread.
1471            let kv_dim = (n_kv * h.head_dim()) as usize;
1472            let kv_proj_bytes = (kv_dim * n_tokens as usize * 4) as etalon::BufferAddress;
1473            let k_proj = self.mc8_k_proj_buf.as_ref().unwrap();
1474            let v_proj = self.mc8_v_proj_buf.as_ref().unwrap();
1475            if !self.encode_gemm_bufs_offset(
1476                &mut enc,
1477                k_info,
1478                k_raw,
1479                n_embd,
1480                kv_dim,
1481                attn_src,
1482                0,
1483                geom.batch_in_bytes,
1484                k_proj,
1485                0,
1486                kv_proj_bytes,
1487                n_tokens,
1488                n_embd as u32,
1489                kv_dim as u32,
1490                uniforms.off_k_gemm,
1491                layer,
1492                Mc8WeightRole::AttnK,
1493            ) {
1494                break;
1495            }
1496            if !self.encode_gemm_bufs_offset(
1497                &mut enc,
1498                v_info,
1499                v_raw,
1500                n_embd,
1501                kv_dim,
1502                attn_src,
1503                0,
1504                geom.batch_in_bytes,
1505                v_proj,
1506                0,
1507                kv_proj_bytes,
1508                n_tokens,
1509                n_embd as u32,
1510                kv_dim as u32,
1511                uniforms.off_v_gemm,
1512                layer,
1513                Mc8WeightRole::AttnV,
1514            ) {
1515                break;
1516            }
1517            if !self.encode_attention_pass_gpu(
1518                &mut enc,
1519                k_proj,
1520                token_buf,
1521                n_embd,
1522                n_tokens,
1523                token_idx,
1524                &layout,
1525                layer,
1526                token_idx,
1527                &h,
1528                k_info,
1529                k_raw,
1530                1,
1531                n_kv_wg,
1532                uniforms.k_off,
1533                Mc8WeightRole::AttnK,
1534            ) {
1535                break;
1536            }
1537            if !self.encode_attention_pass_gpu(
1538                &mut enc,
1539                v_proj,
1540                token_buf,
1541                n_embd,
1542                n_tokens,
1543                token_idx,
1544                &layout,
1545                layer,
1546                token_idx,
1547                &h,
1548                v_info,
1549                v_raw,
1550                2,
1551                n_kv_wg,
1552                uniforms.v_off,
1553                Mc8WeightRole::AttnV,
1554            ) {
1555                break;
1556            }
1557            // Phase 5.4: NO per-layer flush. KV-cache + work-buffer visibility now relies on WebGPU's
1558            // automatic intra-encoder barriers between compute passes (the per-layer write_buffer
1559            // races that previously forced a flush are gone: resident norms + accumulating uniforms).
1560            let work_a = self.prefill_work_buf_a.as_ref().unwrap();
1561            let work_b = self.prefill_work_buf_b.as_ref().unwrap();
1562            if !self.encode_prefill_q_ffn_tail_fused(
1563                &mut enc,
1564                index,
1565                layer,
1566                &tensors,
1567                batch_buf,
1568                attn_src,
1569                work_a,
1570                work_b,
1571                n_tokens,
1572                token_idx,
1573                emb_dim,
1574                used_attn_norm,
1575                &uniforms,
1576                &geom,
1577            ) {
1578                break;
1579            }
1580            ran += 1;
1581        }
1582        // Phase 5.4: ONE submit for the whole forward (or per chunk if n_layer > MC8_LAYERS_PER_ENCODER).
1583        self.mc8_flush(&mut enc);
1584        if ran > 0 && !self.pipeline_read_hidden(emb_dim, hidden).await {
1585            return 0;
1586        }
1587        ran
1588    }
1589
1590    /// Fused forward + output norm + argmax in a single encoder/submit/readback.
1591    /// Eliminates 1 of 2 GPU round-trips per token vs the separate forward+argmax path.
1592    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
1593    pub async fn dispatch_forward_and_argmax_fused_async(
1594        &mut self,
1595        index: &crate::gguf_sharder::GgufTensorIndex,
1596        hidden: &mut [f32],
1597        emb_dim: usize,
1598        token_idx: u32,
1599        max_layers: u32,
1600        _chunk_logits: &mut [f32],
1601        max_chunks: u32,
1602    ) -> Option<StreamingArgmaxResult> {
1603        let n_layer = index.hyperparams.n_layer;
1604        if n_layer == 0 || !self.mc8_buffers_ready() {
1605            return None;
1606        }
1607        if self.prefill_work_buf_a.is_none() || self.prefill_work_buf_b.is_none() {
1608            return None;
1609        }
1610        if !self.mc8_weights_resident {
1611            let _ = self.mc8_upload_all_resident_weights(index);
1612        }
1613        let limit = if max_layers == 0 {
1614            n_layer
1615        } else {
1616            max_layers.min(n_layer)
1617        };
1618        let n_embd = index.hyperparams.n_embd as usize;
1619        if emb_dim < n_embd || n_embd > hidden.len() || n_embd > self.gemm_max_input_floats {
1620            return None;
1621        }
1622        let prefill_scratch = self.prefill_scratch_buf.as_ref()?;
1623        let batch_buf = self.gemm_input_buf.as_ref().unwrap();
1624        let norm_buf = self.norm_weight_buf.as_ref().unwrap();
1625        let mmap = self.model_bytes()?;
1626        let layout = self.kv_layout?;
1627
1628        // Upload hidden to batch_buf
1629        self.gpu_queue()
1630            .write_buffer(batch_buf, 0, bytemuck::cast_slice(&hidden[..n_embd]));
1631
1632        // === Forward pass (same as dispatch_transformer_forward_async but without flush) ===
1633        let n_tokens = 1u32;
1634        let mut ran = 0u32;
1635        let mut layer_uniform_cursors = Mc8ChunkUniformCursors {
1636            attn: 0,
1637            elem: 0,
1638            gemm: 0,
1639        };
1640        let mut enc = WasmGpuPipeline::begin(self);
1641        for layer in 0..limit {
1642            if layer > 0 && (layer % MC8_LAYERS_PER_ENCODER) == 0 {
1643                self.mc8_flush(&mut enc);
1644                layer_uniform_cursors.reset();
1645            }
1646            let tensors = index.get_layer_tensors(layer);
1647            let k_info = match tensors.attn_k.as_ref() {
1648                Some(i) => i,
1649                None => break,
1650            };
1651            let v_info = match tensors.attn_v.as_ref() {
1652                Some(i) => i,
1653                None => break,
1654            };
1655            let k_raw =
1656                match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, k_info)
1657                {
1658                    Ok(s) => s,
1659                    Err(_) => break,
1660                };
1661            let v_raw =
1662                match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, v_info)
1663                {
1664                    Ok(s) => s,
1665                    Err(_) => break,
1666                };
1667            let h = index.hyperparams;
1668            let n_kv = h.effective_n_kv_head();
1669            let used_attn_norm = tensors.attn_norm.is_some();
1670            let (uniforms, geom) = match self.mc8_stage_prefill_layer_super_arena(
1671                index,
1672                layer,
1673                &tensors,
1674                token_idx,
1675                n_tokens,
1676                emb_dim,
1677                used_attn_norm,
1678                k_info,
1679                &k_raw,
1680                v_info,
1681                &v_raw,
1682                &mut layer_uniform_cursors,
1683            ) {
1684                Some(v) => v,
1685                None => break,
1686            };
1687            let attn_src = if used_attn_norm {
1688                if let (Some(norm), Some(off)) =
1689                    (tensors.attn_norm.as_ref(), uniforms.attn_norm_elem_off)
1690                {
1691                    let (norm_b, norm_b_off) = match self.mc8_norm_source(
1692                        mmap,
1693                        index.tensor_data_start,
1694                        norm,
1695                        n_embd,
1696                        layer,
1697                        false,
1698                    ) {
1699                        Some(v) => v,
1700                        None => break,
1701                    };
1702                    self.encode_elem_offset(
1703                        &mut enc,
1704                        ELEM_OP_RMS_NORM,
1705                        n_embd as u32,
1706                        n_tokens,
1707                        batch_buf,
1708                        0,
1709                        geom.batch_in_bytes,
1710                        norm_b,
1711                        norm_b_off,
1712                        geom.n_embd_bytes,
1713                        prefill_scratch,
1714                        0,
1715                        geom.batch_in_bytes,
1716                        0,
1717                        0,
1718                        0,
1719                        0,
1720                        0,
1721                        0,
1722                        off,
1723                    );
1724                }
1725                prefill_scratch
1726            } else {
1727                batch_buf
1728            };
1729            let n_kv_wg = n_tokens.saturating_mul(n_kv);
1730            let kv_dim = (n_kv * h.head_dim()) as usize;
1731            let kv_proj_bytes = (kv_dim * n_tokens as usize * 4) as etalon::BufferAddress;
1732            let k_proj = self.mc8_k_proj_buf.as_ref().unwrap();
1733            let v_proj = self.mc8_v_proj_buf.as_ref().unwrap();
1734            if !self.encode_gemm_bufs_offset(
1735                &mut enc,
1736                k_info,
1737                k_raw,
1738                n_embd,
1739                kv_dim,
1740                attn_src,
1741                0,
1742                geom.batch_in_bytes,
1743                k_proj,
1744                0,
1745                kv_proj_bytes,
1746                n_tokens,
1747                n_embd as u32,
1748                kv_dim as u32,
1749                uniforms.off_k_gemm,
1750                layer,
1751                Mc8WeightRole::AttnK,
1752            ) {
1753                break;
1754            }
1755            if !self.encode_gemm_bufs_offset(
1756                &mut enc,
1757                v_info,
1758                v_raw,
1759                n_embd,
1760                kv_dim,
1761                attn_src,
1762                0,
1763                geom.batch_in_bytes,
1764                v_proj,
1765                0,
1766                kv_proj_bytes,
1767                n_tokens,
1768                n_embd as u32,
1769                kv_dim as u32,
1770                uniforms.off_v_gemm,
1771                layer,
1772                Mc8WeightRole::AttnV,
1773            ) {
1774                break;
1775            }
1776            if !self.encode_attention_pass_gpu(
1777                &mut enc,
1778                k_proj,
1779                self.gemm_output_buf.as_ref().unwrap(),
1780                n_embd,
1781                n_tokens,
1782                token_idx,
1783                &layout,
1784                layer,
1785                token_idx,
1786                &h,
1787                k_info,
1788                k_raw,
1789                1,
1790                n_kv_wg,
1791                uniforms.k_off,
1792                Mc8WeightRole::AttnK,
1793            ) {
1794                break;
1795            }
1796            if !self.encode_attention_pass_gpu(
1797                &mut enc,
1798                v_proj,
1799                self.gemm_output_buf.as_ref().unwrap(),
1800                n_embd,
1801                n_tokens,
1802                token_idx,
1803                &layout,
1804                layer,
1805                token_idx,
1806                &h,
1807                v_info,
1808                v_raw,
1809                2,
1810                n_kv_wg,
1811                uniforms.v_off,
1812                Mc8WeightRole::AttnV,
1813            ) {
1814                break;
1815            }
1816            let work_a = self.prefill_work_buf_a.as_ref().unwrap();
1817            let work_b = self.prefill_work_buf_b.as_ref().unwrap();
1818            if !self.encode_prefill_q_ffn_tail_fused(
1819                &mut enc,
1820                index,
1821                layer,
1822                &tensors,
1823                batch_buf,
1824                attn_src,
1825                work_a,
1826                work_b,
1827                n_tokens,
1828                token_idx,
1829                emb_dim,
1830                used_attn_norm,
1831                &uniforms,
1832                &geom,
1833            ) {
1834                break;
1835            }
1836            ran += 1;
1837        }
1838        if ran == 0 {
1839            self.mc8_flush(&mut enc);
1840            return None;
1841        }
1842
1843        // Flush the transformer layers to the GPU now. This guarantees all layer
1844        // GEMMs execute with their staged parameters, avoids uniform buffer overwrite
1845        // hazards, and ensures the hidden state in batch_buf is fully computed.
1846        self.mc8_flush(&mut enc);
1847
1848        // === Output norm on GPU ===
1849        let output_norm_info = match index.output_norm_info() {
1850            Some(i) => i,
1851            None => {
1852                // No output norm — just read hidden and fall back to CPU argmax
1853                if !self.pipeline_read_hidden(emb_dim, hidden).await {
1854                    return None;
1855                }
1856                return None; // caller will use separate argmax
1857            }
1858        };
1859        // Upload output norm weights to norm_weight_buf
1860        let mut norm_w = [0f32; MAX_HIDDEN_DIM];
1861        if dequant_norm_row_into(mmap, index.tensor_data_start, output_norm_info, &mut norm_w)
1862            < n_embd
1863        {
1864            return None;
1865        }
1866        self.gpu_queue()
1867            .write_buffer(norm_buf, 0, bytemuck::cast_slice(&norm_w[..n_embd]));
1868        // RMS norm: read from batch_buf, write to prefill_scratch
1869        let n_embd_bytes = (n_embd * 4) as etalon::BufferAddress;
1870        self.encode_elem_offset(
1871            &mut enc,
1872            ELEM_OP_RMS_NORM,
1873            n_embd as u32,
1874            1,
1875            batch_buf,
1876            0,
1877            n_embd_bytes,
1878            norm_buf,
1879            0,
1880            n_embd_bytes,
1881            prefill_scratch,
1882            0,
1883            n_embd_bytes,
1884            0,
1885            0,
1886            0,
1887            0,
1888            0,
1889            0,
1890            0,
1891        );
1892        // Copy normed hidden back to batch_buf for argmax GEMM input
1893        enc.encoder
1894            .copy_buffer_to_buffer(prefill_scratch, 0, batch_buf, 0, n_embd_bytes);
1895        // Flush the output norm pass so batch_buf contains the normalized hidden state
1896        // before the logits projection GEMM pass reads it.
1897        self.mc8_flush(&mut enc);
1898
1899        // === Batched argmax in fresh encoder ===
1900        let resident_buf = self.mc8_logits_resident_buf.as_ref()?;
1901        let row_bytes = self.mc8_logits_row_bytes as u64;
1902        let output_buf = self.gemm_output_buf.as_ref().unwrap();
1903        let params_buf = self.gemm_params_buf.as_ref().unwrap();
1904        let staging = self.gemm_output_staging.as_ref().unwrap();
1905        let logits_info = index.logits_projection_info()?;
1906        let (n_in, vocab_size) = Self::matmul_dims(logits_info);
1907        if n_in == 0 || vocab_size == 0 || n_in > emb_dim {
1908            self.mc8_flush(&mut enc);
1909            return None;
1910        }
1911        let full_chunks = vocab_size.div_ceil(VOCAB_CHUNK_ROWS);
1912        let n_chunks = if max_chunks == 0 {
1913            full_chunks
1914        } else {
1915            (max_chunks as usize).min(full_chunks)
1916        };
1917        let top1_plan =
1918            crate::gguf_bridge::browser::webgpu::BrowserTop1Plan::new(vocab_size, n_chunks as u32)?;
1919        if !self.prepare_browser_top1(top1_plan) {
1920            self.mc8_flush(&mut enc);
1921            return None;
1922        }
1923        let params = GemmGpuParams {
1924            n_in: n_in as u32,
1925            n_out: VOCAB_CHUNK_ROWS as u32,
1926            weight_ggml_type: logits_info.ggml_type,
1927            weight_row_elems: logits_info.dims[0] as u32,
1928            weight_byte_len: (VOCAB_CHUNK_ROWS as u64 * row_bytes) as u32,
1929            n_batch: 1,
1930            in_row_stride: 0,
1931            out_row_stride: 0,
1932        };
1933        self.gpu_queue()
1934            .write_buffer(params_buf, 0, bytemuck::bytes_of(&params));
1935        #[cfg(target_arch = "wasm32")]
1936        let use_mmv_q8_0 =
1937            logits_info.ggml_type == crate::ggml_quants::GGML_TYPE_Q8_0 && (n_in % 32 == 0);
1938        #[cfg(target_arch = "wasm32")]
1939        let logits_pipeline: &etalon::ComputePipeline = if use_mmv_q8_0 {
1940            &self.mmv_q8_0_pipeline
1941        } else {
1942            &self.pipeline
1943        };
1944        #[cfg(not(target_arch = "wasm32"))]
1945        let logits_pipeline: &etalon::ComputePipeline = &self.pipeline;
1946        let bind_layout = logits_pipeline.get_bind_group_layout(0);
1947        for chunk_idx in 0..n_chunks {
1948            let row_start = chunk_idx * VOCAB_CHUNK_ROWS;
1949            let chunk_rows = VOCAB_CHUNK_ROWS.min(vocab_size - row_start);
1950            let weight = etalon::BindingResource::Buffer(etalon::BufferBinding {
1951                buffer: resident_buf,
1952                offset: row_start as u64 * row_bytes,
1953                size: std::num::NonZeroU64::new(chunk_rows as u64 * row_bytes),
1954            });
1955            let key = mc8_bg_hash(&[
1956                7,
1957                chunk_idx as u64,
1958                row_start as u64,
1959                row_bytes as u64,
1960                chunk_rows as u64,
1961                resident_buf as *const _ as u64,
1962                batch_buf as *const _ as u64,
1963                output_buf as *const _ as u64,
1964            ]);
1965            let bind_group = {
1966                let cached = self
1967                    .mc8_bg_cache
1968                    .lock()
1969                    .ok()
1970                    .and_then(|c| c.get(&key).cloned());
1971                match cached {
1972                    Some(bg) => bg,
1973                    None => {
1974                        let bg =
1975                            self.gpu_device()
1976                                .create_bind_group(&etalon::BindGroupDescriptor {
1977                                    label: Some("FusedArgmaxBind"),
1978                                    layout: &bind_layout,
1979                                    entries: &[
1980                                        etalon::BindGroupEntry {
1981                                            binding: 0,
1982                                            resource: batch_buf.as_entire_binding(),
1983                                        },
1984                                        etalon::BindGroupEntry {
1985                                            binding: 1,
1986                                            resource: weight,
1987                                        },
1988                                        etalon::BindGroupEntry {
1989                                            binding: 2,
1990                                            resource: Self::mc8_dynamic_uniform_binding(params_buf),
1991                                        },
1992                                        etalon::BindGroupEntry {
1993                                            binding: 3,
1994                                            resource: output_buf.as_entire_binding(),
1995                                        },
1996                                    ],
1997                                });
1998                        if let Ok(mut c) = self.mc8_bg_cache.lock() {
1999                            c.insert(key, bg.clone());
2000                        }
2001                        bg
2002                    }
2003                }
2004            };
2005            {
2006                let mut cpass = enc
2007                    .encoder
2008                    .begin_compute_pass(&etalon::ComputePassDescriptor {
2009                        label: None,
2010                        timestamp_writes: None,
2011                    });
2012                cpass.set_pipeline(logits_pipeline);
2013                cpass.set_bind_group(0, &bind_group, &[0]);
2014                #[cfg(target_arch = "wasm32")]
2015                if use_mmv_q8_0 {
2016                    cpass.dispatch_workgroups((chunk_rows as u32 + 3) / 4, 1, 1);
2017                } else {
2018                    cpass.dispatch_workgroups((chunk_rows as u32 + 63) / 64, 1, 1);
2019                }
2020                #[cfg(not(target_arch = "wasm32"))]
2021                cpass.dispatch_workgroups((chunk_rows as u32 + 63) / 64, 1, 1);
2022            }
2023            if !self.encode_browser_top1_chunk(
2024                &mut enc.encoder,
2025                output_buf,
2026                staging,
2027                top1_plan,
2028                chunk_idx,
2029            ) {
2030                self.mc8_flush(&mut enc);
2031                return None;
2032            }
2033        }
2034
2035        // One submit/fence. Logits remain on-device; only bounded block
2036        // candidates are mapped for the deterministic global merge.
2037        self.submit_commands(Some(enc.encoder.finish()));
2038        self.read_browser_top1(staging, top1_plan).await
2039    }
2040
2041    /// Topological speculative verify — accept longest draft prefix (B3.1d).
2042    pub fn verify_topology_draft_batch(
2043        &mut self,
2044        index: &crate::gguf_sharder::GgufTensorIndex,
2045        ctx: &mut Vec<u32>,
2046        draft: &crate::compute_universe::TopologyDraftBatch,
2047        emb_dim: usize,
2048        emb_buf: &mut [f32],
2049        scratch_a: &mut [f32],
2050        scratch_b: &mut [f32],
2051        max_layers: u32,
2052        max_vocab_chunks: u32,
2053    ) -> u32 {
2054        #[cfg(not(target_arch = "wasm32"))]
2055        if self.gemma4_required || self.qwen35_gguf_required {
2056            // This legacy speculative path lacks architecture-owned state rollback.
2057            return 0;
2058        }
2059        let mmap = match self.clone_model_bytes() {
2060            Some(m) => m,
2061            None => return 0,
2062        };
2063        let gamma = draft.draft_len as usize;
2064        if gamma == 0 || ctx.is_empty() {
2065            return 0;
2066        }
2067        let mut accepted = 0u32;
2068        for i in 0..gamma {
2069            let cur = *ctx.last().unwrap();
2070            let token_idx = ctx.len().saturating_sub(1) as u32;
2071            let hidden_ok =
2072                index.dequantize_token_embedding_into(mmap.as_ref(), cur, &mut emb_buf[..emb_dim]);
2073            if hidden_ok == 0 {
2074                break;
2075            }
2076            let _ = self.dispatch_transformer_forward(
2077                index,
2078                &mut emb_buf[..emb_dim],
2079                emb_dim,
2080                scratch_a,
2081                scratch_b,
2082                token_idx,
2083                max_layers,
2084            );
2085            let pred = if let Some(argmax) = self.dispatch_output_argmax_chunked(
2086                index,
2087                &emb_buf[..emb_dim],
2088                emb_dim,
2089                scratch_a,
2090                max_vocab_chunks,
2091                None,
2092            ) {
2093                if argmax.max_logit > f32::NEG_INFINITY {
2094                    argmax.best_token_id
2095                } else {
2096                    break;
2097                }
2098            } else {
2099                break;
2100            };
2101            if pred != draft.draft_ids[i] {
2102                break;
2103            }
2104            ctx.push(pred);
2105            accepted += 1;
2106        }
2107        accepted
2108    }
2109}