Skip to main content

qualia_core_db/gguf_bridge/
async_dispatch.rs

1//! Decode-tail dispatch: final logits projection + async GEMM / async attention paths.
2//! Split from gguf_bridge/forward.rs (structural; no behaviour change).
3#![allow(dead_code)]
4use super::*;
5
6impl QTensorEngine {
7    /// Final logits via chunked projection into `logits_out` (fills min(vocab, buf) rows).
8    pub fn dispatch_output_logits_into(
9        &self,
10        index: &crate::gguf_sharder::GgufTensorIndex,
11        hidden: &[f32],
12        emb_dim: usize,
13        logits_out: &mut [f32],
14    ) -> usize {
15        #[cfg(not(target_arch = "wasm32"))]
16        if let Some(ref pkg) = self.ftw_package {
17            if let (Some(packed), Some(scales), Some(globals)) = (
18                pkg.fetch_tensor_bytes("lm_head.weight"),
19                pkg.fetch_tensor_bytes("lm_head.weight_scale"),
20                pkg.fetch_tensor_bytes("lm_head.weight_global"),
21            ) {
22                let vocab_size = globals.len() / 2;
23                let fill = vocab_size.min(logits_out.len());
24                let row_bytes = emb_dim / 2;
25                let scale_bytes = emb_dim / 16;
26                if emb_dim == 0 || hidden.len() < emb_dim || fill == 0 {
27                    return 0;
28                }
29                if packed.len() < vocab_size * row_bytes || scales.len() < vocab_size * scale_bytes
30                {
31                    return 0;
32                }
33                for token_id in 0..fill {
34                    let g_val = half::f16::from_le_bytes([
35                        globals[token_id * 2],
36                        globals[token_id * 2 + 1],
37                    ])
38                    .to_f32();
39                    let p_row = &packed[token_id * row_bytes..(token_id + 1) * row_bytes];
40                    let s_row = &scales[token_id * scale_bytes..(token_id + 1) * scale_bytes];
41                    logits_out[token_id] = etalon_inference::ordinary::output::nvfp4_output_row(
42                        p_row, s_row, g_val, hidden, emb_dim,
43                    );
44                }
45                return fill;
46            }
47        }
48
49        let Some(info) = index.logits_projection_info() else {
50            let n = emb_dim.min(logits_out.len());
51            logits_out[..n].copy_from_slice(&hidden[..n]);
52            return n;
53        };
54        let (n_in, vocab_size) = Self::matmul_dims(info);
55        let fill = vocab_size.min(logits_out.len());
56        if n_in > emb_dim || fill == 0 {
57            let n = emb_dim.min(logits_out.len());
58            logits_out[..n].copy_from_slice(&hidden[..n]);
59            return n;
60        }
61        let mmap = match self.model_bytes() {
62            Some(m) => m,
63            None => {
64                let n = emb_dim.min(logits_out.len());
65                logits_out[..n].copy_from_slice(&hidden[..n]);
66                return n;
67            }
68        };
69        let mut written = 0usize;
70        let n_chunks = vocab_size.div_ceil(VOCAB_CHUNK_ROWS);
71        for chunk_idx in 0..n_chunks {
72            if written >= fill {
73                break;
74            }
75            let row_start = chunk_idx * VOCAB_CHUNK_ROWS;
76            let chunk_rows = VOCAB_CHUNK_ROWS.min(vocab_size - row_start);
77            let raw = match crate::ggml_quants::fetch_tensor_row_range_bytes(
78                mmap,
79                index.tensor_data_start,
80                info,
81                row_start,
82                chunk_rows,
83            ) {
84                Ok(s) => s,
85                Err(_) => break,
86            };
87            let out_rows = chunk_rows.min(fill - written);
88            if !self.dispatch_gemm_raw_into(
89                info,
90                raw,
91                &hidden[..n_in],
92                &mut logits_out[written..written + out_rows],
93                n_in,
94                out_rows,
95            ) {
96                break;
97            }
98            written += out_rows;
99        }
100        if written > 0 {
101            written
102        } else {
103            let n = emb_dim.min(logits_out.len());
104            logits_out[..n].copy_from_slice(&hidden[..n]);
105            n
106        }
107    }
108
109    /// Decodes `lexicon_bound`; fails closed on malformed input.
110    pub fn decode_lexicon_bound(&self, _logits: &[f32], valid_lexicon_ids: &[u64]) -> u64 {
111        if valid_lexicon_ids.is_empty() {
112            0
113        } else {
114            valid_lexicon_ids[0]
115        }
116    }
117
118    #[cfg(target_arch = "wasm32")]
119    /// Async dispatch: gemm-raw-into.
120    pub(crate) async fn dispatch_gemm_raw_into_async(
121        &self,
122        info: &GgufTensorInfo,
123        raw: &[u8],
124        input: &[f32],
125        out: &mut [f32],
126        n_in: usize,
127        n_out: usize,
128    ) -> bool {
129        etalon_inference::browser_projection::BrowserProjectionEngine { host: self }
130            .dispatch_gemm_raw_into_async(info, raw, input, out, n_in, n_out)
131            .await
132    }
133
134    /// Phase 5.3: quantized GEMM against a **resident** weight sub-range (no per-call weight upload).
135    /// Same dispatch+readback as `dispatch_gemm_raw_into_async`, but binds the caller-provided
136    /// `weight` BindingResource (a slice of the resident output-projection buffer) instead of
137    /// `write_buffer`-ing the chunk. Used by the decode argmax once the logits projection is resident.
138    #[cfg(target_arch = "wasm32")]
139    pub(crate) async fn dispatch_gemm_resident_chunk_async(
140        &self,
141        weight_ggml_type: u32,
142        weight_row_elems: u32,
143        weight: etalon::BindingResource<'_>,
144        weight_byte_len: u32,
145        input: &[f32],
146        out: &mut [f32],
147        n_in: usize,
148        n_out: usize,
149    ) -> bool {
150        etalon_inference::browser_projection::BrowserProjectionEngine { host: self }
151            .dispatch_gemm_resident_chunk_async(
152                weight_ggml_type,
153                weight_row_elems,
154                weight,
155                weight_byte_len,
156                input,
157                out,
158                n_in,
159                n_out,
160            )
161            .await
162    }
163    #[cfg(target_arch = "wasm32")]
164    /// Async dispatch: gemm-into.
165    pub async fn dispatch_gemm_into_async(
166        &self,
167        index: &crate::gguf_sharder::GgufTensorIndex,
168        info: &GgufTensorInfo,
169        input: &[f32],
170        out: &mut [f32],
171        n_in: usize,
172        n_out: usize,
173    ) -> bool {
174        etalon_inference::browser_projection::BrowserProjectionEngine { host: self }
175            .dispatch_gemm_into_async(index, info, input, out, n_in, n_out)
176            .await
177    }
178    #[cfg(target_arch = "wasm32")]
179    /// Async dispatch: attention-layer.
180    pub(crate) async fn dispatch_attention_layer_async(
181        &mut self,
182        index: &crate::gguf_sharder::GgufTensorIndex,
183        layer: u32,
184        token_idx: u32,
185        hidden: &[f32],
186        emb_dim: usize,
187        tensors: &crate::gguf_sharder::LayerTensors,
188        scratch_a: &mut [f32],
189        scratch_b: &mut [f32],
190    ) -> Option<usize> {
191        etalon_inference::ordinary::async_attention::dispatch_attention_layer_async(
192            self, index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
193        )
194        .await
195    }
196
197    #[cfg(target_arch = "wasm32")]
198    /// Async dispatch: attention-pass.
199    pub(crate) async fn dispatch_attention_pass_async(
200        &self,
201        hidden: &[f32],
202        n_embd: usize,
203        num_tokens_in_batch: u32,
204        batch_start_token_idx: u32,
205        layout: &KvCacheLayout,
206        layer: u32,
207        token_idx: u32,
208        h: &crate::gguf_sharder::GgufHyperparams,
209        info: &GgufTensorInfo,
210        raw_weights: &[u8],
211        proj_kind: u32,
212        n_workgroups: u32,
213        readback_out: Option<&mut [f32]>,
214    ) -> bool {
215        if !ggml_gpu_quant_supported(info.ggml_type) {
216            wlog(&format!(
217                "[attn_pass_async] GUARD unsupported quant kind={proj_kind}"
218            ));
219            return false;
220        }
221        if !ggml_gpu_attention_shader_supported(info.ggml_type) {
222            return self.cpu_attention_pass(
223                hidden,
224                n_embd,
225                num_tokens_in_batch,
226                batch_start_token_idx,
227                layout,
228                layer,
229                h,
230                info,
231                raw_weights,
232                proj_kind,
233                None,
234                readback_out,
235            );
236        }
237        let batch = num_tokens_in_batch.max(1) as usize;
238        let hidden_elems = n_embd.checked_mul(batch).unwrap_or(0);
239        if hidden_elems > hidden.len()
240            || hidden_elems > self.gemm_max_input_floats
241            || raw_weights.len() > self.max_tensor_bytes
242            || self.gemm_input_buf.is_none()
243            || self.kv_cache_gpu.is_none()
244            || self.attention_params_buf.is_none()
245            || self.attention_mask_buf.is_none()
246        {
247            wlog(&format!(
248                "[attn_pass_async] GUARD buffers kind={proj_kind} hidden_elems={hidden_elems} hidden={} gemm_in={} raw_w={} max_w={}",
249                hidden.len(),
250                self.gemm_max_input_floats,
251                raw_weights.len(),
252                self.max_tensor_bytes,
253            ));
254            return false;
255        }
256
257        let (mask_words, mask_active, mask_word_count) =
258            Self::attention_kv_mask_for_dispatch(layout, token_idx, proj_kind);
259        let params = Self::attention_gpu_params(
260            h,
261            layout,
262            layer,
263            token_idx,
264            info,
265            raw_weights.len(),
266            proj_kind,
267            num_tokens_in_batch.max(1),
268            batch_start_token_idx,
269            mask_active,
270            mask_word_count,
271            0,
272        );
273        let input_buf = self.gemm_input_buf.as_ref().unwrap();
274        let weight_buf = self.gemm_weight_buf.as_ref().unwrap();
275        let output_buf = self.gemm_output_buf.as_ref().unwrap();
276        let params_buf = self.attention_params_buf.as_ref().unwrap();
277        let mask_buf = self.attention_mask_buf.as_ref().unwrap();
278        let kv_buf = self.kv_cache_gpu.as_ref().unwrap();
279        let staging = self.gemm_output_staging.as_ref().unwrap();
280
281        self.gpu_queue()
282            .write_buffer(input_buf, 0, bytemuck::cast_slice(&hidden[..hidden_elems]));
283        self.write_weight_words(raw_weights, self.max_tensor_bytes);
284        self.gpu_queue()
285            .write_buffer(params_buf, 0, bytemuck::bytes_of(&params));
286        self.gpu_queue()
287            .write_buffer(mask_buf, 0, bytemuck::cast_slice(&mask_words));
288
289        let resources = etalon_inference::gpu_attention::PassResources {
290            input: input_buf,
291            weight: weight_buf,
292            output: output_buf,
293            params: params_buf,
294            mask: mask_buf,
295            kv: kv_buf,
296            staging,
297            bind_layout: &self.attention_pipeline.get_bind_group_layout(0),
298            pipeline: &self.attention_pipeline,
299        };
300        let readback_elems = readback_out.as_ref().map(|o| o.len()).unwrap_or(0);
301        let mut encoder = match etalon_inference::gpu_attention::encode_pass(
302            self.gpu_device(),
303            self.gpu_queue(),
304            &resources,
305            &params,
306            &mask_words,
307            n_workgroups,
308            readback_elems,
309            crate::llm_gpu_profiler::pass_writes_both(),
310        ) {
311            Some(encoder) => encoder,
312            None => return false,
313        };
314        crate::llm_gpu_profiler::resolve(&mut encoder);
315        self.submit_commands(Some(encoder.finish()));
316        crate::llm_gpu_profiler::accumulate(crate::llm_gpu_profiler::Phase::Attention);
317
318        if readback_elems == 0 {
319            return true;
320        }
321
322        let out_bytes = (readback_elems * 4) as etalon::BufferAddress;
323        let slice = staging.slice(..out_bytes);
324        if await_wgpu_map(slice).await {
325            let data = slice
326                .get_mapped_range()
327                .expect("wgpu buffer map_range failed");
328            let floats: &[f32] = bytemuck::cast_slice(&data);
329            if let Some(out) = readback_out {
330                out[..readback_elems].copy_from_slice(&floats[..readback_elems]);
331            }
332            drop(data);
333            staging.unmap();
334            return true;
335        }
336        let _ = staging.unmap();
337        false
338    }
339    #[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
340    /// Async dispatch: attention-q-ffn-token.
341    pub(crate) async fn dispatch_attention_q_ffn_token_async(
342        &mut self,
343        index: &crate::gguf_sharder::GgufTensorIndex,
344        layer: u32,
345        token_idx: u32,
346        hidden: &mut [f32],
347        emb_dim: usize,
348        tensors: &crate::gguf_sharder::LayerTensors,
349        scratch_a: &mut [f32],
350        scratch_b: &mut [f32],
351    ) -> bool {
352        etalon_inference::ordinary::async_attention_token::dispatch_attention_q_ffn_token_async(
353            self, index, layer, token_idx, hidden, emb_dim, tensors, scratch_a, scratch_b,
354        )
355        .await
356    }
357}