Skip to main content

qualia_core_db/gguf_bridge/
output.rs

1//! Output projection: chunked argmax + GPU top-k, output RMSNorm
2//! Split from gguf_bridge/mod.rs (structural refactor; no behaviour change).
3use super::*;
4
5impl QTensorEngine {
6    /// Chunked vocabulary projection with streaming argmax (zero heap, stack chunk buffer only).
7    /// `max_chunks`: `0` sweeps the full vocabulary; otherwise caps chunk iterations (tests).
8    pub fn dispatch_output_argmax_chunked(
9        &self,
10        index: &crate::gguf_sharder::GgufTensorIndex,
11        hidden: &[f32],
12        emb_dim: usize,
13        chunk_logits: &mut [f32],
14        max_chunks: u32,
15        sieve_mask: Option<&crate::neuro_symbolic_sieve::SieveStateMask>,
16    ) -> Option<StreamingArgmaxResult> {
17        #[cfg(not(target_arch = "wasm32"))]
18        if self.gemma4_required && !self.gemma4_ready(index) {
19            return None;
20        }
21        #[cfg(not(target_arch = "wasm32"))]
22        if self.qwen35_gguf_required && !self.dense_qwen_ready(index) {
23            return None;
24        }
25        if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
26            #[cfg(not(target_arch = "wasm32"))]
27            return self.qwen35_argmax(index, hidden, emb_dim, max_chunks, sieve_mask);
28            #[cfg(target_arch = "wasm32")]
29            return None;
30        }
31        etalon_inference::projections::ProjectionEngine { host: self }
32            .dispatch_output_argmax_chunked(
33                index,
34                hidden,
35                emb_dim,
36                chunk_logits,
37                max_chunks,
38                sieve_mask,
39            )
40            .map(|r| {
41                let softcap = index.hyperparams.logit_softcap;
42                let max_logit = if softcap > 0.0 {
43                    (r.max_logit / softcap).tanh() * softcap
44                } else {
45                    r.max_logit
46                };
47                StreamingArgmaxResult {
48                    best_token_id: r.best_token_id,
49                    max_logit,
50                }
51            })
52    }
53
54    /// A1a: create the persistent GPU top-k pipeline + small candidate/staging buffers (once).
55    #[cfg(feature = "gpu-runtime")]
56    pub(crate) fn init_output_topk(&mut self) {
57        #[cfg(not(target_arch = "wasm32"))]
58        let cache = self.native_pipeline_cache_ref();
59        #[cfg(target_arch = "wasm32")]
60        let cache = None;
61        let resources = etalon_inference::projections::topk_resources::TopkResources::build(
62            self.gpu_device(),
63            cache,
64            etalon_inference::gpu_resources::ResourceBudget::for_device(self.gpu_device()),
65        )
66        .expect("output top-k resources exceed device limits");
67        self.topk_cand_val_buf = Some(resources.values);
68        self.topk_cand_idx_buf = Some(resources.indices);
69        self.topk_cand_staging = Some(resources.staging);
70        self.topk_params_buf = Some(resources.params);
71        self.output_topk_bind_layout = Some(resources.layout);
72        self.output_topk_pipeline = Some(resources.pipeline);
73    }
74
75    /// Native decode fast path: output projection plus GPU block argmax (`k=1`) with one tiny
76    /// candidate readback after all vocab chunks have been submitted. This avoids the full
77    /// chunk-logit readback in [`Self::dispatch_output_argmax_chunked`] and avoids heap allocation
78    /// in the decode loop.
79    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
80    pub fn dispatch_output_top1_chunked(
81        &self,
82        index: &crate::gguf_sharder::GgufTensorIndex,
83        hidden: &[f32],
84        emb_dim: usize,
85    ) -> Option<StreamingArgmaxResult> {
86        if self.qwen35_gguf_required || self.gemma4_required {
87            return None;
88        }
89        etalon_inference::projections::ProjectionGpuEngine::new(self)
90            .dispatch_output_top1_chunked(index, hidden, emb_dim)
91            .map(|r| {
92                let softcap = index.hyperparams.logit_softcap;
93                let max_logit = if softcap > 0.0 {
94                    (r.max_logit / softcap).tanh() * softcap
95                } else {
96                    r.max_logit
97                };
98                StreamingArgmaxResult {
99                    best_token_id: r.best_token_id,
100                    max_logit,
101                }
102            })
103    }
104
105    /// A1a: GPU top-k over the output projection — the logits stay on-GPU (`gemm_output_buf`), the
106    /// top-k kernel reduces them per chunk, and only K `(id, logit)` candidates are read back (vs the
107    /// 196 KB/token full-logit readback + CPU argmax in `dispatch_output_argmax_chunked`). Returns the
108    /// merged global top-K, or `None` to signal the caller to fall back to the argmax path. v1: no
109    /// sieve coupling (caller routes here only when no sieve mask is active).
110    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
111    pub fn dispatch_output_topk_chunked(
112        &self,
113        index: &crate::gguf_sharder::GgufTensorIndex,
114        hidden: &[f32],
115        emb_dim: usize,
116        k: usize,
117    ) -> Option<Vec<crate::topk::TopKItem>> {
118        if self.qwen35_gguf_required || self.gemma4_required {
119            return None;
120        }
121        let mut selected = [crate::topk::TopKItem {
122            token_id: 0,
123            logit: f32::NEG_INFINITY,
124        }; crate::topk::TOPK_MAX_K];
125        let count = etalon_inference::projections::ProjectionGpuEngine::new(self)
126            .dispatch_output_topk_chunked_into(index, hidden, emb_dim, k, &mut selected)?;
127        Some(selected[..count].to_vec())
128    }
129
130    /// Final `output_norm` RMSNorm in-place before vocabulary projection (Pre-Norm LLM tail).
131    /// REQUIRED on all targets — native previously skipped it → logits from an un-normed hidden.
132    pub fn apply_output_norm_inplace(
133        &self,
134        index: &crate::gguf_sharder::GgufTensorIndex,
135        hidden: &mut [f32],
136        emb_dim: usize,
137    ) -> bool {
138        #[cfg(not(target_arch = "wasm32"))]
139        if self.gemma4_required {
140            return self.gemma4_output_norm(index, hidden, emb_dim);
141        }
142        #[cfg(not(target_arch = "wasm32"))]
143        if self.qwen35_gguf_required {
144            return self.dense_qwen_output_norm(index, hidden, emb_dim);
145        }
146        if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
147            #[cfg(not(target_arch = "wasm32"))]
148            return self.qwen35_output_norm(index, hidden, emb_dim);
149            #[cfg(target_arch = "wasm32")]
150            return false;
151        }
152        etalon_inference::projections::ProjectionEngine { host: self }
153            .apply_output_norm_inplace(index, hidden, emb_dim)
154    }
155}