Skip to main content

qualia_core_db/gguf_bridge/
resident_decode.rs

1//! Native GPU-resident single-fence token decode.
2//!
3//! The legacy native decode path round-trips the hidden state through the CPU
4//! twice per layer (attention readback + FFN readback), paying ~2 blocking
5//! `submit → poll(wait)` fences and two CPU RMSNorm/residual-adds per layer —
6//! ~107 fences/token measured on SmolLM2-360M (A2000, Vulkan), ~24% of the
7//! token in pure fence latency plus the GPU idling through every CPU
8//! turnaround.
9//!
10//! This module keeps the hidden state resident in VRAM for the WHOLE token:
11//! per layer it encodes RMSNorm (elem op) → K/V/Q preprojection (coop GEMV) +
12//! KV-cache write → Q-SDPA (reads precomputed Q) → O-projection → residual add
13//! (elem) → RMSNorm (elem) → fused FFN expansion (`fused_ffn.wgsl` when gate/up
14//! share a supported quant; else gate/up GEMV + SiLU·mul) → down GEMV → residual
15//! add (elem), then the output RMSNorm and the chunked logits GEMV + top-1
16//! block reduction — all into ONE command encoder, ONE submit, ONE fence, with
17//! a ~400-byte candidate readback.
18//! It is the native mirror of the proven wasm MC8 fused-encoder design
19//! (`mc8_wasm/`), built from the same kernels the legacy path already runs:
20//! `coop_gemv`/`main` GEMV, `fused_ffn.wgsl`, `fused_attention.wgsl`,
21//! `wasm_elementwise.wgsl`, `topk_reduction.wgsl`. The GPU RMSNorm reduces in
22//! the same sequential order as the CPU `rms_norm_inplace`, so decode output is
23//! expected token-identical to the legacy path (asserted by the `a1d` differential test).
24//!
25//! All bind groups and static uniform slots are created ONCE per model (the
26//! weights are Phase-2 resident, so bindings are stable). Per token the driver
27//! writes: the embedded token (n_embd floats), the per-layer attention uniform
28//! block (token_idx / KV mask fields — one `write_buffer`), and the KV mask
29//! words. Everything else is pre-encoded state.
30//!
31//! Toggle: `QUALIA_LLM_RESIDENT_DECODE` / `set_resident_decode` (default ON).
32//! Any ineligibility (unsupported quant, missing resident logits, layer cap,
33//! sieve mask) falls back to the legacy per-layer path unchanged.
34
35#![cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
36
37use super::*;
38pub(crate) use etalon_inference::native_resident::resident_decode_plan::*;
39
40impl QTensorEngine {
41    /// Exact steady-state compute dispatch count for the prepared resident plan.
42    pub fn resident_dispatches_per_token(&self) -> Option<u32> {
43        match &self.resident_decode {
44            ResidentDecodeState::Ready(plan) => Some(plan.dispatches_per_token),
45            _ => None,
46        }
47    }
48
49    /// Exact candidate staging copy size for one resident greedy token.
50    pub fn resident_readback_bytes_per_token(&self) -> Option<u32> {
51        match &self.resident_decode {
52            ResidentDecodeState::Ready(plan) => Some(plan.readback_bytes_per_token),
53            _ => None,
54        }
55    }
56
57    fn plan_key(&self, index: &crate::gguf_sharder::GgufTensorIndex) -> (u64, u64, u32) {
58        let base = self.model_bytes().map(|m| m.as_ptr() as u64).unwrap_or(0);
59        (base, index.tensor_data_start, index.hyperparams.n_layer)
60    }
61
62    /// Single-fence resident-token decode (greedy top-1). Returns the argmax token, or `None`
63    /// on any ineligibility (caller falls back to the legacy per-layer path).
64    pub fn dispatch_token_forward_resident(
65        &mut self,
66        index: &crate::gguf_sharder::GgufTensorIndex,
67        emb: &[f32],
68        token_idx: u32,
69    ) -> Option<StreamingArgmaxResult> {
70        match self.with_resident_plan(index, |this, plan| {
71            this.run_resident_token(plan, emb, token_idx, None)
72        })? {
73            Some(ResidentTokenOutcome::Argmax(a)) => Some(StreamingArgmaxResult {
74                best_token_id: a.best_token_id,
75                max_logit: a.max_logit,
76            }),
77            _ => None,
78        }
79    }
80
81    /// Sampler-compatible resident forward: same single-fence layer stack + output
82    /// RMSNorm, then read back the normed hidden into `out_hidden` so the caller can
83    /// project full logits + sample on CPU without the legacy ~107-fence path.
84    ///
85    /// `emb` is the token embedding input; `out_hidden` receives post-output-norm state
86    /// (may be the same logical buffer only if the caller copies input first — they must
87    /// not alias while the upload of `emb` is live; pass distinct slices).
88    pub fn dispatch_token_forward_resident_hidden(
89        &mut self,
90        index: &crate::gguf_sharder::GgufTensorIndex,
91        emb: &[f32],
92        token_idx: u32,
93        out_hidden: &mut [f32],
94    ) -> bool {
95        matches!(
96            self.with_resident_plan(index, |this, plan| {
97                this.run_resident_token(plan, emb, token_idx, Some(out_hidden))
98            }),
99            Some(Some(ResidentTokenOutcome::HiddenReady))
100        )
101    }
102
103    fn with_resident_plan<R>(
104        &mut self,
105        index: &crate::gguf_sharder::GgufTensorIndex,
106        f: impl FnOnce(&Self, &mut ResidentDecodePlan) -> R,
107    ) -> Option<R> {
108        if !crate::llm_bench::resident_decode_enabled()
109            || !crate::llm_bench::resident_weights_enabled()
110            || crate::llm_bench::cpu_attention_enabled()
111            || crate::llm_bench::kv_dict_enabled()
112        {
113            return None;
114        }
115        // Note: do NOT skip resident for mode=cuda — the wgpu mega-pass is still faster
116        // end-to-end than legacy host-layered CUDA GEMVs. CUDA FFN block helps the
117        // non-resident / fused-resident FFN cold path only.
118        let key = self.plan_key(index);
119        let state = std::mem::replace(&mut self.resident_decode, ResidentDecodeState::Unbuilt);
120        let mut plan = match state {
121            ResidentDecodeState::Ready(p) if p.key == key => p,
122            ResidentDecodeState::Ineligible(k) if k == key => {
123                self.resident_decode = ResidentDecodeState::Ineligible(k);
124                return None;
125            }
126            _ => match self.build_resident_plan(index, key) {
127                Some(p) => p,
128                None => {
129                    wlog("[resident-decode] plan build ineligible — legacy path");
130                    self.resident_decode = ResidentDecodeState::Ineligible(key);
131                    return None;
132                }
133            },
134        };
135        let result = f(self, &mut plan);
136        self.resident_decode = ResidentDecodeState::Ready(plan);
137        Some(result)
138    }
139
140    fn run_resident_token(
141        &self,
142        plan: &mut ResidentDecodePlan,
143        emb: &[f32],
144        token_idx: u32,
145        out_hidden: Option<&mut [f32]>,
146    ) -> Option<ResidentTokenOutcome> {
147        etalon_inference::native_resident::ResidentEngine { host: self }
148            .run_resident_token(plan, emb, token_idx, out_hidden)
149    }
150
151    /// Build the per-model plan: force weight residency, create activation
152    /// buffers + static uniform slots + all bind groups. Any missing piece →
153    /// `None` (the caller records Ineligible and the legacy path runs).
154    fn build_resident_plan(
155        &mut self,
156        index: &crate::gguf_sharder::GgufTensorIndex,
157        key: (u64, u64, u32),
158    ) -> Option<Box<ResidentDecodePlan>> {
159        etalon_inference::native_resident::planning::ResidentPlanner { host: self }
160            .build_resident_plan(index, key)
161    }
162}