qualia_core_db/gguf_bridge/resident_decode.rs
1//! Native GPU-resident single-fence token decode.
2//!
3//! The legacy native decode path round-trips the hidden state through the CPU
4//! twice per layer (attention readback + FFN readback), paying ~2 blocking
5//! `submit → poll(wait)` fences and two CPU RMSNorm/residual-adds per layer —
6//! ~107 fences/token measured on SmolLM2-360M (A2000, Vulkan), ~24% of the
7//! token in pure fence latency plus the GPU idling through every CPU
8//! turnaround.
9//!
10//! This module keeps the hidden state resident in VRAM for the WHOLE token:
11//! per layer it encodes RMSNorm (elem op) → K/V/Q preprojection (coop GEMV) +
12//! KV-cache write → Q-SDPA (reads precomputed Q) → O-projection → residual add
13//! (elem) → RMSNorm (elem) → fused FFN expansion (`fused_ffn.wgsl` when gate/up
14//! share a supported quant; else gate/up GEMV + SiLU·mul) → down GEMV → residual
15//! add (elem), then the output RMSNorm and the chunked logits GEMV + top-1
16//! block reduction — all into ONE command encoder, ONE submit, ONE fence, with
17//! a ~400-byte candidate readback.
18//! It is the native mirror of the proven wasm MC8 fused-encoder design
19//! (`mc8_wasm/`), built from the same kernels the legacy path already runs:
20//! `coop_gemv`/`main` GEMV, `fused_ffn.wgsl`, `fused_attention.wgsl`,
21//! `wasm_elementwise.wgsl`, `topk_reduction.wgsl`. The GPU RMSNorm reduces in
22//! the same sequential order as the CPU `rms_norm_inplace`, so decode output is
23//! expected token-identical to the legacy path (asserted by the `a1d` differential test).
24//!
25//! All bind groups and static uniform slots are created ONCE per model (the
26//! weights are Phase-2 resident, so bindings are stable). Per token the driver
27//! writes: the embedded token (n_embd floats), the per-layer attention uniform
28//! block (token_idx / KV mask fields — one `write_buffer`), and the KV mask
29//! words. Everything else is pre-encoded state.
30//!
31//! Toggle: `QUALIA_LLM_RESIDENT_DECODE` / `set_resident_decode` (default ON).
32//! Any ineligibility (unsupported quant, missing resident logits, layer cap,
33//! sieve mask) falls back to the legacy per-layer path unchanged.
34
35#![cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
36
37use super::*;
38pub(crate) use etalon_inference::native_resident::resident_decode_plan::*;
39
40impl QTensorEngine {
41 /// Exact steady-state compute dispatch count for the prepared resident plan.
42 pub fn resident_dispatches_per_token(&self) -> Option<u32> {
43 match &self.resident_decode {
44 ResidentDecodeState::Ready(plan) => Some(plan.dispatches_per_token),
45 _ => None,
46 }
47 }
48
49 /// Exact candidate staging copy size for one resident greedy token.
50 pub fn resident_readback_bytes_per_token(&self) -> Option<u32> {
51 match &self.resident_decode {
52 ResidentDecodeState::Ready(plan) => Some(plan.readback_bytes_per_token),
53 _ => None,
54 }
55 }
56
57 fn plan_key(&self, index: &crate::gguf_sharder::GgufTensorIndex) -> (u64, u64, u32) {
58 let base = self.model_bytes().map(|m| m.as_ptr() as u64).unwrap_or(0);
59 (base, index.tensor_data_start, index.hyperparams.n_layer)
60 }
61
62 /// Single-fence resident-token decode (greedy top-1). Returns the argmax token, or `None`
63 /// on any ineligibility (caller falls back to the legacy per-layer path).
64 pub fn dispatch_token_forward_resident(
65 &mut self,
66 index: &crate::gguf_sharder::GgufTensorIndex,
67 emb: &[f32],
68 token_idx: u32,
69 ) -> Option<StreamingArgmaxResult> {
70 match self.with_resident_plan(index, |this, plan| {
71 this.run_resident_token(plan, emb, token_idx, None)
72 })? {
73 Some(ResidentTokenOutcome::Argmax(a)) => Some(StreamingArgmaxResult {
74 best_token_id: a.best_token_id,
75 max_logit: a.max_logit,
76 }),
77 _ => None,
78 }
79 }
80
81 /// Sampler-compatible resident forward: same single-fence layer stack + output
82 /// RMSNorm, then read back the normed hidden into `out_hidden` so the caller can
83 /// project full logits + sample on CPU without the legacy ~107-fence path.
84 ///
85 /// `emb` is the token embedding input; `out_hidden` receives post-output-norm state
86 /// (may be the same logical buffer only if the caller copies input first — they must
87 /// not alias while the upload of `emb` is live; pass distinct slices).
88 pub fn dispatch_token_forward_resident_hidden(
89 &mut self,
90 index: &crate::gguf_sharder::GgufTensorIndex,
91 emb: &[f32],
92 token_idx: u32,
93 out_hidden: &mut [f32],
94 ) -> bool {
95 matches!(
96 self.with_resident_plan(index, |this, plan| {
97 this.run_resident_token(plan, emb, token_idx, Some(out_hidden))
98 }),
99 Some(Some(ResidentTokenOutcome::HiddenReady))
100 )
101 }
102
103 fn with_resident_plan<R>(
104 &mut self,
105 index: &crate::gguf_sharder::GgufTensorIndex,
106 f: impl FnOnce(&Self, &mut ResidentDecodePlan) -> R,
107 ) -> Option<R> {
108 if !crate::llm_bench::resident_decode_enabled()
109 || !crate::llm_bench::resident_weights_enabled()
110 || crate::llm_bench::cpu_attention_enabled()
111 || crate::llm_bench::kv_dict_enabled()
112 {
113 return None;
114 }
115 // Note: do NOT skip resident for mode=cuda — the wgpu mega-pass is still faster
116 // end-to-end than legacy host-layered CUDA GEMVs. CUDA FFN block helps the
117 // non-resident / fused-resident FFN cold path only.
118 let key = self.plan_key(index);
119 let state = std::mem::replace(&mut self.resident_decode, ResidentDecodeState::Unbuilt);
120 let mut plan = match state {
121 ResidentDecodeState::Ready(p) if p.key == key => p,
122 ResidentDecodeState::Ineligible(k) if k == key => {
123 self.resident_decode = ResidentDecodeState::Ineligible(k);
124 return None;
125 }
126 _ => match self.build_resident_plan(index, key) {
127 Some(p) => p,
128 None => {
129 wlog("[resident-decode] plan build ineligible — legacy path");
130 self.resident_decode = ResidentDecodeState::Ineligible(key);
131 return None;
132 }
133 },
134 };
135 let result = f(self, &mut plan);
136 self.resident_decode = ResidentDecodeState::Ready(plan);
137 Some(result)
138 }
139
140 fn run_resident_token(
141 &self,
142 plan: &mut ResidentDecodePlan,
143 emb: &[f32],
144 token_idx: u32,
145 out_hidden: Option<&mut [f32]>,
146 ) -> Option<ResidentTokenOutcome> {
147 etalon_inference::native_resident::ResidentEngine { host: self }
148 .run_resident_token(plan, emb, token_idx, out_hidden)
149 }
150
151 /// Build the per-model plan: force weight residency, create activation
152 /// buffers + static uniform slots + all bind groups. Any missing piece →
153 /// `None` (the caller records Ineligible and the legacy path runs).
154 fn build_resident_plan(
155 &mut self,
156 index: &crate::gguf_sharder::GgufTensorIndex,
157 key: (u64, u64, u32),
158 ) -> Option<Box<ResidentDecodePlan>> {
159 etalon_inference::native_resident::planning::ResidentPlanner { host: self }
160 .build_resident_plan(index, key)
161 }
162}