qualia_core_db/gguf_bridge/
ssm_forward.rs1#![allow(unused_imports, unused_variables, unused_mut)]
3use crate::gguf_sharder::{GgufTensorIndex, LayerTensors};
4use etalon_inference::ordinary::ssm_layout::{
5 CONV_HISTORY_DEPTH, SsmLayerStateLayout, SsmStateError, ssm_layout_for,
6};
7use etalon_inference::ordinary::{HybridEngine, LayerProvider};
8impl LayerProvider for super::QTensorEngine {
9 type Index = GgufTensorIndex;
10 fn hyperparams(index: &Self::Index) -> etalon_model::inference::ModelHyperparams {
11 index.hyperparams.into()
12 }
13 fn moe_routing(
14 index: &Self::Index,
15 ) -> etalon_model::inference::moe_routing::MoeRoutingSettings {
16 index.moe_routing
17 }
18 fn data_start(index: &Self::Index) -> u64 {
19 index.tensor_data_start
20 }
21 fn tensors(index: &Self::Index, layer: u32) -> LayerTensors {
22 index.get_layer_tensors(layer)
23 }
24 fn model_bytes(&self) -> Option<&[u8]> {
25 super::QTensorEngine::model_bytes(self)
26 }
27 fn state(&self) -> &[f32] {
28 &self.ssm_recurrent_state
29 }
30 fn state_mut(&mut self) -> &mut [f32] {
31 &mut self.ssm_recurrent_state
32 }
33 fn gemm(
34 &self,
35 index: &Self::Index,
36 info: &crate::gguf_sharder::GgufTensorInfo,
37 input: &[f32],
38 out: &mut [f32],
39 n_in: usize,
40 n_out: usize,
41 ) -> bool {
42 self.dispatch_gemm_into(index, info, input, out, n_in, n_out)
43 }
44 fn norm(&self, info: &crate::gguf_sharder::GgufTensorInfo, out: &mut [f32]) -> usize {
45 self.dequant_norm_tensor_into(info, out)
46 }
47 fn raw(&self, info: &crate::gguf_sharder::GgufTensorInfo) -> Option<&[u8]> {
48 self.fetch_tensor_raw_bytes(info)
49 }
50 fn event(&self, event: etalon_inference::ordinary::LayerEvent) {
51 use etalon_inference::ordinary::LayerEvent;
52 match event {
53 LayerEvent::StateMissing(layer) => {
54 super::wlog(&format!("[ssm] layer={layer} refused: state not allocated"))
55 }
56 LayerEvent::SsmUnrecognized(layer) => {
57 super::wlog(&format!("[ssm] layer={layer} unrecognised SSM flavour"))
58 }
59 LayerEvent::GraniteModelMissing(layer) => {
60 super::wlog(&format!("[granite_ssm] layer={layer} mmap None"))
61 }
62 LayerEvent::GraniteInputProjection(layer) => {
63 super::wlog(&format!("[granite_ssm] layer={layer} ssm_in GEMM fail"))
64 }
65 LayerEvent::GraniteConvolutionFetch(layer) => {
66 super::wlog(&format!("[granite_ssm] layer={layer} conv fetch fail"))
67 }
68 LayerEvent::QwenModelMissing(layer) => {
69 super::wlog(&format!("[qwen_ssm] layer={layer} mmap None"))
70 }
71 LayerEvent::QwenQkvProjection(layer) => {
72 super::wlog(&format!("[qwen_ssm] layer={layer} QKV GEMM fail"))
73 }
74 }
75 }
76 fn moe(
77 &mut self,
78 index: &Self::Index,
79 emb_dim: usize,
80 tensors: &LayerTensors,
81 out: &mut [f32],
82 input: &[f32],
83 ) -> bool {
84 self.dispatch_moe_ffn(index, emb_dim, tensors, out, input)
85 }
86 fn record_ffn_input(&self, input: &[f32]) {
87 crate::llm_awq::record_ffn_input(input)
88 }
89 fn attention(
90 &mut self,
91 index: &Self::Index,
92 layer: u32,
93 token: u32,
94 hidden: &[f32],
95 emb_dim: usize,
96 tensors: &LayerTensors,
97 a: &mut [f32],
98 b: &mut [f32],
99 ) -> Option<usize> {
100 self.dispatch_attention_layer(index, layer, token, hidden, emb_dim, tensors, a, b)
101 }
102 fn ffn_dispatch(
103 &mut self,
104 index: &Self::Index,
105 hidden: &mut [f32],
106 emb_dim: usize,
107 tensors: &LayerTensors,
108 a: &mut [f32],
109 b: &mut [f32],
110 ) -> bool {
111 self.dispatch_ffn_block_pre_norm(index, hidden, emb_dim, tensors, a, b)
112 }
113 fn profile_attention(&self, ns: u64) {
114 #[cfg(not(target_arch = "wasm32"))]
115 crate::llm_bench::add_decode_attn_ns(ns);
116 }
117 fn profile_ffn(&self, ns: u64) {
118 #[cfg(not(target_arch = "wasm32"))]
119 crate::llm_bench::add_decode_ffn_ns(ns);
120 }
121 fn layer_diagnostic(
122 &self,
123 layer: u32,
124 attn_ok: bool,
125 tensors: &LayerTensors,
126 attention: &[f32],
127 hidden: &[f32],
128 ) {
129 if layer < 3 && crate::gguf_bridge::execution_env::var("QUALIA_LLM_DEBUG_DECODE").is_ok() {
130 let max_attn = attention.iter().fold(0f32, |m, &v| m.max(v.abs()));
131 let max_hid = hidden.iter().fold(0f32, |m, &v| m.max(v.abs()));
132 eprintln!(
133 "[layer-dbg] L{} attn_ok={} attn_norm={} ffn_norm={} max|attn_out|={:.4} max|hidden_postattn|={:.4}",
134 layer,
135 attn_ok,
136 tensors.attn_norm.is_some(),
137 tensors.ffn_norm.is_some(),
138 max_attn,
139 max_hid
140 );
141 }
142 }
143 fn expert_source(
144 &self,
145 ) -> Option<&dyn etalon_inference::ordinary::expert_source::ExpertSource> {
146 #[cfg(not(target_arch = "wasm32"))]
147 {
148 self.ftw_package
149 .as_deref()
150 .map(|s| s as &dyn etalon_inference::ordinary::expert_source::ExpertSource)
151 }
152 #[cfg(target_arch = "wasm32")]
153 {
154 None
155 }
156 }
157}
158impl super::QTensorEngine {
159 pub fn set_ssm_state_byte_budget(&mut self, byte_budget: usize) {
162 self.ssm_state_byte_budget = byte_budget;
163 }
164
165 pub fn init_ssm_state_with_budget(
166 &mut self,
167 index: &GgufTensorIndex,
168 byte_budget: usize,
169 ) -> Result<usize, SsmStateError> {
170 use etalon_inference::ordinary::ssm_layout::{
171 allocate_recurrent_state, checked_ssm_layout_for,
172 };
173 let h = index.hyperparams.into();
174 let state = allocate_recurrent_state(
175 (0..index.hyperparams.n_layer).filter_map(|layer| {
176 let tensors = index.get_layer_tensors(layer);
177 tensors.is_hybrid_ssm_layer().then(|| {
178 checked_ssm_layout_for(&tensors, &h, index.hyperparams.n_embd as usize)
179 .map(|layout| layout.stride)
180 })
181 }),
182 byte_budget,
183 )?;
184 let count = state.len();
185 self.ssm_recurrent_state = state;
186 Ok(count)
187 }
188
189 pub fn init_ssm_state(&mut self, index: &GgufTensorIndex) -> usize {
192 match self.init_ssm_state_with_budget(index, self.ssm_state_byte_budget) {
193 Ok(count) => count,
194 Err(_) => {
195 self.ssm_recurrent_state = Box::new([]);
196 0
197 }
198 }
199 }
200
201 pub fn reset_recurrent_state(&mut self) {
202 #[cfg(not(target_arch = "wasm32"))]
203 if let Some(session) = &mut self.gemma4 {
204 session.reset();
205 }
206 #[cfg(not(target_arch = "wasm32"))]
207 if let Some(session) = &mut self.qwen35_gguf {
208 session.reset();
209 }
210 #[cfg(not(target_arch = "wasm32"))]
211 if let Some(session) = &mut self.qwen35_ftw {
212 session.reset();
213 }
214 self.ssm_recurrent_state.fill(0.0);
215 }
216
217 #[must_use]
221 pub fn snapshot_recurrent_state_into(&self, out: &mut [f32]) -> Option<usize> {
222 #[cfg(not(target_arch = "wasm32"))]
223 if self.qwen35_gguf_required || self.gemma4_required {
224 return None;
225 }
226 let len = self.ssm_recurrent_state.len();
227 if out.len() < len {
228 return None;
229 }
230 out[..len].copy_from_slice(&self.ssm_recurrent_state);
231 Some(len)
232 }
233
234 #[must_use]
237 pub fn restore_recurrent_state_from_slice(&mut self, src: &[f32]) -> bool {
238 #[cfg(not(target_arch = "wasm32"))]
239 if self.qwen35_gguf_required || self.gemma4_required {
240 return false;
241 }
242 let len = self.ssm_recurrent_state.len();
243 if len == 0 || src.len() != len {
244 return false;
245 }
246 self.ssm_recurrent_state.copy_from_slice(src);
247 true
248 }
249
250 pub(crate) fn dispatch_hybrid_ssm_layer(
251 &mut self,
252 index: &GgufTensorIndex,
253 layer: u32,
254 hidden: &mut [f32],
255 emb_dim: usize,
256 tensors: &LayerTensors,
257 scratch_a: &mut [f32],
258 scratch_b: &mut [f32],
259 ) -> bool {
260 #[cfg(not(target_arch = "wasm32"))]
261 if self.qwen35_gguf_required || self.gemma4_required {
262 return false;
263 }
264 if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
265 #[cfg(not(target_arch = "wasm32"))]
266 return self.qwen35_attention(index, layer, hidden, emb_dim, tensors);
267 #[cfg(target_arch = "wasm32")]
268 return false;
269 }
270 HybridEngine { host: self }
271 .dispatch(index, layer, hidden, emb_dim, tensors, scratch_a, scratch_b)
272 }
273}
274#[cfg(test)]
275mod tests {
276 use super::*;
277 use crate::gguf_sharder::{GgufHyperparams, LayerTensors};
278
279 #[test]
280 fn ssm_layout_new_granite_style() {
281 let layout = SsmLayerStateLayout::new(4096, 128, 64, 1);
282 assert_eq!(layout.conv_offset, 0);
283 assert_eq!(layout.conv_len, CONV_HISTORY_DEPTH * 4096);
284 assert_eq!(layout.delta_offset, layout.conv_len);
285 assert_eq!(layout.delta_len, 128 * 64 * 1);
286 assert_eq!(layout.stride, layout.conv_len + layout.delta_len);
287 }
288
289 #[test]
290 fn ssm_layout_new_qwen_style() {
291 let layout = SsmLayerStateLayout::new(2048, 128, 128, 16);
292 assert_eq!(layout.conv_len, CONV_HISTORY_DEPTH * 2048);
293 assert_eq!(layout.delta_len, 128 * 128 * 16);
294 assert_eq!(layout.stride, layout.conv_len + layout.delta_len);
295 }
296
297 #[test]
298 fn ssm_layout_for_uses_explicit_ssm_inner_size() {
299 let mut h = GgufHyperparams::default();
300 h.ssm_inner_size = 1024;
301 h.ssm_state_size = 64;
302 h.ssm_group_count = 4;
303 h.n_embd = 512;
304 let tensors = LayerTensors::default();
305 let layout = ssm_layout_for(&tensors, &h.into(), 512);
306 assert_eq!(layout.conv_len, CONV_HISTORY_DEPTH * 1024);
307 assert!(layout.delta_len > 0);
308 }
309
310 #[test]
311 fn ssm_state_size_example_external_model_arena() {
312 let layout = SsmLayerStateLayout::new(4096, 128, 64, 1);
314 let bytes = layout.stride * 96 * 4;
315 assert!(
317 bytes < 8 * 1024 * 1024,
318 "SSM example external state exceeds 8 MB: {bytes} bytes"
319 );
320 }
321 #[test]
322 fn reset_sequence_clears_recurrence_without_kv_layout() {
323 use crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc;
324 let mut engine = super::super::QTensorEngine::new();
325 engine.ssm_recurrent_state = vec![1.0, -2.0, 3.0].into_boxed_slice();
326 engine.kv_layout = None;
327 assert_zero_alloc("reset sequence", || engine.reset_kv_cache());
328 assert_eq!(&*engine.ssm_recurrent_state, &[0.0; 3]);
329 }
330
331 #[test]
332 fn attention_only_replacement_drops_old_recurrence() {
333 let mut engine = super::super::QTensorEngine::new();
334 engine.ssm_recurrent_state = vec![9.0; 8].into_boxed_slice();
335 let index = GgufTensorIndex {
336 entries: Vec::new(),
337 tensor_data_start: 0,
338 token_embd: None,
339 output_weight: None,
340 output_norm: None,
341 ple_ngram_embedding: None,
342 moe_routing: Default::default(),
343 ple_config: None,
344 hyperparams: GgufHyperparams::default(),
345 max_tensor_bytes: 0,
346 max_layer_tensor_bytes: 0,
347 };
348 assert_eq!(engine.init_ssm_state_with_budget(&index, 0), Ok(0));
349 assert!(engine.ssm_recurrent_state.is_empty());
350 }
351}