Skip to main content

qualia_core_db/gguf_bridge/
ssm_forward.rs

1//! Hybrid numerical execution adapter; provider dispatch stays in QTensorEngine.
2#![allow(unused_imports, unused_variables, unused_mut)]
3use crate::gguf_sharder::{GgufTensorIndex, LayerTensors};
4use etalon_inference::ordinary::ssm_layout::{
5    CONV_HISTORY_DEPTH, SsmLayerStateLayout, SsmStateError, ssm_layout_for,
6};
7use etalon_inference::ordinary::{HybridEngine, LayerProvider};
8impl LayerProvider for super::QTensorEngine {
9    type Index = GgufTensorIndex;
10    fn hyperparams(index: &Self::Index) -> etalon_model::inference::ModelHyperparams {
11        index.hyperparams.into()
12    }
13    fn moe_routing(
14        index: &Self::Index,
15    ) -> etalon_model::inference::moe_routing::MoeRoutingSettings {
16        index.moe_routing
17    }
18    fn data_start(index: &Self::Index) -> u64 {
19        index.tensor_data_start
20    }
21    fn tensors(index: &Self::Index, layer: u32) -> LayerTensors {
22        index.get_layer_tensors(layer)
23    }
24    fn model_bytes(&self) -> Option<&[u8]> {
25        super::QTensorEngine::model_bytes(self)
26    }
27    fn state(&self) -> &[f32] {
28        &self.ssm_recurrent_state
29    }
30    fn state_mut(&mut self) -> &mut [f32] {
31        &mut self.ssm_recurrent_state
32    }
33    fn gemm(
34        &self,
35        index: &Self::Index,
36        info: &crate::gguf_sharder::GgufTensorInfo,
37        input: &[f32],
38        out: &mut [f32],
39        n_in: usize,
40        n_out: usize,
41    ) -> bool {
42        self.dispatch_gemm_into(index, info, input, out, n_in, n_out)
43    }
44    fn norm(&self, info: &crate::gguf_sharder::GgufTensorInfo, out: &mut [f32]) -> usize {
45        self.dequant_norm_tensor_into(info, out)
46    }
47    fn raw(&self, info: &crate::gguf_sharder::GgufTensorInfo) -> Option<&[u8]> {
48        self.fetch_tensor_raw_bytes(info)
49    }
50    fn event(&self, event: etalon_inference::ordinary::LayerEvent) {
51        use etalon_inference::ordinary::LayerEvent;
52        match event {
53            LayerEvent::StateMissing(layer) => {
54                super::wlog(&format!("[ssm] layer={layer} refused: state not allocated"))
55            }
56            LayerEvent::SsmUnrecognized(layer) => {
57                super::wlog(&format!("[ssm] layer={layer} unrecognised SSM flavour"))
58            }
59            LayerEvent::GraniteModelMissing(layer) => {
60                super::wlog(&format!("[granite_ssm] layer={layer} mmap None"))
61            }
62            LayerEvent::GraniteInputProjection(layer) => {
63                super::wlog(&format!("[granite_ssm] layer={layer} ssm_in GEMM fail"))
64            }
65            LayerEvent::GraniteConvolutionFetch(layer) => {
66                super::wlog(&format!("[granite_ssm] layer={layer} conv fetch fail"))
67            }
68            LayerEvent::QwenModelMissing(layer) => {
69                super::wlog(&format!("[qwen_ssm] layer={layer} mmap None"))
70            }
71            LayerEvent::QwenQkvProjection(layer) => {
72                super::wlog(&format!("[qwen_ssm] layer={layer} QKV GEMM fail"))
73            }
74        }
75    }
76    fn moe(
77        &mut self,
78        index: &Self::Index,
79        emb_dim: usize,
80        tensors: &LayerTensors,
81        out: &mut [f32],
82        input: &[f32],
83    ) -> bool {
84        self.dispatch_moe_ffn(index, emb_dim, tensors, out, input)
85    }
86    fn record_ffn_input(&self, input: &[f32]) {
87        crate::llm_awq::record_ffn_input(input)
88    }
89    fn attention(
90        &mut self,
91        index: &Self::Index,
92        layer: u32,
93        token: u32,
94        hidden: &[f32],
95        emb_dim: usize,
96        tensors: &LayerTensors,
97        a: &mut [f32],
98        b: &mut [f32],
99    ) -> Option<usize> {
100        self.dispatch_attention_layer(index, layer, token, hidden, emb_dim, tensors, a, b)
101    }
102    fn ffn_dispatch(
103        &mut self,
104        index: &Self::Index,
105        hidden: &mut [f32],
106        emb_dim: usize,
107        tensors: &LayerTensors,
108        a: &mut [f32],
109        b: &mut [f32],
110    ) -> bool {
111        self.dispatch_ffn_block_pre_norm(index, hidden, emb_dim, tensors, a, b)
112    }
113    fn profile_attention(&self, ns: u64) {
114        #[cfg(not(target_arch = "wasm32"))]
115        crate::llm_bench::add_decode_attn_ns(ns);
116    }
117    fn profile_ffn(&self, ns: u64) {
118        #[cfg(not(target_arch = "wasm32"))]
119        crate::llm_bench::add_decode_ffn_ns(ns);
120    }
121    fn layer_diagnostic(
122        &self,
123        layer: u32,
124        attn_ok: bool,
125        tensors: &LayerTensors,
126        attention: &[f32],
127        hidden: &[f32],
128    ) {
129        if layer < 3 && crate::gguf_bridge::execution_env::var("QUALIA_LLM_DEBUG_DECODE").is_ok() {
130            let max_attn = attention.iter().fold(0f32, |m, &v| m.max(v.abs()));
131            let max_hid = hidden.iter().fold(0f32, |m, &v| m.max(v.abs()));
132            eprintln!(
133                "[layer-dbg] L{} attn_ok={} attn_norm={} ffn_norm={} max|attn_out|={:.4} max|hidden_postattn|={:.4}",
134                layer,
135                attn_ok,
136                tensors.attn_norm.is_some(),
137                tensors.ffn_norm.is_some(),
138                max_attn,
139                max_hid
140            );
141        }
142    }
143    fn expert_source(
144        &self,
145    ) -> Option<&dyn etalon_inference::ordinary::expert_source::ExpertSource> {
146        #[cfg(not(target_arch = "wasm32"))]
147        {
148            self.ftw_package
149                .as_deref()
150                .map(|s| s as &dyn etalon_inference::ordinary::expert_source::ExpertSource)
151        }
152        #[cfg(target_arch = "wasm32")]
153        {
154            None
155        }
156    }
157}
158impl super::QTensorEngine {
159    /// Select the external recurrent-state budget for subsequent cold model loads.
160    /// This arena is separate from the 42 MiB Sentinel arena.
161    pub fn set_ssm_state_byte_budget(&mut self, byte_budget: usize) {
162        self.ssm_state_byte_budget = byte_budget;
163    }
164
165    pub fn init_ssm_state_with_budget(
166        &mut self,
167        index: &GgufTensorIndex,
168        byte_budget: usize,
169    ) -> Result<usize, SsmStateError> {
170        use etalon_inference::ordinary::ssm_layout::{
171            allocate_recurrent_state, checked_ssm_layout_for,
172        };
173        let h = index.hyperparams.into();
174        let state = allocate_recurrent_state(
175            (0..index.hyperparams.n_layer).filter_map(|layer| {
176                let tensors = index.get_layer_tensors(layer);
177                tensors.is_hybrid_ssm_layer().then(|| {
178                    checked_ssm_layout_for(&tensors, &h, index.hyperparams.n_embd as usize)
179                        .map(|layout| layout.stride)
180                })
181            }),
182            byte_budget,
183        )?;
184        let count = state.len();
185        self.ssm_recurrent_state = state;
186        Ok(count)
187    }
188
189    /// Compatibility entry point. Loaders use fallible admission; a refused arena
190    /// remains empty and hybrid dispatch fails closed.
191    pub fn init_ssm_state(&mut self, index: &GgufTensorIndex) -> usize {
192        match self.init_ssm_state_with_budget(index, self.ssm_state_byte_budget) {
193            Ok(count) => count,
194            Err(_) => {
195                self.ssm_recurrent_state = Box::new([]);
196                0
197            }
198        }
199    }
200
201    pub fn reset_recurrent_state(&mut self) {
202        #[cfg(not(target_arch = "wasm32"))]
203        if let Some(session) = &mut self.gemma4 {
204            session.reset();
205        }
206        #[cfg(not(target_arch = "wasm32"))]
207        if let Some(session) = &mut self.qwen35_gguf {
208            session.reset();
209        }
210        #[cfg(not(target_arch = "wasm32"))]
211        if let Some(session) = &mut self.qwen35_ftw {
212            session.reset();
213        }
214        self.ssm_recurrent_state.fill(0.0);
215    }
216
217    /// Export a zero-heap snapshot of the current hybrid SSM / GatedDeltaNet recurrent state
218    /// (`ssm_recurrent_state`) into caller-supplied `out` (`f32` slice).
219    /// Returns `Some(self.ssm_recurrent_state.len())` when `out` has sufficient capacity.
220    #[must_use]
221    pub fn snapshot_recurrent_state_into(&self, out: &mut [f32]) -> Option<usize> {
222        #[cfg(not(target_arch = "wasm32"))]
223        if self.qwen35_gguf_required || self.gemma4_required {
224            return None;
225        }
226        let len = self.ssm_recurrent_state.len();
227        if out.len() < len {
228            return None;
229        }
230        out[..len].copy_from_slice(&self.ssm_recurrent_state);
231        Some(len)
232    }
233
234    /// Restore a hybrid SSM / GatedDeltaNet recurrent state snapshot from a `.p64` neural
235    /// extension cartridge or pre-warmed scene seed into `self.ssm_recurrent_state` (zero-heap).
236    #[must_use]
237    pub fn restore_recurrent_state_from_slice(&mut self, src: &[f32]) -> bool {
238        #[cfg(not(target_arch = "wasm32"))]
239        if self.qwen35_gguf_required || self.gemma4_required {
240            return false;
241        }
242        let len = self.ssm_recurrent_state.len();
243        if len == 0 || src.len() != len {
244            return false;
245        }
246        self.ssm_recurrent_state.copy_from_slice(src);
247        true
248    }
249
250    pub(crate) fn dispatch_hybrid_ssm_layer(
251        &mut self,
252        index: &GgufTensorIndex,
253        layer: u32,
254        hidden: &mut [f32],
255        emb_dim: usize,
256        tensors: &LayerTensors,
257        scratch_a: &mut [f32],
258        scratch_b: &mut [f32],
259    ) -> bool {
260        #[cfg(not(target_arch = "wasm32"))]
261        if self.qwen35_gguf_required || self.gemma4_required {
262            return false;
263        }
264        if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
265            #[cfg(not(target_arch = "wasm32"))]
266            return self.qwen35_attention(index, layer, hidden, emb_dim, tensors);
267            #[cfg(target_arch = "wasm32")]
268            return false;
269        }
270        HybridEngine { host: self }
271            .dispatch(index, layer, hidden, emb_dim, tensors, scratch_a, scratch_b)
272    }
273}
274#[cfg(test)]
275mod tests {
276    use super::*;
277    use crate::gguf_sharder::{GgufHyperparams, LayerTensors};
278
279    #[test]
280    fn ssm_layout_new_granite_style() {
281        let layout = SsmLayerStateLayout::new(4096, 128, 64, 1);
282        assert_eq!(layout.conv_offset, 0);
283        assert_eq!(layout.conv_len, CONV_HISTORY_DEPTH * 4096);
284        assert_eq!(layout.delta_offset, layout.conv_len);
285        assert_eq!(layout.delta_len, 128 * 64 * 1);
286        assert_eq!(layout.stride, layout.conv_len + layout.delta_len);
287    }
288
289    #[test]
290    fn ssm_layout_new_qwen_style() {
291        let layout = SsmLayerStateLayout::new(2048, 128, 128, 16);
292        assert_eq!(layout.conv_len, CONV_HISTORY_DEPTH * 2048);
293        assert_eq!(layout.delta_len, 128 * 128 * 16);
294        assert_eq!(layout.stride, layout.conv_len + layout.delta_len);
295    }
296
297    #[test]
298    fn ssm_layout_for_uses_explicit_ssm_inner_size() {
299        let mut h = GgufHyperparams::default();
300        h.ssm_inner_size = 1024;
301        h.ssm_state_size = 64;
302        h.ssm_group_count = 4;
303        h.n_embd = 512;
304        let tensors = LayerTensors::default();
305        let layout = ssm_layout_for(&tensors, &h.into(), 512);
306        assert_eq!(layout.conv_len, CONV_HISTORY_DEPTH * 1024);
307        assert!(layout.delta_len > 0);
308    }
309
310    #[test]
311    fn ssm_state_size_example_external_model_arena() {
312        // 96-layer Granite: stride per SSM layer = 3*4096 + 128*64 = 20480 f32
313        let layout = SsmLayerStateLayout::new(4096, 128, 64, 1);
314        let bytes = layout.stride * 96 * 4;
315        // 96 * 20480 * 4 = 7,864,320 < 8 MB
316        assert!(
317            bytes < 8 * 1024 * 1024,
318            "SSM example external state exceeds 8 MB: {bytes} bytes"
319        );
320    }
321    #[test]
322    fn reset_sequence_clears_recurrence_without_kv_layout() {
323        use crate::specialized_libs::computational_geometry::allocation_counter::assert_zero_alloc;
324        let mut engine = super::super::QTensorEngine::new();
325        engine.ssm_recurrent_state = vec![1.0, -2.0, 3.0].into_boxed_slice();
326        engine.kv_layout = None;
327        assert_zero_alloc("reset sequence", || engine.reset_kv_cache());
328        assert_eq!(&*engine.ssm_recurrent_state, &[0.0; 3]);
329    }
330
331    #[test]
332    fn attention_only_replacement_drops_old_recurrence() {
333        let mut engine = super::super::QTensorEngine::new();
334        engine.ssm_recurrent_state = vec![9.0; 8].into_boxed_slice();
335        let index = GgufTensorIndex {
336            entries: Vec::new(),
337            tensor_data_start: 0,
338            token_embd: None,
339            output_weight: None,
340            output_norm: None,
341            ple_ngram_embedding: None,
342            moe_routing: Default::default(),
343            ple_config: None,
344            hyperparams: GgufHyperparams::default(),
345            max_tensor_bytes: 0,
346            max_layer_tensor_bytes: 0,
347        };
348        assert_eq!(engine.init_ssm_state_with_budget(&index, 0), Ok(0));
349        assert!(engine.ssm_recurrent_state.is_empty());
350    }
351}