Skip to main content

qualia_core_db/gguf_bridge/
moe_ffn.rs

1//! MoE FFN dispatch: routing, expert GEMV accumulation, and FTW package adoption.
2
3use super::*;
4#[cfg(not(target_arch = "wasm32"))]
5use crate::inference::moe::ftw_loader::FtwModelPackage;
6#[cfg(not(target_arch = "wasm32"))]
7use std::sync::Arc;
8
9impl QTensorEngine {
10    /// Adopt an FTW multi-shard model directory for native MoE execution.
11    #[cfg(not(target_arch = "wasm32"))]
12    pub fn adopt_ftw_package(&mut self, dir: &std::path::Path) -> Result<GgufLoadReport, String> {
13        self.adopt_ftw_package_with_budget(dir, 1024, 128 * 1024 * 1024)
14    }
15
16    /// Select the external whole-model state budget and context capacity for
17    /// admitted Qwen execution. This is separate from the legacy SSM-only
18    /// budget and the 42 MiB Sentinel arena; the default reserves at most 128 MiB.
19    #[cfg(not(target_arch = "wasm32"))]
20    pub fn adopt_ftw_package_with_budget(
21        &mut self,
22        dir: &std::path::Path,
23        max_context: usize,
24        model_state_byte_budget: usize,
25    ) -> Result<GgufLoadReport, String> {
26        let metadata_bytes = std::fs::metadata(dir.join("freetoken_weight.json"))
27            .map_err(|e| format!("FTW manifest metadata: {e}"))?
28            .len();
29        if max_context == 0 || metadata_bytes > model_state_byte_budget as u64 {
30            return Err("FTW model state/metadata budget exhausted".into());
31        }
32        let pkg = Arc::new(FtwModelPackage::open_from_dir(dir)?);
33        let index = pkg.tensor_index.clone();
34        // A Qwen artifact needs an architecture-owned plan, not permission to
35        // run through the generic Llama forward. Automatic admission is bound
36        // to a reviewed exporter receipt; other FTW formats retain their guard.
37        let qwen35 = if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
38            let plan = qwen35::ftw::admission::admit_known(
39                Arc::clone(&pkg),
40                dir,
41                max_context,
42                model_state_byte_budget,
43            )?;
44            let remaining = model_state_byte_budget
45                .checked_sub(plan.reserved_bytes())
46                .ok_or("Qwen plan exhausts caller byte budget")?;
47            Some(
48                qwen35::ftw::session::Session::new(plan, remaining)
49                    .map_err(|e| format!("Qwen session admission: {e:?}"))?,
50            )
51        } else {
52            index.hyperparams.decode_supported()?;
53            self.init_ssm_state_with_budget(&index, self.ssm_state_byte_budget)
54                .map_err(|e| e.to_string())?;
55            None
56        };
57        self.qwen35_gguf = None;
58        self.qwen35_gguf_required = false;
59        self.gemma4 = None;
60        self.gemma4_required = false;
61        self.qwen35_ftw = qwen35;
62        self.tensor_index_cache = Some(index.clone());
63        self.hyperparams = index.hyperparams;
64        self.tensor_data_offset = index.tensor_data_start;
65        self.max_tensor_bytes = index.max_layer_tensor_bytes;
66        #[cfg(feature = "gpu-runtime")]
67        self.invalidate_resident_projection_weights();
68        #[cfg(not(target_arch = "wasm32"))]
69        {
70            self.prepared_native_bytes = None;
71            self.prepared_native_sources = None;
72        }
73        self.gguf_mmap = pkg.shards.first().cloned();
74        self.ftw_package = Some(pkg);
75        if self.qwen35_ftw.is_some() {
76            // The architecture owner includes both recurrent and full-attention
77            // storage in its checked budget, outside the Sentinel arena.
78            self.ssm_recurrent_state = Box::new([]);
79            self.kv_layout = None;
80            self.kv_cache_cpu = None;
81        } else {
82            self.ensure_kv_cache(&index.hyperparams);
83        }
84
85        Ok(GgufLoadReport {
86            mapped_bytes: self
87                .ftw_package
88                .as_ref()
89                .map(|p| p.manifest.total_bytes)
90                .unwrap_or(0),
91            tensor_data_offset: self.tensor_data_offset,
92            n_layer: self.hyperparams.n_layer,
93            n_head: self.hyperparams.n_head,
94            n_kv_head: self.hyperparams.n_kv_head,
95            max_tensor_bytes: self.max_tensor_bytes,
96            kv_cache_bytes: self.kv_cache_bytes(),
97            directml_enabled: false,
98        })
99    }
100
101    /// Read raw bytes for a tensor from either the FTW package shards or the primary GGUF mmap.
102    pub fn fetch_tensor_raw_bytes<'a>(
103        &'a self,
104        info: &crate::gguf_sharder::GgufTensorInfo,
105    ) -> Option<&'a [u8]> {
106        #[cfg(not(target_arch = "wasm32"))]
107        {
108            let len = crate::ggml_quants::tensor_byte_len(info)?;
109            if let Some(ref pkg) = self.ftw_package {
110                if let Some(b) = pkg.fetch_by_global_offset(info.byte_offset, len) {
111                    return Some(b);
112                }
113            }
114        }
115        let mmap = self.model_bytes()?;
116        crate::ggml_quants::fetch_tensor_bytes(mmap, self.tensor_data_offset, info).ok()
117    }
118
119    /// Dequantize norm weights from FTW shards or GGUF mmap into caller slice.
120    pub(crate) fn dequant_norm_tensor_into(
121        &self,
122        info: &crate::gguf_sharder::GgufTensorInfo,
123        out: &mut [f32],
124    ) -> usize {
125        #[cfg(not(target_arch = "wasm32"))]
126        if let Some(raw) = self.fetch_tensor_raw_bytes(info) {
127            return cpu_ops::dequant_norm_from_raw(raw, info, out);
128        }
129        if let Some(mmap) = self.model_bytes() {
130            return cpu_ops::dequant_norm_row_into(mmap, self.tensor_data_offset, info, out);
131        }
132        0
133    }
134
135    pub(crate) fn dispatch_moe_ffn(
136        &mut self,
137        index: &crate::gguf_sharder::GgufTensorIndex,
138        emb_dim: usize,
139        tensors: &crate::gguf_sharder::LayerTensors,
140        scratch_a: &mut [f32],
141        ffn_input: &[f32],
142    ) -> bool {
143        etalon_inference::ordinary::moe::execute_moe(
144            self, index, emb_dim, tensors, scratch_a, ffn_input,
145        )
146    }
147
148    /// Dispatches MoE layer using the zero-heap `dispatch_moe_step` driver with expert fetcher (EOS-070).
149    pub fn dispatch_moe_step_routed<F>(
150        &mut self,
151        emb_dim: usize,
152        router_weights: &[f32],
153        num_experts: usize,
154        topk: usize,
155        expert_fetcher: F,
156        shared_expert: Option<&crate::inference::moe::dispatch::ExpertWeightView<'_>>,
157        shared_gate_weight: f32,
158        ffn_input: &[f32],
159        scratch_a: &mut [f32],
160    ) -> bool
161    where
162        F: FnMut(u16) -> Option<crate::inference::moe::dispatch::ExpertWeightView<'static>>,
163    {
164        crate::inference::moe::dispatch::dispatch_moe_step(
165            &ffn_input[..emb_dim],
166            emb_dim,
167            router_weights,
168            num_experts,
169            topk,
170            expert_fetcher,
171            shared_expert,
172            shared_gate_weight,
173            &mut scratch_a[..emb_dim],
174        )
175        .is_ok()
176    }
177
178    /// Dispatches MoE layer using the clustered shared-down MoE operator (EOS-071).
179    pub fn dispatch_clustered_moe_operator(
180        &mut self,
181        op: &crate::inference::operator_runtime::ClusteredMoEOperator,
182        routed_experts: &[crate::inference::operator_runtime::RoutedExpert],
183        ffn_input: &[f32],
184        output: &mut [f32],
185        scratch: &mut crate::inference::operator_runtime::ClusteredMoeScratch<'_>,
186    ) -> bool {
187        op.dispatch(routed_experts, ffn_input, output, scratch)
188            .is_ok()
189    }
190}