qualia_core_db/gguf_bridge/
moe_ffn.rs1use super::*;
4#[cfg(not(target_arch = "wasm32"))]
5use crate::inference::moe::ftw_loader::FtwModelPackage;
6#[cfg(not(target_arch = "wasm32"))]
7use std::sync::Arc;
8
9impl QTensorEngine {
10 #[cfg(not(target_arch = "wasm32"))]
12 pub fn adopt_ftw_package(&mut self, dir: &std::path::Path) -> Result<GgufLoadReport, String> {
13 self.adopt_ftw_package_with_budget(dir, 1024, 128 * 1024 * 1024)
14 }
15
16 #[cfg(not(target_arch = "wasm32"))]
20 pub fn adopt_ftw_package_with_budget(
21 &mut self,
22 dir: &std::path::Path,
23 max_context: usize,
24 model_state_byte_budget: usize,
25 ) -> Result<GgufLoadReport, String> {
26 let metadata_bytes = std::fs::metadata(dir.join("freetoken_weight.json"))
27 .map_err(|e| format!("FTW manifest metadata: {e}"))?
28 .len();
29 if max_context == 0 || metadata_bytes > model_state_byte_budget as u64 {
30 return Err("FTW model state/metadata budget exhausted".into());
31 }
32 let pkg = Arc::new(FtwModelPackage::open_from_dir(dir)?);
33 let index = pkg.tensor_index.clone();
34 let qwen35 = if index.hyperparams.architecture == etalon_model::inference::ARCH_QWEN35_MOE {
38 let plan = qwen35::ftw::admission::admit_known(
39 Arc::clone(&pkg),
40 dir,
41 max_context,
42 model_state_byte_budget,
43 )?;
44 let remaining = model_state_byte_budget
45 .checked_sub(plan.reserved_bytes())
46 .ok_or("Qwen plan exhausts caller byte budget")?;
47 Some(
48 qwen35::ftw::session::Session::new(plan, remaining)
49 .map_err(|e| format!("Qwen session admission: {e:?}"))?,
50 )
51 } else {
52 index.hyperparams.decode_supported()?;
53 self.init_ssm_state_with_budget(&index, self.ssm_state_byte_budget)
54 .map_err(|e| e.to_string())?;
55 None
56 };
57 self.qwen35_gguf = None;
58 self.qwen35_gguf_required = false;
59 self.gemma4 = None;
60 self.gemma4_required = false;
61 self.qwen35_ftw = qwen35;
62 self.tensor_index_cache = Some(index.clone());
63 self.hyperparams = index.hyperparams;
64 self.tensor_data_offset = index.tensor_data_start;
65 self.max_tensor_bytes = index.max_layer_tensor_bytes;
66 #[cfg(feature = "gpu-runtime")]
67 self.invalidate_resident_projection_weights();
68 #[cfg(not(target_arch = "wasm32"))]
69 {
70 self.prepared_native_bytes = None;
71 self.prepared_native_sources = None;
72 }
73 self.gguf_mmap = pkg.shards.first().cloned();
74 self.ftw_package = Some(pkg);
75 if self.qwen35_ftw.is_some() {
76 self.ssm_recurrent_state = Box::new([]);
79 self.kv_layout = None;
80 self.kv_cache_cpu = None;
81 } else {
82 self.ensure_kv_cache(&index.hyperparams);
83 }
84
85 Ok(GgufLoadReport {
86 mapped_bytes: self
87 .ftw_package
88 .as_ref()
89 .map(|p| p.manifest.total_bytes)
90 .unwrap_or(0),
91 tensor_data_offset: self.tensor_data_offset,
92 n_layer: self.hyperparams.n_layer,
93 n_head: self.hyperparams.n_head,
94 n_kv_head: self.hyperparams.n_kv_head,
95 max_tensor_bytes: self.max_tensor_bytes,
96 kv_cache_bytes: self.kv_cache_bytes(),
97 directml_enabled: false,
98 })
99 }
100
101 pub fn fetch_tensor_raw_bytes<'a>(
103 &'a self,
104 info: &crate::gguf_sharder::GgufTensorInfo,
105 ) -> Option<&'a [u8]> {
106 #[cfg(not(target_arch = "wasm32"))]
107 {
108 let len = crate::ggml_quants::tensor_byte_len(info)?;
109 if let Some(ref pkg) = self.ftw_package {
110 if let Some(b) = pkg.fetch_by_global_offset(info.byte_offset, len) {
111 return Some(b);
112 }
113 }
114 }
115 let mmap = self.model_bytes()?;
116 crate::ggml_quants::fetch_tensor_bytes(mmap, self.tensor_data_offset, info).ok()
117 }
118
119 pub(crate) fn dequant_norm_tensor_into(
121 &self,
122 info: &crate::gguf_sharder::GgufTensorInfo,
123 out: &mut [f32],
124 ) -> usize {
125 #[cfg(not(target_arch = "wasm32"))]
126 if let Some(raw) = self.fetch_tensor_raw_bytes(info) {
127 return cpu_ops::dequant_norm_from_raw(raw, info, out);
128 }
129 if let Some(mmap) = self.model_bytes() {
130 return cpu_ops::dequant_norm_row_into(mmap, self.tensor_data_offset, info, out);
131 }
132 0
133 }
134
135 pub(crate) fn dispatch_moe_ffn(
136 &mut self,
137 index: &crate::gguf_sharder::GgufTensorIndex,
138 emb_dim: usize,
139 tensors: &crate::gguf_sharder::LayerTensors,
140 scratch_a: &mut [f32],
141 ffn_input: &[f32],
142 ) -> bool {
143 etalon_inference::ordinary::moe::execute_moe(
144 self, index, emb_dim, tensors, scratch_a, ffn_input,
145 )
146 }
147
148 pub fn dispatch_moe_step_routed<F>(
150 &mut self,
151 emb_dim: usize,
152 router_weights: &[f32],
153 num_experts: usize,
154 topk: usize,
155 expert_fetcher: F,
156 shared_expert: Option<&crate::inference::moe::dispatch::ExpertWeightView<'_>>,
157 shared_gate_weight: f32,
158 ffn_input: &[f32],
159 scratch_a: &mut [f32],
160 ) -> bool
161 where
162 F: FnMut(u16) -> Option<crate::inference::moe::dispatch::ExpertWeightView<'static>>,
163 {
164 crate::inference::moe::dispatch::dispatch_moe_step(
165 &ffn_input[..emb_dim],
166 emb_dim,
167 router_weights,
168 num_experts,
169 topk,
170 expert_fetcher,
171 shared_expert,
172 shared_gate_weight,
173 &mut scratch_a[..emb_dim],
174 )
175 .is_ok()
176 }
177
178 pub fn dispatch_clustered_moe_operator(
180 &mut self,
181 op: &crate::inference::operator_runtime::ClusteredMoEOperator,
182 routed_experts: &[crate::inference::operator_runtime::RoutedExpert],
183 ffn_input: &[f32],
184 output: &mut [f32],
185 scratch: &mut crate::inference::operator_runtime::ClusteredMoeScratch<'_>,
186 ) -> bool {
187 op.dispatch(routed_experts, ffn_input, output, scratch)
188 .is_ok()
189 }
190}