1use super::*;
3
4pub struct QTensorEngine {
5 #[cfg(not(target_arch = "wasm32"))]
8 pub(super) prepared_native_bytes: Option<ModelByteHandle>,
9 #[cfg(not(target_arch = "wasm32"))]
10 pub(super) prepared_native_sources: Option<NativeModelSources>,
11 #[cfg(not(target_arch = "wasm32"))]
13 pub(super) qwen35_ftw: Option<qwen35::ftw::session::Session>,
14 #[cfg(not(target_arch = "wasm32"))]
15 pub(super) qwen35_gguf: Option<qwen35::gguf::Session>,
16 #[cfg(not(target_arch = "wasm32"))]
17 pub(super) qwen35_gguf_required: bool,
18 #[cfg(not(target_arch = "wasm32"))]
20 pub(super) gemma4: Option<gemma4::Session>,
21 #[cfg(not(target_arch = "wasm32"))]
22 pub(super) gemma4_required: bool,
23 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
25 pub(super) device: etalon::Device,
26 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
27 pub(super) queue: etalon::Queue,
28 #[cfg(feature = "gpu-runtime")]
29 pub pipeline: etalon::ComputePipeline,
30 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
32 pub mmv_q8_0_pipeline: etalon::ComputePipeline,
33 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
34 pub(super) native_pipeline_cache: Option<etalon::PipelineCache>,
35 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
36 pub(super) pipeline_bind_layout: etalon::BindGroupLayout,
37 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
41 pub(crate) coop_gemv_pipeline: etalon::ComputePipeline,
42 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
43 pub(super) coop_gemv_bind_layout: etalon::BindGroupLayout,
44 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
46 pub(crate) coop_gemv_mr_pipeline: etalon::ComputePipeline,
47 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
49 pub(crate) coop_gemv_residual_pipeline: etalon::ComputePipeline,
50 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
51 pub(super) coop_gemv_residual_bind_layout: etalon::BindGroupLayout,
52 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
54 pub(crate) coop_gemv_residual_mr_pipeline: etalon::ComputePipeline,
55 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
57 pub(crate) coop_gemv_warp_pipeline: etalon::ComputePipeline,
58 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
59 pub(crate) coop_gemv_residual_warp_pipeline: etalon::ComputePipeline,
60 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
62 pub(super) mock_pipeline: etalon::ComputePipeline,
63 #[cfg(feature = "gpu-runtime")]
65 pub embedding_pipeline: etalon::ComputePipeline,
66 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
67 pub(super) embedding_bind_layout: etalon::BindGroupLayout,
68 pub is_initialized: bool,
69 #[cfg(target_os = "windows")]
71 pub dml: Option<crate::directml_bridge::DmlDevice>,
72 #[cfg(not(target_arch = "wasm32"))]
74 pub gguf_mmap: Option<Arc<memmap2::Mmap>>,
75 #[cfg(target_arch = "wasm32")]
76 pub gguf_mmap: Option<Arc<[u8]>>,
77 #[cfg(target_arch = "wasm32")]
79 pub cached_tokenizer: Option<crate::gguf_sharder::GgufTokenizer>,
80 #[cfg(target_arch = "wasm32")]
82 pub cached_tensor_index: Option<crate::gguf_sharder::GgufTensorIndex>,
83 #[cfg(target_arch = "wasm32")]
85 pub cached_token_embd: Option<Arc<[u8]>>,
86 #[cfg(target_arch = "wasm32")]
88 pub p64_resident: Option<Arc<[u8]>>,
89 #[cfg(not(target_arch = "wasm32"))]
91 pub p64_index: Option<crate::p64_weight::P64TensorIndex>,
92 #[cfg(not(target_arch = "wasm32"))]
94 pub tensor_index_cache: Option<crate::gguf_sharder::GgufTensorIndex>,
95 #[cfg(not(target_arch = "wasm32"))]
97 pub ftw_package: Option<Arc<crate::inference::moe::ftw_loader::FtwModelPackage>>,
98
99 pub tensor_data_offset: u64,
101 pub hyperparams: crate::gguf_sharder::GgufHyperparams,
102 pub max_tensor_bytes: usize,
103 #[cfg(feature = "gpu-runtime")]
105 pub(super) gemm_input_buf: Option<etalon::Buffer>,
106 #[cfg(feature = "gpu-runtime")]
107 pub(super) gemm_weight_buf: Option<etalon::Buffer>,
108 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
110 pub(super) mc8_weight_arena: Option<Mc8WeightArenaBufs>,
111 #[cfg(target_arch = "wasm32")]
114 pub(super) mc8_weights_resident: bool,
115 #[cfg(target_arch = "wasm32")]
117 pub(super) mc8_weight_role_stride: [u64; 7],
118 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
120 pub(super) gemm_weight_buf_b: Option<etalon::Buffer>,
121 #[cfg(feature = "gpu-runtime")]
122 pub(super) gemm_output_buf: Option<etalon::Buffer>,
123 #[cfg(feature = "gpu-runtime")]
124 pub(super) gemm_params_buf: Option<etalon::Buffer>,
125 #[cfg(feature = "gpu-runtime")]
126 pub(super) gemm_output_staging: Option<etalon::Buffer>,
127 #[cfg(feature = "gpu-runtime")]
131 pub(super) output_topk_pipeline: Option<etalon::ComputePipeline>,
132 #[cfg(feature = "gpu-runtime")]
133 pub(super) output_topk_bind_layout: Option<etalon::BindGroupLayout>,
134 #[cfg(feature = "gpu-runtime")]
135 pub(super) topk_cand_val_buf: Option<etalon::Buffer>,
136 #[cfg(feature = "gpu-runtime")]
137 pub(super) topk_cand_idx_buf: Option<etalon::Buffer>,
138 #[cfg(feature = "gpu-runtime")]
139 pub(super) topk_cand_staging: Option<etalon::Buffer>,
140 #[cfg(feature = "gpu-runtime")]
141 pub(super) topk_params_buf: Option<etalon::Buffer>,
142 #[cfg(feature = "gpu-runtime")]
144 pub(super) gemm_aux_buf: Option<etalon::Buffer>,
145 #[cfg(feature = "gpu-runtime")]
147 pub(super) gemm_ffn_buf: Option<etalon::Buffer>,
148 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
150 pub(super) prefill_scratch_buf: Option<etalon::Buffer>,
151 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
153 pub(super) prefill_work_buf_a: Option<etalon::Buffer>,
154 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
155 pub(super) prefill_work_buf_b: Option<etalon::Buffer>,
156 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
158 pub(super) mc8_q_proj_buf: Option<etalon::Buffer>,
159 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
160 pub(super) mc8_k_proj_buf: Option<etalon::Buffer>,
161 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
162 pub(super) mc8_v_proj_buf: Option<etalon::Buffer>,
163 pub(super) gemm_max_out_dim: u32,
164 pub(super) gemm_max_input_floats: usize,
165 pub(super) kv_layout: Option<KvCacheLayout>,
167 #[cfg(feature = "gpu-runtime")]
168 pub(super) kv_cache_gpu: Option<etalon::Buffer>,
169 pub(super) kv_cache_cpu: Option<Box<[f32]>>,
171 #[cfg(feature = "gpu-runtime")]
172 pub(super) attention_pipeline: etalon::ComputePipeline,
173 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
174 pub(super) attention_bind_layout: etalon::BindGroupLayout,
175 #[cfg(feature = "gpu-runtime")]
176 pub(super) attention_params_buf: Option<etalon::Buffer>,
177 #[cfg(feature = "gpu-runtime")]
178 pub(super) attention_mask_buf: Option<etalon::Buffer>,
179 #[cfg(feature = "gpu-runtime")]
181 pub(super) elem_rms_norm_pipeline: etalon::ComputePipeline,
182 #[cfg(feature = "gpu-runtime")]
183 pub(super) elem_silu_mul_pipeline: etalon::ComputePipeline,
184 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
185 pub(super) elem_silu_mul_bind_layout: etalon::BindGroupLayout,
186 #[cfg(feature = "gpu-runtime")]
187 pub(super) elem_add_residual_pipeline: etalon::ComputePipeline,
188 #[cfg(feature = "gpu-runtime")]
189 pub(super) elem_params_buf: Option<etalon::Buffer>,
190 #[cfg(feature = "gpu-runtime")]
191 pub(super) norm_weight_buf: Option<etalon::Buffer>,
192 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
194 pub(super) mc8_gemm_bind_layout: etalon::BindGroupLayout,
195 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
196 pub(super) mc8_elem_bind_layout: etalon::BindGroupLayout,
197 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
198 pub(super) mc8_attn_bind_layout: etalon::BindGroupLayout,
199 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
201 pub(super) mc8_ffn_fused_bind_layout: etalon::BindGroupLayout,
202 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
203 pub(super) mc8_ffn_fused_pipeline: etalon::ComputePipeline,
204 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
206 pub(super) ffn_fused_bind_layout: etalon::BindGroupLayout,
207 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
209 pub(super) ffn_fused_pipeline: etalon::ComputePipeline,
210 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
212 pub(super) ffn_fused_coop_pipeline: etalon::ComputePipeline,
213 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
215 pub(super) ffn_fused_mr_pipeline: etalon::ComputePipeline,
216 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
218 pub(super) ffn_fused_warp_pipeline: etalon::ComputePipeline,
219 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
221 pub(super) dual_gemv_pipeline: etalon::ComputePipeline,
222 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
224 pub(super) dual_gemv_mr_pipeline: etalon::ComputePipeline,
225 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
226 pub(super) dual_gemv_bind_layout: etalon::BindGroupLayout,
227 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
229 pub(super) triple_gemv_pipeline: etalon::ComputePipeline,
230 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
231 pub(super) triple_gemv_bind_layout: etalon::BindGroupLayout,
232 #[cfg(feature = "gpu-runtime")]
237 pub(super) mc8_logits_resident_buf: Option<etalon::Buffer>,
238 pub(super) mc8_logits_row_bytes: u32,
239 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
244 pub(super) ternary_ffn: Option<crate::ternary_gpu::TernaryFfnResident>,
245 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
249 pub(super) projection_cold_byte_budget: usize,
250 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
251 pub(super) projection_readback: etalon_inference::projections::readback::ReadbackCompletion,
252 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
253 pub(super) gemm_resident_weights: std::sync::Mutex<super::gemm::ResidentProjectionCache>,
254 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
258 pub(super) ffn_fused_params: Option<etalon::Buffer>,
259 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
263 pub(super) attention_kv_gemm_params: Option<etalon::Buffer>,
264 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
265 pub(super) attention_kv_params: Option<etalon::Buffer>,
266 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
270 pub(super) mc8_norm_resident_buf: Option<etalon::Buffer>,
271 #[cfg(target_arch = "wasm32")]
272 pub(super) mc8_norm_stride: u32,
273 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
275 pub(super) resident_decode: resident_decode::ResidentDecodeState,
276 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
278 pub(super) cuda_decode_plan: cuda_decode_plan::CudaDecodePlanState,
279 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
281 pub(super) prefill_arena: prefill_arena::PrefillArenaState,
282 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
284 pub(super) verify_arena: verify_arena::VerifyArenaState,
285 #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
290 pub(super) mc8_bg_cache: std::sync::Mutex<std::collections::HashMap<u64, etalon::BindGroup>>,
291 pub(crate) ssm_recurrent_state: Box<[f32]>,
297 pub(crate) ssm_state_byte_budget: usize,
298}