Skip to main content

qualia_core_db/gguf_bridge/
state.rs

1// Engine field ownership; moved mechanically from the parent module.
2use super::*;
3
4pub struct QTensorEngine {
5    /// Private prepared backing retains lease exclusion or immutable snapshot
6    /// ownership. Public legacy mappings cannot override this execution source.
7    #[cfg(not(target_arch = "wasm32"))]
8    pub(super) prepared_native_bytes: Option<ModelByteHandle>,
9    #[cfg(not(target_arch = "wasm32"))]
10    pub(super) prepared_native_sources: Option<NativeModelSources>,
11    /// Architecture-owned Qwen3.5 state; generic Llama execution never admits it.
12    #[cfg(not(target_arch = "wasm32"))]
13    pub(super) qwen35_ftw: Option<qwen35::ftw::session::Session>,
14    #[cfg(not(target_arch = "wasm32"))]
15    pub(super) qwen35_gguf: Option<qwen35::gguf::Session>,
16    #[cfg(not(target_arch = "wasm32"))]
17    pub(super) qwen35_gguf_required: bool,
18    /// Checked token-aware Gemma graph owns its per-layer KV and PLE state.
19    #[cfg(not(target_arch = "wasm32"))]
20    pub(super) gemma4: Option<gemma4::Session>,
21    #[cfg(not(target_arch = "wasm32"))]
22    pub(super) gemma4_required: bool,
23    /// Browser WASM keeps a private device; native reuses `gpu_context::shared_gpu()`.
24    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
25    pub(super) device: etalon::Device,
26    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
27    pub(super) queue: etalon::Queue,
28    #[cfg(feature = "gpu-runtime")]
29    pub pipeline: etalon::ComputePipeline,
30    /// WASM multi-row Q8_0 GEMV (llama.cpp-style: 64 thr, 4 rows/WG, u32 packed reads).
31    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
32    pub mmv_q8_0_pipeline: etalon::ComputePipeline,
33    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
34    pub(super) native_pipeline_cache: Option<etalon::PipelineCache>,
35    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
36    pub(super) pipeline_bind_layout: etalon::BindGroupLayout,
37    /// 0.0.21: cooperative GEMV (one workgroup per output row, shared-memory reduction). Same shader
38    /// MODULE as `pipeline`, entry point `coop_gemv` / `coop_gemv_sg`. Selected per-call when
39    /// `llm_bench::coop_gemv_enabled()`. Native only (the wasm decode path is the MC8 arena).
40    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
41    pub(crate) coop_gemv_pipeline: etalon::ComputePipeline,
42    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
43    pub(super) coop_gemv_bind_layout: etalon::BindGroupLayout,
44    /// Multi-row coop GEMV (8 rows/WG) for Q4_K_SOA large n_out — see `coop_gemv_mr`.
45    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
46    pub(crate) coop_gemv_mr_pipeline: etalon::ComputePipeline,
47    /// GEMV + residual add in one dispatch (O-proj / down-proj in resident mega-pass).
48    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
49    pub(crate) coop_gemv_residual_pipeline: etalon::ComputePipeline,
50    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
51    pub(super) coop_gemv_residual_bind_layout: etalon::BindGroupLayout,
52    /// Multi-row residual GEMV for Q4_K_SOA.
53    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
54    pub(crate) coop_gemv_residual_mr_pipeline: etalon::ComputePipeline,
55    /// Warp GEMV (32 thr/row) for Q4_K_SOA.
56    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
57    pub(crate) coop_gemv_warp_pipeline: etalon::ComputePipeline,
58    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
59    pub(crate) coop_gemv_residual_warp_pipeline: etalon::ComputePipeline,
60    /// Legacy f32×f32 mock block for offset-0 `QTensor` fallback (no mmap).
61    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
62    pub(super) mock_pipeline: etalon::ComputePipeline,
63    /// GPU-side Q6_K embedding dequant + matmul (zero CPU dequant).
64    #[cfg(feature = "gpu-runtime")]
65    pub embedding_pipeline: etalon::ComputePipeline,
66    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
67    pub(super) embedding_bind_layout: etalon::BindGroupLayout,
68    pub is_initialized: bool,
69    /// DirectML device — Some on Windows when DirectML 1.15 is linked.
70    #[cfg(target_os = "windows")]
71    pub dml: Option<crate::directml_bridge::DmlDevice>,
72    /// Memory-mapped GGUF file (set after `load_gguf`).
73    #[cfg(not(target_arch = "wasm32"))]
74    pub gguf_mmap: Option<Arc<memmap2::Mmap>>,
75    #[cfg(target_arch = "wasm32")]
76    pub gguf_mmap: Option<Arc<[u8]>>,
77    /// WASM: cached tokenizer extracted from gguf_mmap before dropping it.
78    #[cfg(target_arch = "wasm32")]
79    pub cached_tokenizer: Option<crate::gguf_sharder::GgufTokenizer>,
80    /// WASM: cached tensor index extracted from gguf_mmap before dropping it.
81    #[cfg(target_arch = "wasm32")]
82    pub cached_tensor_index: Option<crate::gguf_sharder::GgufTensorIndex>,
83    /// WASM: raw bytes of the token_embd tensor (for embedding lookup after dropping gguf_mmap).
84    #[cfg(target_arch = "wasm32")]
85    pub cached_token_embd: Option<Arc<[u8]>>,
86    /// Resident P64 container bytes.
87    #[cfg(target_arch = "wasm32")]
88    pub p64_resident: Option<Arc<[u8]>>,
89    /// Cached P64 index after `adopt_resident_p64_*` — decode must not re-CRC the container.
90    #[cfg(not(target_arch = "wasm32"))]
91    pub p64_index: Option<crate::p64_weight::P64TensorIndex>,
92    /// Cached synthetic GGUF index built from `p64_index` (or from GGUF parse).
93    #[cfg(not(target_arch = "wasm32"))]
94    pub tensor_index_cache: Option<crate::gguf_sharder::GgufTensorIndex>,
95    /// FTW multi-shard package for native MoE execution.
96    #[cfg(not(target_arch = "wasm32"))]
97    pub ftw_package: Option<Arc<crate::inference::moe::ftw_loader::FtwModelPackage>>,
98
99    /// Byte offset into the mmap where tensor data begins.
100    pub tensor_data_offset: u64,
101    pub hyperparams: crate::gguf_sharder::GgufHyperparams,
102    pub max_tensor_bytes: usize,
103    /// Reused layer staging buffers (one layer in VRAM at a time).
104    #[cfg(feature = "gpu-runtime")]
105    pub(super) gemm_input_buf: Option<etalon::Buffer>,
106    #[cfg(feature = "gpu-runtime")]
107    pub(super) gemm_weight_buf: Option<etalon::Buffer>,
108    /// MC8 Part 3t: disjoint per-role weight arena (prefill single-submit).
109    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
110    pub(super) mc8_weight_arena: Option<Mc8WeightArenaBufs>,
111    /// MC8 Part 3x: when set, the 7 role buffers hold ALL layers' weights (uploaded once);
112    /// hot-path encoders bind a per-layer sub-range instead of re-`write_buffer`ing per forward.
113    #[cfg(target_arch = "wasm32")]
114    pub(super) mc8_weights_resident: bool,
115    /// Per-role 256-byte-aligned per-layer stride (bytes), indexed by `Mc8WeightRole::idx()`.
116    #[cfg(target_arch = "wasm32")]
117    pub(super) mc8_weight_role_stride: [u64; 7],
118    /// Legacy decode-path ping-pong (decode tail not on weight arena yet).
119    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
120    pub(super) gemm_weight_buf_b: Option<etalon::Buffer>,
121    #[cfg(feature = "gpu-runtime")]
122    pub(super) gemm_output_buf: Option<etalon::Buffer>,
123    #[cfg(feature = "gpu-runtime")]
124    pub(super) gemm_params_buf: Option<etalon::Buffer>,
125    #[cfg(feature = "gpu-runtime")]
126    pub(super) gemm_output_staging: Option<etalon::Buffer>,
127    // A1a (STELLAR §A): persistent GPU top-k output-projection pipeline + small candidate buffers.
128    // Lets the output logits stay on-GPU (top-k over them, read back only K pairs) instead of the
129    // 196 KB/token full-logit readback. Created once in `ensure_gemm_buffers`.
130    #[cfg(feature = "gpu-runtime")]
131    pub(super) output_topk_pipeline: Option<etalon::ComputePipeline>,
132    #[cfg(feature = "gpu-runtime")]
133    pub(super) output_topk_bind_layout: Option<etalon::BindGroupLayout>,
134    #[cfg(feature = "gpu-runtime")]
135    pub(super) topk_cand_val_buf: Option<etalon::Buffer>,
136    #[cfg(feature = "gpu-runtime")]
137    pub(super) topk_cand_idx_buf: Option<etalon::Buffer>,
138    #[cfg(feature = "gpu-runtime")]
139    pub(super) topk_cand_staging: Option<etalon::Buffer>,
140    #[cfg(feature = "gpu-runtime")]
141    pub(super) topk_params_buf: Option<etalon::Buffer>,
142    /// MC8 FFN / attention scratch (gate, up, o_proj).
143    #[cfg(feature = "gpu-runtime")]
144    pub(super) gemm_aux_buf: Option<etalon::Buffer>,
145    /// MC8 SwiGLU up-projection scratch (cannot alias gemm_output/work — in-place GEMM invalid).
146    #[cfg(feature = "gpu-runtime")]
147    pub(super) gemm_ffn_buf: Option<etalon::Buffer>,
148    /// Batched prefill RMS output (same span as `gemm_input_buf`; avoids in-place on batch_buf).
149    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
150    pub(super) prefill_scratch_buf: Option<etalon::Buffer>,
151    /// Strided prefill ping-pong rows (`PREFILL_CHUNK_SIZE × row_stride` floats each).
152    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
153    pub(super) prefill_work_buf_a: Option<etalon::Buffer>,
154    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
155    pub(super) prefill_work_buf_b: Option<etalon::Buffer>,
156    /// Phase 5.5: Q/K/V projection scratch (parallel-GEMM output → lightweight attention shader).
157    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
158    pub(super) mc8_q_proj_buf: Option<etalon::Buffer>,
159    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
160    pub(super) mc8_k_proj_buf: Option<etalon::Buffer>,
161    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
162    pub(super) mc8_v_proj_buf: Option<etalon::Buffer>,
163    pub(super) gemm_max_out_dim: u32,
164    pub(super) gemm_max_input_floats: usize,
165    /// Static KV ring-buffer (allocated once at `load_gguf`).
166    pub(super) kv_layout: Option<KvCacheLayout>,
167    #[cfg(feature = "gpu-runtime")]
168    pub(super) kv_cache_gpu: Option<etalon::Buffer>,
169    /// CPU mirror for quantized-attention fallback (no growth during decode).
170    pub(super) kv_cache_cpu: Option<Box<[f32]>>,
171    #[cfg(feature = "gpu-runtime")]
172    pub(super) attention_pipeline: etalon::ComputePipeline,
173    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
174    pub(super) attention_bind_layout: etalon::BindGroupLayout,
175    #[cfg(feature = "gpu-runtime")]
176    pub(super) attention_params_buf: Option<etalon::Buffer>,
177    #[cfg(feature = "gpu-runtime")]
178    pub(super) attention_mask_buf: Option<etalon::Buffer>,
179    /// MC8 elementwise GPU ops (RMSNorm / SiLU×mul / residual).
180    #[cfg(feature = "gpu-runtime")]
181    pub(super) elem_rms_norm_pipeline: etalon::ComputePipeline,
182    #[cfg(feature = "gpu-runtime")]
183    pub(super) elem_silu_mul_pipeline: etalon::ComputePipeline,
184    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
185    pub(super) elem_silu_mul_bind_layout: etalon::BindGroupLayout,
186    #[cfg(feature = "gpu-runtime")]
187    pub(super) elem_add_residual_pipeline: etalon::ComputePipeline,
188    #[cfg(feature = "gpu-runtime")]
189    pub(super) elem_params_buf: Option<etalon::Buffer>,
190    #[cfg(feature = "gpu-runtime")]
191    pub(super) norm_weight_buf: Option<etalon::Buffer>,
192    /// MC8 Part 3s: dynamic-offset bind group layouts (uniform race elimination).
193    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
194    pub(super) mc8_gemm_bind_layout: etalon::BindGroupLayout,
195    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
196    pub(super) mc8_elem_bind_layout: etalon::BindGroupLayout,
197    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
198    pub(super) mc8_attn_bind_layout: etalon::BindGroupLayout,
199    /// Phase 5 dispatch fusion: SwiGLU expansion (gate · SiLU · up) collapsed into one pass.
200    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
201    pub(super) mc8_ffn_fused_bind_layout: etalon::BindGroupLayout,
202    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
203    pub(super) mc8_ffn_fused_pipeline: etalon::ComputePipeline,
204    /// Native T-A1: same fused FFN expansion, static uniform (resident mega-pass).
205    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
206    pub(super) ffn_fused_bind_layout: etalon::BindGroupLayout,
207    /// Naive 64-thread/row fused expansion (wasm-style; fallback).
208    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
209    pub(super) ffn_fused_pipeline: etalon::ComputePipeline,
210    /// T-A1b: coop 256-thread/row fused expansion (preferred when coop GEMV is on).
211    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
212    pub(super) ffn_fused_coop_pipeline: etalon::ComputePipeline,
213    /// Multi-row fused FFN (4 rows/WG) for Q4_K_SOA.
214    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
215    pub(super) ffn_fused_mr_pipeline: etalon::ComputePipeline,
216    /// Warp fused FFN (32 thr/row) for Q4_K_SOA.
217    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
218    pub(super) ffn_fused_warp_pipeline: etalon::ComputePipeline,
219    /// Dual K+V GEMV (shared act) for resident mega-pass.
220    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
221    pub(super) dual_gemv_pipeline: etalon::ComputePipeline,
222    /// Dual multi-row (4 rows/WG) — default for SoA K+V.
223    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
224    pub(super) dual_gemv_mr_pipeline: etalon::ComputePipeline,
225    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
226    pub(super) dual_gemv_bind_layout: etalon::BindGroupLayout,
227    /// Triple Q+K+V GEMV (shared act, GQA-safe) — one dispatch replaces dual+Q.
228    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
229    pub(super) triple_gemv_pipeline: etalon::ComputePipeline,
230    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
231    pub(super) triple_gemv_bind_layout: etalon::BindGroupLayout,
232    /// Phase 5.3: the output/logits projection (tied `token_embd`, ~50 MB) uploaded to VRAM
233    /// once at init so the per-token argmax binds resident sub-ranges instead of re-uploading
234    /// the whole matrix every token (the decode throughput killer). A1a step-2 ports this to the
235    /// native top-k decode path, so these two fields are available on both targets.
236    #[cfg(feature = "gpu-runtime")]
237    pub(super) mc8_logits_resident_buf: Option<etalon::Buffer>,
238    pub(super) mc8_logits_row_bytes: u32,
239    /// A1b (STELLAR §A): resident 2-bit ternary-FFN GEMM dispatcher, built once at P64 boot from
240    /// the container's base-3 FFN blobs (rebaked to 2-bit, uploaded once). `None` until a ternary
241    /// P64 is adopted; the FFN dispatch branch (`dispatch_ternary_ffn`) uses it when present +
242    /// the toggle is on, else the CPU oracle. Native-only; the wasm ternary path is a later step.
243    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
244    pub(super) ternary_ffn: Option<crate::ternary_gpu::TernaryFfnResident>,
245    /// Bounded cold resident projection cache. Retains immutable source ownership, validates
246    /// device and byte-region length, and invalidates model replacement before dropping sources.
247    /// Same leased source is reused across requests; no per-request epoch or full-weight hashing.
248    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
249    pub(super) projection_cold_byte_budget: usize,
250    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
251    pub(super) projection_readback: etalon_inference::projections::readback::ReadbackCompletion,
252    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
253    pub(super) gemm_resident_weights: std::sync::Mutex<super::gemm::ResidentProjectionCache>,
254    /// Phase 3 (FFN fusion): a small uniform buffer holding the gate/up/down GEMM `GemmGpuParams`
255    /// at 256-aligned sub-ranges (3 slots), so all three GEMMs of one fused FFN submit can bind
256    /// distinct params simultaneously. Lazily created native-only on the first fused FFN.
257    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
258    pub(super) ffn_fused_params: Option<etalon::Buffer>,
259    /// Native attention preproject fusion: two 256-byte-aligned GEMM uniform slots
260    /// (K,V) and two attention uniform slots (K-write,V-write), allowing K/V
261    /// projection + KV-cache writes to share one submit without uniform races.
262    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
263    pub(super) attention_kv_gemm_params: Option<etalon::Buffer>,
264    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
265    pub(super) attention_kv_params: Option<etalon::Buffer>,
266    /// Phase 5.4: all layers' attn_norm + ffn_norm weights resident (slot 2L = attn, 2L+1 = ffn),
267    /// so RMSNorm binds a per-layer sub-range instead of re-`write_buffer`ing a shared single-layer
268    /// `norm_weight_buf` every layer (the second per-layer write_buffer race blocking single-submit).
269    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
270    pub(super) mc8_norm_resident_buf: Option<etalon::Buffer>,
271    #[cfg(target_arch = "wasm32")]
272    pub(super) mc8_norm_stride: u32,
273    /// Native GPU-resident single-fence decode plan (see `resident_decode.rs`).
274    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
275    pub(super) resident_decode: resident_decode::ResidentDecodeState,
276    /// Cold-built host descriptor for the native CUDA all-layer plan.
277    #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
278    pub(super) cuda_decode_plan: cuda_decode_plan::CudaDecodePlanState,
279    /// W3: native GPU-resident single-fence-per-chunk prefill plan (see `prefill_arena.rs`).
280    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
281    pub(super) prefill_arena: prefill_arena::PrefillArenaState,
282    /// W6a: batched speculative-verify forward plan (per-position argmax; see `verify_arena.rs`).
283    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
284    pub(super) verify_arena: verify_arena::VerifyArenaState,
285    /// Bind group cache: eliminates per-token `create_bind_group` calls by caching
286    /// bind groups keyed on (buffer addresses, offsets, weight role, layer). Bind groups
287    /// are identical across tokens for the same layer/op since only dynamic uniform
288    /// offsets change — those are passed at `set_bind_group` time, not baked into the BG.
289    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
290    pub(super) mc8_bg_cache: std::sync::Mutex<std::collections::HashMap<u64, etalon::BindGroup>>,
291    /// Flat recurrent-state arena for all hybrid SSM layers (Granite Hybrid / Qwen GatedDeltaNet).
292    ///
293    /// Allocated once by `ssm_forward::init_ssm_state` after model load; empty (len=0) for
294    /// pure-attention models.  During the hot decode loop the arena is accessed as a
295    /// `&mut [f32]` slice — no per-token Vec allocation (Tier-1 zero-heap).
296    pub(crate) ssm_recurrent_state: Box<[f32]>,
297    pub(crate) ssm_state_byte_budget: usize,
298}