Skip to main content

qualia_core_db/gguf_bridge/
init.rs

1//! Engine init/lifecycle: GPU device/queue accessors, try_new, new, KV-cache + GEMM-buffer
2//! allocation/reset. Split from gguf_bridge/mod.rs (structural; no behaviour change).
3use super::*;
4
5impl QTensorEngine {
6    /// Resolve the MC8 elementwise GPU pipeline for a given opcode.
7    #[cfg(feature = "gpu-runtime")]
8    pub(crate) fn elem_gpu_pipeline(&self, op: u32) -> Option<&etalon::ComputePipeline> {
9        use super::gpu_params::{ELEM_OP_ADD_RESIDUAL, ELEM_OP_RMS_NORM};
10        match op {
11            ELEM_OP_RMS_NORM => Some(&self.elem_rms_norm_pipeline),
12            ELEM_OP_ADD_RESIDUAL => Some(&self.elem_add_residual_pipeline),
13            _ => None,
14        }
15    }
16    #[cfg(feature = "gpu-runtime")]
17    /// wgpu device.
18    pub(crate) fn gpu_device(&self) -> &etalon::Device {
19        #[cfg(target_arch = "wasm32")]
20        {
21            return &self.device;
22        }
23        #[cfg(not(target_arch = "wasm32"))]
24        {
25            &crate::gpu_context::shared_gpu().device
26        }
27    }
28
29    #[inline]
30    #[cfg(feature = "gpu-runtime")]
31    /// wgpu queue.
32    pub(crate) fn gpu_queue(&self) -> &etalon::Queue {
33        #[cfg(target_arch = "wasm32")]
34        {
35            return &self.queue;
36        }
37        #[cfg(not(target_arch = "wasm32"))]
38        {
39            &crate::gpu_context::shared_gpu().queue
40        }
41    }
42
43    /// Shared process-wide wgpu device (LLM + render coexistence).
44    #[inline]
45    #[cfg(feature = "gpu-runtime")]
46    pub fn device(&self) -> &etalon::Device {
47        self.gpu_device()
48    }
49
50    /// Shared process-wide wgpu queue.
51    #[inline]
52    #[cfg(feature = "gpu-runtime")]
53    pub fn queue(&self) -> &etalon::Queue {
54        self.gpu_queue()
55    }
56
57    #[cfg(feature = "gpu-runtime")]
58    /// Tries to build the engine; error when GPU init fails.
59    pub async fn try_new() -> Result<Self, String> {
60        #[cfg(not(target_arch = "wasm32"))]
61        log::info!(
62            "LLM_LOAD|engine-init|0.10|Initializing native GGUF runtime (shared GpuContext)"
63        );
64        #[cfg(target_arch = "wasm32")]
65        log::info!("LLM_LOAD|engine-init|0.10|Initializing WASM GGUF runtime");
66
67        #[cfg(not(target_arch = "wasm32"))]
68        let shared = crate::gpu_context::shared_gpu();
69        #[cfg(not(target_arch = "wasm32"))]
70        let device = &shared.device;
71        #[cfg(not(target_arch = "wasm32"))]
72        let _queue = &shared.queue;
73        #[cfg(not(target_arch = "wasm32"))]
74        log::info!("LLM_LOAD|gpu-device|0.35|Reusing process-wide wgpu device");
75        // NVIDIA: create CUDA multi-weight context early so the GPU leaves idle clocks
76        // even for portable/FastVerify resident decode (measured 3B ~1.5 → ~7 tok/s).
77        #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
78        {
79            let _ = crate::warm_cuda_context();
80        }
81
82        #[cfg(target_arch = "wasm32")]
83        let (wasm_device, wasm_queue) = {
84            crate::gguf_bridge::wasm_yield::phase("Acquiring shared WebGPU device…").await;
85            crate::gpu_context::ensure_shared_gpu().await?;
86            let shared = crate::gpu_context::try_shared_gpu().ok_or_else(|| {
87                "shared WebGPU device missing after ensure_shared_gpu".to_string()
88            })?;
89            let info = shared.adapter.get_info();
90            log::info!(
91                "LLM_LOAD|webgpu-adapter|0.20|name={} backend={:?} device={:?} vendor={:?}",
92                info.name,
93                info.backend,
94                info.device,
95                info.vendor
96            );
97            (shared.device.clone(), shared.queue.clone())
98        };
99        #[cfg(target_arch = "wasm32")]
100        let device = &wasm_device;
101        // One yield after device so the UI paints before pipeline compile freezes the main thread.
102        #[cfg(target_arch = "wasm32")]
103        crate::gguf_bridge::wasm_yield::phase(
104            "Compiling compute pipelines (UI may pause 30–90s on phones — leave tab open)…",
105        )
106        .await;
107        #[cfg(not(target_arch = "wasm32"))]
108        let native_pipeline_cache = create_native_pipeline_cache(device);
109        #[cfg(target_arch = "wasm32")]
110        let native_pipeline_cache: Option<etalon::PipelineCache> = None;
111
112        let resources = etalon_inference::gpu_resources::EngineResources::build(
113            device,
114            &native_pipeline_cache,
115            etalon_inference::gpu_resources::PipelineOptions {
116                subgroups: device.features().contains(etalon::Features::SUBGROUP),
117            },
118            etalon_inference::gpu_resources::ResourceBudget::for_device(device),
119            |_| Ok(()),
120        )
121        .map_err(|error| format!("inference GPU resources: {error:?}"))?;
122        let etalon_inference::gpu_resources::EngineResources {
123            gemv,
124            common,
125            ffn,
126            projections,
127            #[cfg(not(target_arch = "wasm32"))]
128            attention_kv_gemm_params,
129            #[cfg(not(target_arch = "wasm32"))]
130            attention_kv_params,
131        } = resources;
132        let etalon_inference::gpu_resources::GemvPipelines {
133            pipeline,
134            #[cfg(target_arch = "wasm32")]
135            mmv_q8_0_pipeline,
136            #[cfg(not(target_arch = "wasm32"))]
137            coop_gemv_pipeline,
138            #[cfg(not(target_arch = "wasm32"))]
139            coop_gemv_residual_pipeline,
140            #[cfg(not(target_arch = "wasm32"))]
141            coop_gemv_bind_layout,
142            #[cfg(not(target_arch = "wasm32"))]
143            coop_gemv_residual_bind_layout,
144            #[cfg(not(target_arch = "wasm32"))]
145            coop_gemv_mr_pipeline,
146            #[cfg(not(target_arch = "wasm32"))]
147            coop_gemv_residual_mr_pipeline,
148            #[cfg(not(target_arch = "wasm32"))]
149            coop_gemv_warp_pipeline,
150            #[cfg(not(target_arch = "wasm32"))]
151            coop_gemv_residual_warp_pipeline,
152            #[cfg(not(target_arch = "wasm32"))]
153            mock_pipeline,
154            #[cfg(target_arch = "wasm32")]
155            mc8_gemm_bind_layout,
156            #[cfg(not(target_arch = "wasm32"))]
157            pipeline_bind_layout,
158        } = gemv;
159        let etalon_inference::gpu_resources::CommonPipelines {
160            embedding_pipeline,
161            attention_pipeline,
162            elem_rms_norm_pipeline,
163            elem_silu_mul_pipeline,
164            elem_add_residual_pipeline,
165            #[cfg(target_arch = "wasm32")]
166            mc8_attn_bind_layout,
167            #[cfg(target_arch = "wasm32")]
168            mc8_elem_bind_layout,
169            #[cfg(not(target_arch = "wasm32"))]
170            embedding_bind_layout,
171            #[cfg(not(target_arch = "wasm32"))]
172            attention_bind_layout,
173            #[cfg(not(target_arch = "wasm32"))]
174            elem_silu_mul_bind_layout,
175        } = common;
176        let etalon_inference::gpu_resources::FfnPipelines {
177            #[cfg(target_arch = "wasm32")]
178            mc8_ffn_fused_bind_layout,
179            #[cfg(target_arch = "wasm32")]
180            mc8_ffn_fused_pipeline,
181            #[cfg(not(target_arch = "wasm32"))]
182            ffn_fused_bind_layout,
183            #[cfg(not(target_arch = "wasm32"))]
184            ffn_fused_pipeline,
185            #[cfg(not(target_arch = "wasm32"))]
186            ffn_fused_coop_pipeline,
187            #[cfg(not(target_arch = "wasm32"))]
188            ffn_fused_mr_pipeline,
189            #[cfg(not(target_arch = "wasm32"))]
190            ffn_fused_warp_pipeline,
191        } = ffn;
192        let etalon_inference::gpu_resources::ProjectionsPipelines {
193            #[cfg(not(target_arch = "wasm32"))]
194            dual_gemv_bind_layout,
195            #[cfg(not(target_arch = "wasm32"))]
196            dual_gemv_pipeline,
197            #[cfg(not(target_arch = "wasm32"))]
198            dual_gemv_mr_pipeline,
199            #[cfg(not(target_arch = "wasm32"))]
200            triple_gemv_bind_layout,
201            #[cfg(not(target_arch = "wasm32"))]
202            triple_gemv_pipeline,
203        } = projections;
204        #[cfg(not(target_arch = "wasm32"))]
205        log::info!(
206            "LLM_LOAD|fused_ffn|coop_entry={}|mr=coop_fused_ffn_mr|warp=coop_fused_ffn_warp",
207            if device.features().contains(etalon::Features::SUBGROUP) {
208                "coop_fused_ffn_sg"
209            } else {
210                "coop_fused_ffn_expansion"
211            }
212        );
213
214        // DirectML is a *second* D3D12 device next to wgpu. Always-on init competed for
215        // A2000 VRAM and added driver overhead while the resident decode path uses wgpu.
216        // Opt in with QUALIA_DIRECTML=1 (or QUALIA_LLM_DIRECTML=1). Default: wgpu-only.
217        #[cfg(target_os = "windows")]
218        let dml_status = {
219            let want = matches!(
220                crate::gguf_bridge::execution_env::var("QUALIA_DIRECTML")
221                    .or_else(|_| crate::gguf_bridge::execution_env::var("QUALIA_LLM_DIRECTML"))
222                    .ok()
223                    .as_deref(),
224                Some("1") | Some("true") | Some("on")
225            );
226            if !want {
227                log::info!(
228                    "LLM_LOAD|gpu-backend|0.45|DirectML deferred (set QUALIA_DIRECTML=1 to enable second D3D12 device)"
229                );
230                None
231            } else {
232                match crate::directml_bridge::DmlDevice::new() {
233                    Ok(device) => {
234                        log::info!(
235                            "DirectML device initialization: Ok({})",
236                            device.adapter_desc
237                        );
238                        log::info!(
239                            "LLM_LOAD|gpu-backend|0.45|DirectML ready on {}",
240                            device.adapter_desc
241                        );
242                        log::info!(
243                            "LLM_LOAD|gpu-route|0.48|Streaming weights through DirectML with {:.1} GiB VRAM free",
244                            bytes_to_gib(
245                                device
246                                    .local_budget_bytes
247                                    .saturating_sub(device.local_usage_bytes)
248                            )
249                        );
250                        Some(device)
251                    }
252                    Err(err) => {
253                        log::warn!("DirectML device initialization failed: {:?}", err);
254                        log::info!(
255                            "LLM_LOAD|gpu-backend|0.45|DirectML unavailable; using wgpu fallback"
256                        );
257                        None
258                    }
259                }
260            }
261        };
262
263        #[cfg(not(target_os = "windows"))]
264        {
265            log::info!("LLM_LOAD|gpu-backend|0.45|Using wgpu fallback backend for native compute");
266        }
267
268        #[cfg(not(target_arch = "wasm32"))]
269        let projection_cold_byte_budget = crate::gpu_context::global_vram_ledger().budget();
270        #[cfg(target_os = "windows")]
271        let projection_cold_byte_budget =
272            dml_status
273                .as_ref()
274                .map_or(projection_cold_byte_budget, |device| {
275                    device
276                        .local_budget_bytes
277                        .saturating_sub(device.local_usage_bytes)
278                });
279        #[cfg(not(target_arch = "wasm32"))]
280        let projection_cold_byte_budget = usize::try_from(projection_cold_byte_budget).unwrap_or(0);
281
282        let engine = Self {
283            #[cfg(target_arch = "wasm32")]
284            device: wasm_device,
285            #[cfg(target_arch = "wasm32")]
286            queue: wasm_queue,
287            pipeline,
288            #[cfg(target_arch = "wasm32")]
289            mmv_q8_0_pipeline,
290            #[cfg(not(target_arch = "wasm32"))]
291            native_pipeline_cache,
292            #[cfg(not(target_arch = "wasm32"))]
293            pipeline_bind_layout,
294            #[cfg(not(target_arch = "wasm32"))]
295            coop_gemv_pipeline,
296            #[cfg(not(target_arch = "wasm32"))]
297            coop_gemv_bind_layout,
298            #[cfg(not(target_arch = "wasm32"))]
299            coop_gemv_mr_pipeline,
300            #[cfg(not(target_arch = "wasm32"))]
301            coop_gemv_residual_pipeline,
302            #[cfg(not(target_arch = "wasm32"))]
303            coop_gemv_residual_bind_layout,
304            #[cfg(not(target_arch = "wasm32"))]
305            coop_gemv_residual_mr_pipeline,
306            #[cfg(not(target_arch = "wasm32"))]
307            coop_gemv_warp_pipeline,
308            #[cfg(not(target_arch = "wasm32"))]
309            coop_gemv_residual_warp_pipeline,
310            #[cfg(not(target_arch = "wasm32"))]
311            mock_pipeline,
312            embedding_pipeline,
313            #[cfg(not(target_arch = "wasm32"))]
314            embedding_bind_layout,
315            attention_pipeline,
316            #[cfg(not(target_arch = "wasm32"))]
317            attention_bind_layout,
318            is_initialized: true,
319            #[cfg(target_os = "windows")]
320            dml: dml_status,
321            #[cfg(not(target_arch = "wasm32"))]
322            prepared_native_bytes: None,
323            #[cfg(not(target_arch = "wasm32"))]
324            prepared_native_sources: None,
325            #[cfg(not(target_arch = "wasm32"))]
326            qwen35_ftw: None,
327            #[cfg(not(target_arch = "wasm32"))]
328            qwen35_gguf: None,
329            #[cfg(not(target_arch = "wasm32"))]
330            qwen35_gguf_required: false,
331            #[cfg(not(target_arch = "wasm32"))]
332            gemma4: None,
333            #[cfg(not(target_arch = "wasm32"))]
334            gemma4_required: false,
335            gguf_mmap: None,
336            #[cfg(target_arch = "wasm32")]
337            cached_tokenizer: None,
338            #[cfg(target_arch = "wasm32")]
339            cached_tensor_index: None,
340            #[cfg(target_arch = "wasm32")]
341            cached_token_embd: None,
342            #[cfg(target_arch = "wasm32")]
343            p64_resident: None,
344            #[cfg(not(target_arch = "wasm32"))]
345            p64_index: None,
346            #[cfg(not(target_arch = "wasm32"))]
347            tensor_index_cache: None,
348            #[cfg(not(target_arch = "wasm32"))]
349            ftw_package: None,
350            tensor_data_offset: 0,
351            hyperparams: crate::gguf_sharder::GgufHyperparams::default(),
352            max_tensor_bytes: 0,
353            gemm_input_buf: None,
354            gemm_weight_buf: None,
355            #[cfg(target_arch = "wasm32")]
356            mc8_weight_arena: None,
357            #[cfg(target_arch = "wasm32")]
358            mc8_weights_resident: false,
359            #[cfg(target_arch = "wasm32")]
360            mc8_weight_role_stride: [0u64; 7],
361            #[cfg(target_arch = "wasm32")]
362            gemm_weight_buf_b: None,
363            gemm_output_buf: None,
364            gemm_params_buf: None,
365            gemm_output_staging: None,
366            output_topk_pipeline: None,
367            output_topk_bind_layout: None,
368            topk_cand_val_buf: None,
369            topk_cand_idx_buf: None,
370            topk_cand_staging: None,
371            topk_params_buf: None,
372            gemm_aux_buf: None,
373            gemm_ffn_buf: None,
374            #[cfg(target_arch = "wasm32")]
375            prefill_scratch_buf: None,
376            #[cfg(target_arch = "wasm32")]
377            prefill_work_buf_a: None,
378            #[cfg(target_arch = "wasm32")]
379            prefill_work_buf_b: None,
380            #[cfg(target_arch = "wasm32")]
381            mc8_q_proj_buf: None,
382            #[cfg(target_arch = "wasm32")]
383            mc8_k_proj_buf: None,
384            #[cfg(target_arch = "wasm32")]
385            mc8_v_proj_buf: None,
386            gemm_max_out_dim: MAX_STACK_GEMM_OUT as u32,
387            gemm_max_input_floats: 0,
388            kv_layout: None,
389            kv_cache_gpu: None,
390            kv_cache_cpu: None,
391            attention_params_buf: None,
392            attention_mask_buf: None,
393            elem_rms_norm_pipeline,
394            elem_silu_mul_pipeline,
395            #[cfg(not(target_arch = "wasm32"))]
396            elem_silu_mul_bind_layout,
397            elem_add_residual_pipeline,
398            elem_params_buf: None,
399            norm_weight_buf: None,
400            #[cfg(target_arch = "wasm32")]
401            mc8_gemm_bind_layout,
402            #[cfg(target_arch = "wasm32")]
403            mc8_elem_bind_layout,
404            #[cfg(target_arch = "wasm32")]
405            mc8_attn_bind_layout,
406            #[cfg(target_arch = "wasm32")]
407            mc8_ffn_fused_bind_layout,
408            #[cfg(target_arch = "wasm32")]
409            mc8_ffn_fused_pipeline,
410            #[cfg(not(target_arch = "wasm32"))]
411            ffn_fused_bind_layout,
412            #[cfg(not(target_arch = "wasm32"))]
413            ffn_fused_pipeline,
414            #[cfg(not(target_arch = "wasm32"))]
415            ffn_fused_coop_pipeline,
416            #[cfg(not(target_arch = "wasm32"))]
417            ffn_fused_mr_pipeline,
418            #[cfg(not(target_arch = "wasm32"))]
419            ffn_fused_warp_pipeline,
420            #[cfg(not(target_arch = "wasm32"))]
421            dual_gemv_pipeline,
422            #[cfg(not(target_arch = "wasm32"))]
423            dual_gemv_mr_pipeline,
424            #[cfg(not(target_arch = "wasm32"))]
425            dual_gemv_bind_layout,
426            #[cfg(not(target_arch = "wasm32"))]
427            triple_gemv_pipeline,
428            #[cfg(not(target_arch = "wasm32"))]
429            triple_gemv_bind_layout,
430            mc8_logits_resident_buf: None,
431            mc8_logits_row_bytes: 0,
432            #[cfg(not(target_arch = "wasm32"))]
433            ternary_ffn: None,
434            #[cfg(not(target_arch = "wasm32"))]
435            resident_decode: super::resident_decode::ResidentDecodeState::Unbuilt,
436            #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
437            cuda_decode_plan: super::cuda_decode_plan::CudaDecodePlanState::Unbuilt,
438            #[cfg(not(target_arch = "wasm32"))]
439            prefill_arena: super::prefill_arena::PrefillArenaState::Unbuilt,
440            #[cfg(not(target_arch = "wasm32"))]
441            verify_arena: super::verify_arena::VerifyArenaState::Unbuilt,
442            #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
443            projection_cold_byte_budget,
444            #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
445            projection_readback: Default::default(),
446            #[cfg(not(target_arch = "wasm32"))]
447            gemm_resident_weights: std::sync::Mutex::new(
448                super::gemm::ResidentProjectionCache::default(),
449            ),
450            #[cfg(not(target_arch = "wasm32"))]
451            ffn_fused_params: None,
452            #[cfg(not(target_arch = "wasm32"))]
453            attention_kv_gemm_params: Some(attention_kv_gemm_params),
454            #[cfg(not(target_arch = "wasm32"))]
455            attention_kv_params: Some(attention_kv_params),
456            #[cfg(target_arch = "wasm32")]
457            mc8_norm_resident_buf: None,
458            #[cfg(target_arch = "wasm32")]
459            mc8_norm_stride: 0,
460            #[cfg(target_arch = "wasm32")]
461            mc8_bg_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
462            ssm_recurrent_state: Box::new([]),
463            ssm_state_byte_budget: DEFAULT_SSM_STATE_BYTE_BUDGET,
464        };
465        // Exercise the CPU elementwise oracle (ReLU) once at engine init so the fallback
466        // path stays linked when GPU elem kernels are unavailable.
467        let mut relu_probe = [-1.0f32, 2.0];
468        let _ =
469            super::cpu_ops::apply_cpu_elem_op(super::gpu_params::ELEM_OP_RELU, &mut relu_probe, 2);
470        let _ = super::gpu_params::elem_op_label(super::gpu_params::ELEM_OP_RMS_NORM);
471        let _ = engine.elem_gpu_pipeline(super::gpu_params::ELEM_OP_RMS_NORM);
472        let _ = engine.elem_gpu_pipeline(super::gpu_params::ELEM_OP_ADD_RESIDUAL);
473        Ok(engine)
474    }
475
476    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
477    /// Builds the engine; panics when GPU init fails.
478    pub fn new() -> Self {
479        let handle = tokio::runtime::Handle::try_current().unwrap_or_else(|_| {
480            let rt = Box::leak(Box::new(tokio::runtime::Runtime::new().unwrap()));
481            rt.handle().clone()
482        });
483        tokio::task::block_in_place(|| {
484            handle
485                .block_on(Self::try_new())
486                .expect("Failed to initialize native GGUF engine")
487        })
488    }
489
490    /// CPU-only engine: mmap + hyperparams + dequant/GEMM. No wgpu, no device.
491    #[cfg(not(feature = "gpu-runtime"))]
492    fn cpu_runtime() -> Self {
493        Self {
494            is_initialized: true,
495            #[cfg(target_os = "windows")]
496            dml: None,
497            #[cfg(not(target_arch = "wasm32"))]
498            prepared_native_bytes: None,
499            #[cfg(not(target_arch = "wasm32"))]
500            prepared_native_sources: None,
501            #[cfg(not(target_arch = "wasm32"))]
502            qwen35_ftw: None,
503            #[cfg(not(target_arch = "wasm32"))]
504            qwen35_gguf: None,
505            #[cfg(not(target_arch = "wasm32"))]
506            qwen35_gguf_required: false,
507            #[cfg(not(target_arch = "wasm32"))]
508            gemma4: None,
509            #[cfg(not(target_arch = "wasm32"))]
510            gemma4_required: false,
511            gguf_mmap: None,
512            #[cfg(target_arch = "wasm32")]
513            cached_tokenizer: None,
514            #[cfg(target_arch = "wasm32")]
515            cached_tensor_index: None,
516            #[cfg(target_arch = "wasm32")]
517            cached_token_embd: None,
518            #[cfg(target_arch = "wasm32")]
519            p64_resident: None,
520            #[cfg(not(target_arch = "wasm32"))]
521            p64_index: None,
522            #[cfg(not(target_arch = "wasm32"))]
523            tensor_index_cache: None,
524            #[cfg(not(target_arch = "wasm32"))]
525            ftw_package: None,
526            tensor_data_offset: 0,
527            hyperparams: crate::gguf_sharder::GgufHyperparams::default(),
528            max_tensor_bytes: 0,
529            #[cfg(target_arch = "wasm32")]
530            mc8_weights_resident: false,
531            #[cfg(target_arch = "wasm32")]
532            mc8_weight_role_stride: [0u64; 7],
533            gemm_max_out_dim: MAX_STACK_GEMM_OUT as u32,
534            gemm_max_input_floats: 0,
535            kv_layout: None,
536            kv_cache_cpu: None,
537            mc8_logits_row_bytes: 0,
538            #[cfg(target_arch = "wasm32")]
539            mc8_norm_stride: 0,
540            #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
541            cuda_decode_plan: super::cuda_decode_plan::CudaDecodePlanState::Unbuilt,
542            ssm_recurrent_state: Box::new([]),
543            ssm_state_byte_budget: DEFAULT_SSM_STATE_BYTE_BUDGET,
544        }
545    }
546
547    #[cfg(not(feature = "gpu-runtime"))]
548    /// Tries to build the engine; error when GPU init fails.
549    pub async fn try_new() -> Result<Self, String> {
550        log::info!(
551            "LLM_LOAD|engine-init|0.10|Initializing CPU-only GGUF runtime (gpu-runtime off)"
552        );
553        Ok(Self::cpu_runtime())
554    }
555
556    #[cfg(not(feature = "gpu-runtime"))]
557    /// Builds the engine; panics when GPU init fails.
558    pub fn new() -> Self {
559        Self::cpu_runtime()
560    }
561
562    #[cfg(feature = "gpu-runtime")]
563    /// Ensures the KV cache is allocated.
564    pub(crate) fn ensure_kv_cache(&mut self, h: &crate::gguf_sharder::GgufHyperparams) {
565        #[cfg(not(target_arch = "wasm32"))]
566        let candidate = self.dense_qwen_kv_layout(h);
567        #[cfg(target_arch = "wasm32")]
568        let candidate = KvCacheLayout::from_hyperparams(h);
569        let layout = match candidate {
570            Some(l) => l,
571            None => {
572                #[cfg(target_arch = "wasm32")]
573                wlog(
574                    "[kv_cache] FAILED from_hyperparams (zero dims or exceeds KV_CACHE_MAX_BYTES)",
575                );
576                return;
577            }
578        };
579        let bytes = (layout.total_f32_elems * std::mem::size_of::<f32>()) as etalon::BufferAddress;
580        let attn_params_bytes = {
581            #[cfg(target_arch = "wasm32")]
582            {
583                (MC8_MAX_ATTN_UNIFORM_CHUNK_SLOTS * MC8_UNIFORM_ALIGN) as etalon::BufferAddress
584            }
585            #[cfg(not(target_arch = "wasm32"))]
586            {
587                std::mem::size_of::<AttentionGpuParams>() as etalon::BufferAddress
588            }
589        };
590        use etalon_inference::gpu_resources::{KvBuffers, KvGeometry, ResourceBudget};
591        let resources = match KvBuffers::build(
592            self.gpu_device(),
593            KvGeometry {
594                cache_bytes: bytes,
595                attention_params_bytes: attn_params_bytes,
596                mask_bytes: (MAX_ATTN_MASK_UPLOAD_WORDS * std::mem::size_of::<u32>()).max(4) as u64,
597            },
598            ResourceBudget::for_device(self.gpu_device()),
599            |_| {
600                #[cfg(not(target_arch = "wasm32"))]
601                {
602                    let ledger = crate::gpu_context::global_vram_ledger();
603                    let orch = crate::gpu_context::universe_orchestrator();
604                    if !ledger.can_allocate_in_universe(
605                        &orch,
606                        crate::gpu_context::ComputeUniverse::LlmInference,
607                        bytes,
608                    ) {
609                        log::warn!(
610                            "LLM_LOAD|kv-cache|denied|U0 budget {:.1} MiB used, need {:.1} MiB (mode {:?})",
611                            ledger.universe_used_bytes(
612                                crate::gpu_context::ComputeUniverse::LlmInference
613                            ) as f64
614                                / (1024.0 * 1024.0),
615                            bytes as f64 / (1024.0 * 1024.0),
616                            orch.active_mode,
617                        );
618                        return Err(etalon_inference::gpu_resources::ResourceError::PolicyDenied);
619                    }
620                }
621                Ok(())
622            },
623        ) {
624            Ok(resources) => resources,
625            Err(error) => {
626                if !matches!(
627                    error,
628                    etalon_inference::gpu_resources::ResourceError::PolicyDenied
629                ) && (h.architecture == crate::gguf_sharder::ARCH_GEMMA4
630                    || h.ssm_conv_kernel > 0
631                    || (h.arch_flags & crate::gguf_sharder::ARCH_FLAG_HAS_QK_NORM) != 0
632                    || crate::llm_bench::cpu_attention_enabled())
633                {
634                    let cpu = vec![0f32; layout.total_f32_elems].into_boxed_slice();
635                    self.kv_layout = Some(layout);
636                    self.kv_cache_cpu = Some(cpu);
637                    self.kv_cache_gpu = None;
638                    log::info!(
639                        "LLM_LOAD|kv-cache|0.86|Reserved {:.1} MiB host KV cache (context {})",
640                        bytes as f64 / (1024.0 * 1024.0),
641                        layout.max_context,
642                    );
643                    return;
644                }
645                log::warn!("inference KV resource admission failed: {error:?}");
646                return;
647            }
648        };
649        let gpu = resources.cache;
650        let cpu = vec![0f32; layout.total_f32_elems].into_boxed_slice();
651        self.attention_params_buf = Some(resources.attention_params);
652        self.attention_mask_buf = Some(resources.mask);
653        self.kv_layout = Some(layout);
654        self.kv_cache_gpu = Some(gpu);
655        self.kv_cache_cpu = Some(cpu);
656        // W5b Phase 4b: seed the dictionary atoms into each layer's arena tail (after the codes).
657        #[cfg(not(target_arch = "wasm32"))]
658        self.upload_dict_atoms();
659        #[cfg(not(target_arch = "wasm32"))]
660        {
661            let ledger = crate::gpu_context::global_vram_ledger();
662            ledger.record_kv_cache(bytes);
663        }
664        log::info!(
665            "LLM_LOAD|kv-cache|0.86|Reserved {:.1} MiB KV cache (GPU + CPU mirror, context {})",
666            bytes as f64 / (1024.0 * 1024.0),
667            layout.max_context,
668        );
669        #[cfg(not(target_arch = "wasm32"))]
670        eprintln!(
671            "[gguf_bridge] KV arena {} slots ({:.1} MiB, {}), context={}",
672            layout.total_f32_elems,
673            bytes as f64 / (1024.0 * 1024.0),
674            if layout.dict_k > 0 {
675                "dict-coded"
676            } else if layout.int8 {
677                "int8+scale"
678            } else {
679                "f32"
680            },
681            layout.max_context,
682        );
683    }
684
685    /// CPU-only KV arena: layout + host mirror. No wgpu buffers.
686    #[cfg(not(feature = "gpu-runtime"))]
687    pub(crate) fn ensure_kv_cache(&mut self, h: &crate::gguf_sharder::GgufHyperparams) {
688        #[cfg(not(target_arch = "wasm32"))]
689        let candidate = self.dense_qwen_kv_layout(h);
690        #[cfg(target_arch = "wasm32")]
691        let candidate = KvCacheLayout::from_hyperparams(h);
692        let layout = match candidate {
693            Some(l) => l,
694            None => return,
695        };
696        let total = layout.total_f32_elems;
697        let max_context = layout.max_context;
698        let cpu = vec![0f32; total].into_boxed_slice();
699        self.kv_layout = Some(layout);
700        self.kv_cache_cpu = Some(cpu);
701        log::info!(
702            "LLM_LOAD|kv-cache|0.86|Reserved {:.1} MiB CPU KV cache (context {})",
703            (total * std::mem::size_of::<f32>()) as f64 / (1024.0 * 1024.0),
704            max_context,
705        );
706    }
707
708    /// Zero the static KV arena at the start of a new decode context (zero heap in decode).
709    pub fn reset_kv_cache(&mut self) {
710        self.reset_recurrent_state();
711        let Some(layout) = self.kv_layout.as_ref() else {
712            return;
713        };
714        let n = layout.total_f32_elems;
715        if let Some(cpu) = self.kv_cache_cpu.as_mut() {
716            for v in cpu.iter_mut().take(n) {
717                unsafe { core::ptr::write_volatile(v, 0.0) };
718            }
719        }
720        #[cfg(feature = "gpu-runtime")]
721        if let (Some(cpu), Some(gpu)) = (self.kv_cache_cpu.as_ref(), self.kv_cache_gpu.as_ref()) {
722            self.gpu_queue()
723                .write_buffer(gpu, 0, bytemuck::cast_slice(&cpu[..n]));
724        }
725        // W5b Phase 4b: the zero above wiped the atoms tail — re-seed it (dict mode only).
726        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
727        self.upload_dict_atoms();
728    }
729
730    /// W5b Phase 4b: write the installed dictionary atoms into the tail of each layer's arena slice
731    /// (after that layer's code region). No-op unless dict mode is active and atoms are installed.
732    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
733    pub(crate) fn upload_dict_atoms(&self) {
734        let Some(layout) = self.kv_layout.as_ref() else {
735            return;
736        };
737        if layout.dict_k == 0 {
738            return;
739        }
740        let (Some(buf), Some((flat, na, hd))) = (
741            self.kv_cache_gpu.as_ref(),
742            crate::kv_dict_runtime::atoms_flat(),
743        ) else {
744            return;
745        };
746        let code_region = (layout.max_context * 2 * layout.n_kv_head * layout.dict_k) as usize;
747        let per_layer_atoms = 2 * na * hd;
748        let ls = layout.layer_stride as usize;
749        for l in 0..layout.n_layer as usize {
750            let s = l * per_layer_atoms;
751            let e = ((l + 1) * per_layer_atoms).min(flat.len());
752            if e <= s {
753                break;
754            }
755            let dst_word = l * ls + code_region;
756            self.gpu_queue().write_buffer(
757                buf,
758                (dst_word * 4) as u64,
759                bytemuck::cast_slice(&flat[s..e]),
760            );
761        }
762    }
763
764    /// Returns `kv_cache_cpu`.
765    pub fn get_kv_cache_cpu(&self) -> Option<&[f32]> {
766        self.kv_cache_cpu.as_deref()
767    }
768
769    /// Sets `kv_cache_cpu`.
770    pub fn set_kv_cache_cpu(&mut self, data: &[f32]) -> bool {
771        let Some(layout) = self.kv_layout.as_ref() else {
772            return false;
773        };
774        let n = layout.total_f32_elems;
775        if data.len() < n {
776            return false;
777        }
778        let Some(cpu) = self.kv_cache_cpu.as_mut() else {
779            return false;
780        };
781        cpu[..n].copy_from_slice(&data[..n]);
782        #[cfg(feature = "gpu-runtime")]
783        if let (Some(cpu), Some(gpu)) = (self.kv_cache_cpu.as_ref(), self.kv_cache_gpu.as_ref()) {
784            self.gpu_queue()
785                .write_buffer(gpu, 0, bytemuck::cast_slice(&cpu[..n]));
786        }
787        true
788    }
789
790    /// Read the entire GPU KV-cache arena back to host as `f32` (native, cold path — forge KV capture,
791    /// not the decode hot path). The returned flat buffer is interpretable via `KvCacheLayout::
792    /// k_index`/`v_index` **only when the layout is f32** (int8 KV disabled at load); with an int8
793    /// layout the bytes are packed i8 lanes + scales and this returns `None`.
794    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
795    pub fn read_kv_cache_gpu(&self) -> Option<Vec<f32>> {
796        let gpu = self.kv_cache_gpu.as_ref()?;
797        let layout = self.kv_layout.as_ref()?;
798        if layout.int8 {
799            return None;
800        }
801        let n = layout.total_f32_elems;
802        let size = (n * std::mem::size_of::<f32>()) as etalon::BufferAddress;
803        let staging = self.gpu_device().create_buffer(&etalon::BufferDescriptor {
804            label: Some("KvCacheReadback"),
805            size: size.max(4),
806            usage: etalon::BufferUsages::MAP_READ | etalon::BufferUsages::COPY_DST,
807            mapped_at_creation: false,
808        });
809        let mut encoder =
810            self.gpu_device()
811                .create_command_encoder(&etalon::CommandEncoderDescriptor {
812                    label: Some("KvReadback"),
813                });
814        encoder.copy_buffer_to_buffer(gpu, 0, &staging, 0, size);
815        self.submit_commands(Some(encoder.finish()));
816
817        let slice = staging.slice(..);
818        let (tx, rx) = futures_channel::oneshot::channel();
819        slice.map_async(etalon::MapMode::Read, move |v| {
820            let _ = tx.send(v);
821        });
822        self.poll_wait();
823        let handle = tokio::runtime::Handle::try_current().unwrap_or_else(|_| {
824            let rt = Box::leak(Box::new(tokio::runtime::Runtime::new().unwrap()));
825            rt.handle().clone()
826        });
827        if handle.block_on(rx).ok()?.is_err() {
828            return None;
829        }
830        let data = slice
831            .get_mapped_range()
832            .expect("wgpu buffer map_range failed");
833        let out: Vec<f32> = bytemuck::cast_slice(&data).to_vec();
834        drop(data);
835        staging.unmap();
836        Some(out)
837    }
838
839    /// Decode the current f32 KV cache into per-layer K and V vectors for token positions `0..n_tokens`
840    /// (capped at `max_per_layer` per layer per stream). The **GPU-readback** capture route for the
841    /// sparse-KV-dictionary go/no-go — reads the real decode-path K/V straight from VRAM, no CPU
842    /// reference forward. Returns `None` on an int8 layout or if readback fails.
843    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
844    pub fn capture_kv_f32(
845        &self,
846        n_tokens: u32,
847        max_per_layer: usize,
848    ) -> Option<crate::kv_capture::KvCapture> {
849        let layout = *self.kv_layout.as_ref()?;
850        let flat = self.read_kv_cache_gpu()?;
851        let n_layer = layout.n_layer as usize;
852        let n_kv = layout.n_kv_head as usize;
853        let head_dim = layout.head_dim as usize;
854        if head_dim == 0 || n_kv == 0 || n_layer == 0 {
855            return None;
856        }
857        let take = (n_tokens as usize).min(layout.max_context as usize);
858        let mut k = vec![Vec::new(); n_layer];
859        let mut v = vec![Vec::new(); n_layer];
860        let read_vec = |base_of: &dyn Fn(u32) -> usize| -> Option<Vec<f32>> {
861            let mut out = Vec::with_capacity(head_dim);
862            for d in 0..head_dim {
863                let idx = base_of(d as u32);
864                if idx >= flat.len() {
865                    return None;
866                }
867                out.push(flat[idx]);
868            }
869            Some(out)
870        };
871        for l in 0..n_layer {
872            for pos in 0..take {
873                let slot = layout.ring_slot(pos as u32);
874                for hkv in 0..n_kv {
875                    if k[l].len() < max_per_layer {
876                        if let Some(vec_k) =
877                            read_vec(&|d| layout.k_index(l as u32, slot, hkv as u32, d))
878                        {
879                            k[l].push(vec_k);
880                        }
881                    }
882                    if v[l].len() < max_per_layer {
883                        if let Some(vec_v) =
884                            read_vec(&|d| layout.v_index(l as u32, slot, hkv as u32, d))
885                        {
886                            v[l].push(vec_v);
887                        }
888                    }
889                }
890            }
891        }
892        Some(crate::kv_capture::KvCapture { head_dim, k, v })
893    }
894
895    #[cfg(feature = "gpu-runtime")]
896    /// Ensures the GEMM buffers are allocated.
897    pub(crate) fn ensure_gemm_buffers(&mut self, max_weight_bytes: usize, max_out_dim: u32) {
898        // A1a: build the persistent GPU top-k pipeline + candidate buffers once (additive; the
899        // existing argmax path is unaffected whether or not this succeeds).
900        if self.output_topk_pipeline.is_none() {
901            self.init_output_topk();
902        }
903        let need_input = MAX_STACK_GEMM_IN.max(MAX_PREFILL_BATCH_FLOATS);
904        let prefill_bufs_ready = {
905            #[cfg(target_arch = "wasm32")]
906            {
907                self.prefill_scratch_buf.is_some()
908                    && self.prefill_work_buf_a.is_some()
909                    && self.prefill_work_buf_b.is_some()
910            }
911            #[cfg(not(target_arch = "wasm32"))]
912            {
913                true
914            }
915        };
916        #[cfg(target_arch = "wasm32")]
917        let weight_arena_ready = self.mc8_weight_arena.is_some();
918        #[cfg(not(target_arch = "wasm32"))]
919        let weight_arena_ready = true;
920        if self.gemm_weight_buf.is_some()
921            && max_weight_bytes <= self.max_tensor_bytes
922            && self.gemm_max_input_floats >= need_input
923            && prefill_bufs_ready
924            && weight_arena_ready
925        {
926            return;
927        }
928        use etalon_inference::gpu_resources::{GemmBuffers, GemmGeometry, ResourceBudget};
929        let gemm_params_bytes = {
930            #[cfg(target_arch = "wasm32")]
931            {
932                (MC8_MAX_GEMM_UNIFORM_CHUNK_SLOTS * MC8_UNIFORM_ALIGN) as u64
933            }
934            #[cfg(not(target_arch = "wasm32"))]
935            {
936                std::mem::size_of::<GemmGpuParams>() as u64
937            }
938        };
939        let elem_params_bytes = {
940            #[cfg(target_arch = "wasm32")]
941            {
942                (MC8_MAX_ELEM_UNIFORM_CHUNK_SLOTS * MC8_UNIFORM_ALIGN) as u64
943            }
944            #[cfg(not(target_arch = "wasm32"))]
945            {
946                std::mem::size_of::<ElemGpuParams>() as u64
947            }
948        };
949        let out_bytes = (u64::from(max_out_dim) * 4).max(4);
950        let resources = GemmBuffers::build(
951            self.gpu_device(),
952            GemmGeometry {
953                weight_bytes: max_weight_bytes.max(4) as u64,
954                input_bytes: (need_input * 4) as u64,
955                output_bytes: out_bytes,
956                gemm_params_bytes,
957                elem_params_bytes,
958                norm_bytes: (MAX_HIDDEN_DIM * 4) as u64,
959                staging_bytes: if cfg!(target_arch = "wasm32") {
960                    out_bytes.max(65536 * 4)
961                } else {
962                    out_bytes
963                },
964                #[cfg(target_arch = "wasm32")]
965                prefill_work_bytes: (PREFILL_CHUNK_SIZE
966                    * (MAX_HIDDEN_DIM + 2 * max_out_dim as usize + MAX_HIDDEN_DIM).max(4)
967                    * 4)
968                .max(4) as u64,
969            },
970            ResourceBudget::for_device(self.gpu_device()),
971            |_| Ok(()),
972        )
973        .unwrap_or_else(|error| panic!("inference GEMM resources: {error:?}"));
974        self.gemm_input_buf = Some(resources.input);
975        self.gemm_weight_buf = Some(resources.weight);
976        self.gemm_output_buf = Some(resources.output);
977        self.gemm_params_buf = Some(resources.params);
978        self.gemm_output_staging = Some(resources.staging);
979        self.gemm_aux_buf = Some(resources.aux);
980        self.gemm_ffn_buf = Some(resources.ffn);
981        self.elem_params_buf = Some(resources.elem_params);
982        self.norm_weight_buf = Some(resources.norm);
983        #[cfg(target_arch = "wasm32")]
984        {
985            let browser = resources.browser;
986            let [qkv_k, qkv_v, qkv_q, o_proj, gate, up, down] = browser.role_weights;
987            self.mc8_weight_arena = Some(Mc8WeightArenaBufs {
988                qkv_k,
989                qkv_v,
990                qkv_q,
991                o_proj,
992                gate,
993                up,
994                down,
995            });
996            self.gemm_weight_buf_b = Some(browser.weight_b);
997            self.prefill_scratch_buf = Some(browser.prefill_scratch);
998            self.prefill_work_buf_a = Some(browser.work_a);
999            self.prefill_work_buf_b = Some(browser.work_b);
1000            self.mc8_q_proj_buf = Some(browser.q_proj);
1001            self.mc8_k_proj_buf = Some(browser.k_proj);
1002            self.mc8_v_proj_buf = Some(browser.v_proj);
1003        }
1004        self.gemm_max_out_dim = max_out_dim;
1005        self.gemm_max_input_floats = need_input;
1006        self.max_tensor_bytes = max_weight_bytes;
1007    }
1008}
1009
1010/// Compile-time proof that CPU constructors exist when wgpu is not linked.
1011#[cfg(not(feature = "gpu-runtime"))]
1012const _: fn() -> super::QTensorEngine = super::QTensorEngine::new;