1use super::*;
4
5impl QTensorEngine {
6 #[cfg(feature = "gpu-runtime")]
8 pub(crate) fn elem_gpu_pipeline(&self, op: u32) -> Option<&etalon::ComputePipeline> {
9 use super::gpu_params::{ELEM_OP_ADD_RESIDUAL, ELEM_OP_RMS_NORM};
10 match op {
11 ELEM_OP_RMS_NORM => Some(&self.elem_rms_norm_pipeline),
12 ELEM_OP_ADD_RESIDUAL => Some(&self.elem_add_residual_pipeline),
13 _ => None,
14 }
15 }
16 #[cfg(feature = "gpu-runtime")]
17 pub(crate) fn gpu_device(&self) -> &etalon::Device {
19 #[cfg(target_arch = "wasm32")]
20 {
21 return &self.device;
22 }
23 #[cfg(not(target_arch = "wasm32"))]
24 {
25 &crate::gpu_context::shared_gpu().device
26 }
27 }
28
29 #[inline]
30 #[cfg(feature = "gpu-runtime")]
31 pub(crate) fn gpu_queue(&self) -> &etalon::Queue {
33 #[cfg(target_arch = "wasm32")]
34 {
35 return &self.queue;
36 }
37 #[cfg(not(target_arch = "wasm32"))]
38 {
39 &crate::gpu_context::shared_gpu().queue
40 }
41 }
42
43 #[inline]
45 #[cfg(feature = "gpu-runtime")]
46 pub fn device(&self) -> &etalon::Device {
47 self.gpu_device()
48 }
49
50 #[inline]
52 #[cfg(feature = "gpu-runtime")]
53 pub fn queue(&self) -> &etalon::Queue {
54 self.gpu_queue()
55 }
56
57 #[cfg(feature = "gpu-runtime")]
58 pub async fn try_new() -> Result<Self, String> {
60 #[cfg(not(target_arch = "wasm32"))]
61 log::info!(
62 "LLM_LOAD|engine-init|0.10|Initializing native GGUF runtime (shared GpuContext)"
63 );
64 #[cfg(target_arch = "wasm32")]
65 log::info!("LLM_LOAD|engine-init|0.10|Initializing WASM GGUF runtime");
66
67 #[cfg(not(target_arch = "wasm32"))]
68 let shared = crate::gpu_context::shared_gpu();
69 #[cfg(not(target_arch = "wasm32"))]
70 let device = &shared.device;
71 #[cfg(not(target_arch = "wasm32"))]
72 let _queue = &shared.queue;
73 #[cfg(not(target_arch = "wasm32"))]
74 log::info!("LLM_LOAD|gpu-device|0.35|Reusing process-wide wgpu device");
75 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
78 {
79 let _ = crate::warm_cuda_context();
80 }
81
82 #[cfg(target_arch = "wasm32")]
83 let (wasm_device, wasm_queue) = {
84 crate::gguf_bridge::wasm_yield::phase("Acquiring shared WebGPU device…").await;
85 crate::gpu_context::ensure_shared_gpu().await?;
86 let shared = crate::gpu_context::try_shared_gpu().ok_or_else(|| {
87 "shared WebGPU device missing after ensure_shared_gpu".to_string()
88 })?;
89 let info = shared.adapter.get_info();
90 log::info!(
91 "LLM_LOAD|webgpu-adapter|0.20|name={} backend={:?} device={:?} vendor={:?}",
92 info.name,
93 info.backend,
94 info.device,
95 info.vendor
96 );
97 (shared.device.clone(), shared.queue.clone())
98 };
99 #[cfg(target_arch = "wasm32")]
100 let device = &wasm_device;
101 #[cfg(target_arch = "wasm32")]
103 crate::gguf_bridge::wasm_yield::phase(
104 "Compiling compute pipelines (UI may pause 30–90s on phones — leave tab open)…",
105 )
106 .await;
107 #[cfg(not(target_arch = "wasm32"))]
108 let native_pipeline_cache = create_native_pipeline_cache(device);
109 #[cfg(target_arch = "wasm32")]
110 let native_pipeline_cache: Option<etalon::PipelineCache> = None;
111
112 let resources = etalon_inference::gpu_resources::EngineResources::build(
113 device,
114 &native_pipeline_cache,
115 etalon_inference::gpu_resources::PipelineOptions {
116 subgroups: device.features().contains(etalon::Features::SUBGROUP),
117 },
118 etalon_inference::gpu_resources::ResourceBudget::for_device(device),
119 |_| Ok(()),
120 )
121 .map_err(|error| format!("inference GPU resources: {error:?}"))?;
122 let etalon_inference::gpu_resources::EngineResources {
123 gemv,
124 common,
125 ffn,
126 projections,
127 #[cfg(not(target_arch = "wasm32"))]
128 attention_kv_gemm_params,
129 #[cfg(not(target_arch = "wasm32"))]
130 attention_kv_params,
131 } = resources;
132 let etalon_inference::gpu_resources::GemvPipelines {
133 pipeline,
134 #[cfg(target_arch = "wasm32")]
135 mmv_q8_0_pipeline,
136 #[cfg(not(target_arch = "wasm32"))]
137 coop_gemv_pipeline,
138 #[cfg(not(target_arch = "wasm32"))]
139 coop_gemv_residual_pipeline,
140 #[cfg(not(target_arch = "wasm32"))]
141 coop_gemv_bind_layout,
142 #[cfg(not(target_arch = "wasm32"))]
143 coop_gemv_residual_bind_layout,
144 #[cfg(not(target_arch = "wasm32"))]
145 coop_gemv_mr_pipeline,
146 #[cfg(not(target_arch = "wasm32"))]
147 coop_gemv_residual_mr_pipeline,
148 #[cfg(not(target_arch = "wasm32"))]
149 coop_gemv_warp_pipeline,
150 #[cfg(not(target_arch = "wasm32"))]
151 coop_gemv_residual_warp_pipeline,
152 #[cfg(not(target_arch = "wasm32"))]
153 mock_pipeline,
154 #[cfg(target_arch = "wasm32")]
155 mc8_gemm_bind_layout,
156 #[cfg(not(target_arch = "wasm32"))]
157 pipeline_bind_layout,
158 } = gemv;
159 let etalon_inference::gpu_resources::CommonPipelines {
160 embedding_pipeline,
161 attention_pipeline,
162 elem_rms_norm_pipeline,
163 elem_silu_mul_pipeline,
164 elem_add_residual_pipeline,
165 #[cfg(target_arch = "wasm32")]
166 mc8_attn_bind_layout,
167 #[cfg(target_arch = "wasm32")]
168 mc8_elem_bind_layout,
169 #[cfg(not(target_arch = "wasm32"))]
170 embedding_bind_layout,
171 #[cfg(not(target_arch = "wasm32"))]
172 attention_bind_layout,
173 #[cfg(not(target_arch = "wasm32"))]
174 elem_silu_mul_bind_layout,
175 } = common;
176 let etalon_inference::gpu_resources::FfnPipelines {
177 #[cfg(target_arch = "wasm32")]
178 mc8_ffn_fused_bind_layout,
179 #[cfg(target_arch = "wasm32")]
180 mc8_ffn_fused_pipeline,
181 #[cfg(not(target_arch = "wasm32"))]
182 ffn_fused_bind_layout,
183 #[cfg(not(target_arch = "wasm32"))]
184 ffn_fused_pipeline,
185 #[cfg(not(target_arch = "wasm32"))]
186 ffn_fused_coop_pipeline,
187 #[cfg(not(target_arch = "wasm32"))]
188 ffn_fused_mr_pipeline,
189 #[cfg(not(target_arch = "wasm32"))]
190 ffn_fused_warp_pipeline,
191 } = ffn;
192 let etalon_inference::gpu_resources::ProjectionsPipelines {
193 #[cfg(not(target_arch = "wasm32"))]
194 dual_gemv_bind_layout,
195 #[cfg(not(target_arch = "wasm32"))]
196 dual_gemv_pipeline,
197 #[cfg(not(target_arch = "wasm32"))]
198 dual_gemv_mr_pipeline,
199 #[cfg(not(target_arch = "wasm32"))]
200 triple_gemv_bind_layout,
201 #[cfg(not(target_arch = "wasm32"))]
202 triple_gemv_pipeline,
203 } = projections;
204 #[cfg(not(target_arch = "wasm32"))]
205 log::info!(
206 "LLM_LOAD|fused_ffn|coop_entry={}|mr=coop_fused_ffn_mr|warp=coop_fused_ffn_warp",
207 if device.features().contains(etalon::Features::SUBGROUP) {
208 "coop_fused_ffn_sg"
209 } else {
210 "coop_fused_ffn_expansion"
211 }
212 );
213
214 #[cfg(target_os = "windows")]
218 let dml_status = {
219 let want = matches!(
220 crate::gguf_bridge::execution_env::var("QUALIA_DIRECTML")
221 .or_else(|_| crate::gguf_bridge::execution_env::var("QUALIA_LLM_DIRECTML"))
222 .ok()
223 .as_deref(),
224 Some("1") | Some("true") | Some("on")
225 );
226 if !want {
227 log::info!(
228 "LLM_LOAD|gpu-backend|0.45|DirectML deferred (set QUALIA_DIRECTML=1 to enable second D3D12 device)"
229 );
230 None
231 } else {
232 match crate::directml_bridge::DmlDevice::new() {
233 Ok(device) => {
234 log::info!(
235 "DirectML device initialization: Ok({})",
236 device.adapter_desc
237 );
238 log::info!(
239 "LLM_LOAD|gpu-backend|0.45|DirectML ready on {}",
240 device.adapter_desc
241 );
242 log::info!(
243 "LLM_LOAD|gpu-route|0.48|Streaming weights through DirectML with {:.1} GiB VRAM free",
244 bytes_to_gib(
245 device
246 .local_budget_bytes
247 .saturating_sub(device.local_usage_bytes)
248 )
249 );
250 Some(device)
251 }
252 Err(err) => {
253 log::warn!("DirectML device initialization failed: {:?}", err);
254 log::info!(
255 "LLM_LOAD|gpu-backend|0.45|DirectML unavailable; using wgpu fallback"
256 );
257 None
258 }
259 }
260 }
261 };
262
263 #[cfg(not(target_os = "windows"))]
264 {
265 log::info!("LLM_LOAD|gpu-backend|0.45|Using wgpu fallback backend for native compute");
266 }
267
268 #[cfg(not(target_arch = "wasm32"))]
269 let projection_cold_byte_budget = crate::gpu_context::global_vram_ledger().budget();
270 #[cfg(target_os = "windows")]
271 let projection_cold_byte_budget =
272 dml_status
273 .as_ref()
274 .map_or(projection_cold_byte_budget, |device| {
275 device
276 .local_budget_bytes
277 .saturating_sub(device.local_usage_bytes)
278 });
279 #[cfg(not(target_arch = "wasm32"))]
280 let projection_cold_byte_budget = usize::try_from(projection_cold_byte_budget).unwrap_or(0);
281
282 let engine = Self {
283 #[cfg(target_arch = "wasm32")]
284 device: wasm_device,
285 #[cfg(target_arch = "wasm32")]
286 queue: wasm_queue,
287 pipeline,
288 #[cfg(target_arch = "wasm32")]
289 mmv_q8_0_pipeline,
290 #[cfg(not(target_arch = "wasm32"))]
291 native_pipeline_cache,
292 #[cfg(not(target_arch = "wasm32"))]
293 pipeline_bind_layout,
294 #[cfg(not(target_arch = "wasm32"))]
295 coop_gemv_pipeline,
296 #[cfg(not(target_arch = "wasm32"))]
297 coop_gemv_bind_layout,
298 #[cfg(not(target_arch = "wasm32"))]
299 coop_gemv_mr_pipeline,
300 #[cfg(not(target_arch = "wasm32"))]
301 coop_gemv_residual_pipeline,
302 #[cfg(not(target_arch = "wasm32"))]
303 coop_gemv_residual_bind_layout,
304 #[cfg(not(target_arch = "wasm32"))]
305 coop_gemv_residual_mr_pipeline,
306 #[cfg(not(target_arch = "wasm32"))]
307 coop_gemv_warp_pipeline,
308 #[cfg(not(target_arch = "wasm32"))]
309 coop_gemv_residual_warp_pipeline,
310 #[cfg(not(target_arch = "wasm32"))]
311 mock_pipeline,
312 embedding_pipeline,
313 #[cfg(not(target_arch = "wasm32"))]
314 embedding_bind_layout,
315 attention_pipeline,
316 #[cfg(not(target_arch = "wasm32"))]
317 attention_bind_layout,
318 is_initialized: true,
319 #[cfg(target_os = "windows")]
320 dml: dml_status,
321 #[cfg(not(target_arch = "wasm32"))]
322 prepared_native_bytes: None,
323 #[cfg(not(target_arch = "wasm32"))]
324 prepared_native_sources: None,
325 #[cfg(not(target_arch = "wasm32"))]
326 qwen35_ftw: None,
327 #[cfg(not(target_arch = "wasm32"))]
328 qwen35_gguf: None,
329 #[cfg(not(target_arch = "wasm32"))]
330 qwen35_gguf_required: false,
331 #[cfg(not(target_arch = "wasm32"))]
332 gemma4: None,
333 #[cfg(not(target_arch = "wasm32"))]
334 gemma4_required: false,
335 gguf_mmap: None,
336 #[cfg(target_arch = "wasm32")]
337 cached_tokenizer: None,
338 #[cfg(target_arch = "wasm32")]
339 cached_tensor_index: None,
340 #[cfg(target_arch = "wasm32")]
341 cached_token_embd: None,
342 #[cfg(target_arch = "wasm32")]
343 p64_resident: None,
344 #[cfg(not(target_arch = "wasm32"))]
345 p64_index: None,
346 #[cfg(not(target_arch = "wasm32"))]
347 tensor_index_cache: None,
348 #[cfg(not(target_arch = "wasm32"))]
349 ftw_package: None,
350 tensor_data_offset: 0,
351 hyperparams: crate::gguf_sharder::GgufHyperparams::default(),
352 max_tensor_bytes: 0,
353 gemm_input_buf: None,
354 gemm_weight_buf: None,
355 #[cfg(target_arch = "wasm32")]
356 mc8_weight_arena: None,
357 #[cfg(target_arch = "wasm32")]
358 mc8_weights_resident: false,
359 #[cfg(target_arch = "wasm32")]
360 mc8_weight_role_stride: [0u64; 7],
361 #[cfg(target_arch = "wasm32")]
362 gemm_weight_buf_b: None,
363 gemm_output_buf: None,
364 gemm_params_buf: None,
365 gemm_output_staging: None,
366 output_topk_pipeline: None,
367 output_topk_bind_layout: None,
368 topk_cand_val_buf: None,
369 topk_cand_idx_buf: None,
370 topk_cand_staging: None,
371 topk_params_buf: None,
372 gemm_aux_buf: None,
373 gemm_ffn_buf: None,
374 #[cfg(target_arch = "wasm32")]
375 prefill_scratch_buf: None,
376 #[cfg(target_arch = "wasm32")]
377 prefill_work_buf_a: None,
378 #[cfg(target_arch = "wasm32")]
379 prefill_work_buf_b: None,
380 #[cfg(target_arch = "wasm32")]
381 mc8_q_proj_buf: None,
382 #[cfg(target_arch = "wasm32")]
383 mc8_k_proj_buf: None,
384 #[cfg(target_arch = "wasm32")]
385 mc8_v_proj_buf: None,
386 gemm_max_out_dim: MAX_STACK_GEMM_OUT as u32,
387 gemm_max_input_floats: 0,
388 kv_layout: None,
389 kv_cache_gpu: None,
390 kv_cache_cpu: None,
391 attention_params_buf: None,
392 attention_mask_buf: None,
393 elem_rms_norm_pipeline,
394 elem_silu_mul_pipeline,
395 #[cfg(not(target_arch = "wasm32"))]
396 elem_silu_mul_bind_layout,
397 elem_add_residual_pipeline,
398 elem_params_buf: None,
399 norm_weight_buf: None,
400 #[cfg(target_arch = "wasm32")]
401 mc8_gemm_bind_layout,
402 #[cfg(target_arch = "wasm32")]
403 mc8_elem_bind_layout,
404 #[cfg(target_arch = "wasm32")]
405 mc8_attn_bind_layout,
406 #[cfg(target_arch = "wasm32")]
407 mc8_ffn_fused_bind_layout,
408 #[cfg(target_arch = "wasm32")]
409 mc8_ffn_fused_pipeline,
410 #[cfg(not(target_arch = "wasm32"))]
411 ffn_fused_bind_layout,
412 #[cfg(not(target_arch = "wasm32"))]
413 ffn_fused_pipeline,
414 #[cfg(not(target_arch = "wasm32"))]
415 ffn_fused_coop_pipeline,
416 #[cfg(not(target_arch = "wasm32"))]
417 ffn_fused_mr_pipeline,
418 #[cfg(not(target_arch = "wasm32"))]
419 ffn_fused_warp_pipeline,
420 #[cfg(not(target_arch = "wasm32"))]
421 dual_gemv_pipeline,
422 #[cfg(not(target_arch = "wasm32"))]
423 dual_gemv_mr_pipeline,
424 #[cfg(not(target_arch = "wasm32"))]
425 dual_gemv_bind_layout,
426 #[cfg(not(target_arch = "wasm32"))]
427 triple_gemv_pipeline,
428 #[cfg(not(target_arch = "wasm32"))]
429 triple_gemv_bind_layout,
430 mc8_logits_resident_buf: None,
431 mc8_logits_row_bytes: 0,
432 #[cfg(not(target_arch = "wasm32"))]
433 ternary_ffn: None,
434 #[cfg(not(target_arch = "wasm32"))]
435 resident_decode: super::resident_decode::ResidentDecodeState::Unbuilt,
436 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
437 cuda_decode_plan: super::cuda_decode_plan::CudaDecodePlanState::Unbuilt,
438 #[cfg(not(target_arch = "wasm32"))]
439 prefill_arena: super::prefill_arena::PrefillArenaState::Unbuilt,
440 #[cfg(not(target_arch = "wasm32"))]
441 verify_arena: super::verify_arena::VerifyArenaState::Unbuilt,
442 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
443 projection_cold_byte_budget,
444 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
445 projection_readback: Default::default(),
446 #[cfg(not(target_arch = "wasm32"))]
447 gemm_resident_weights: std::sync::Mutex::new(
448 super::gemm::ResidentProjectionCache::default(),
449 ),
450 #[cfg(not(target_arch = "wasm32"))]
451 ffn_fused_params: None,
452 #[cfg(not(target_arch = "wasm32"))]
453 attention_kv_gemm_params: Some(attention_kv_gemm_params),
454 #[cfg(not(target_arch = "wasm32"))]
455 attention_kv_params: Some(attention_kv_params),
456 #[cfg(target_arch = "wasm32")]
457 mc8_norm_resident_buf: None,
458 #[cfg(target_arch = "wasm32")]
459 mc8_norm_stride: 0,
460 #[cfg(target_arch = "wasm32")]
461 mc8_bg_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
462 ssm_recurrent_state: Box::new([]),
463 ssm_state_byte_budget: DEFAULT_SSM_STATE_BYTE_BUDGET,
464 };
465 let mut relu_probe = [-1.0f32, 2.0];
468 let _ =
469 super::cpu_ops::apply_cpu_elem_op(super::gpu_params::ELEM_OP_RELU, &mut relu_probe, 2);
470 let _ = super::gpu_params::elem_op_label(super::gpu_params::ELEM_OP_RMS_NORM);
471 let _ = engine.elem_gpu_pipeline(super::gpu_params::ELEM_OP_RMS_NORM);
472 let _ = engine.elem_gpu_pipeline(super::gpu_params::ELEM_OP_ADD_RESIDUAL);
473 Ok(engine)
474 }
475
476 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
477 pub fn new() -> Self {
479 let handle = tokio::runtime::Handle::try_current().unwrap_or_else(|_| {
480 let rt = Box::leak(Box::new(tokio::runtime::Runtime::new().unwrap()));
481 rt.handle().clone()
482 });
483 tokio::task::block_in_place(|| {
484 handle
485 .block_on(Self::try_new())
486 .expect("Failed to initialize native GGUF engine")
487 })
488 }
489
490 #[cfg(not(feature = "gpu-runtime"))]
492 fn cpu_runtime() -> Self {
493 Self {
494 is_initialized: true,
495 #[cfg(target_os = "windows")]
496 dml: None,
497 #[cfg(not(target_arch = "wasm32"))]
498 prepared_native_bytes: None,
499 #[cfg(not(target_arch = "wasm32"))]
500 prepared_native_sources: None,
501 #[cfg(not(target_arch = "wasm32"))]
502 qwen35_ftw: None,
503 #[cfg(not(target_arch = "wasm32"))]
504 qwen35_gguf: None,
505 #[cfg(not(target_arch = "wasm32"))]
506 qwen35_gguf_required: false,
507 #[cfg(not(target_arch = "wasm32"))]
508 gemma4: None,
509 #[cfg(not(target_arch = "wasm32"))]
510 gemma4_required: false,
511 gguf_mmap: None,
512 #[cfg(target_arch = "wasm32")]
513 cached_tokenizer: None,
514 #[cfg(target_arch = "wasm32")]
515 cached_tensor_index: None,
516 #[cfg(target_arch = "wasm32")]
517 cached_token_embd: None,
518 #[cfg(target_arch = "wasm32")]
519 p64_resident: None,
520 #[cfg(not(target_arch = "wasm32"))]
521 p64_index: None,
522 #[cfg(not(target_arch = "wasm32"))]
523 tensor_index_cache: None,
524 #[cfg(not(target_arch = "wasm32"))]
525 ftw_package: None,
526 tensor_data_offset: 0,
527 hyperparams: crate::gguf_sharder::GgufHyperparams::default(),
528 max_tensor_bytes: 0,
529 #[cfg(target_arch = "wasm32")]
530 mc8_weights_resident: false,
531 #[cfg(target_arch = "wasm32")]
532 mc8_weight_role_stride: [0u64; 7],
533 gemm_max_out_dim: MAX_STACK_GEMM_OUT as u32,
534 gemm_max_input_floats: 0,
535 kv_layout: None,
536 kv_cache_cpu: None,
537 mc8_logits_row_bytes: 0,
538 #[cfg(target_arch = "wasm32")]
539 mc8_norm_stride: 0,
540 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
541 cuda_decode_plan: super::cuda_decode_plan::CudaDecodePlanState::Unbuilt,
542 ssm_recurrent_state: Box::new([]),
543 ssm_state_byte_budget: DEFAULT_SSM_STATE_BYTE_BUDGET,
544 }
545 }
546
547 #[cfg(not(feature = "gpu-runtime"))]
548 pub async fn try_new() -> Result<Self, String> {
550 log::info!(
551 "LLM_LOAD|engine-init|0.10|Initializing CPU-only GGUF runtime (gpu-runtime off)"
552 );
553 Ok(Self::cpu_runtime())
554 }
555
556 #[cfg(not(feature = "gpu-runtime"))]
557 pub fn new() -> Self {
559 Self::cpu_runtime()
560 }
561
562 #[cfg(feature = "gpu-runtime")]
563 pub(crate) fn ensure_kv_cache(&mut self, h: &crate::gguf_sharder::GgufHyperparams) {
565 #[cfg(not(target_arch = "wasm32"))]
566 let candidate = self.dense_qwen_kv_layout(h);
567 #[cfg(target_arch = "wasm32")]
568 let candidate = KvCacheLayout::from_hyperparams(h);
569 let layout = match candidate {
570 Some(l) => l,
571 None => {
572 #[cfg(target_arch = "wasm32")]
573 wlog(
574 "[kv_cache] FAILED from_hyperparams (zero dims or exceeds KV_CACHE_MAX_BYTES)",
575 );
576 return;
577 }
578 };
579 let bytes = (layout.total_f32_elems * std::mem::size_of::<f32>()) as etalon::BufferAddress;
580 let attn_params_bytes = {
581 #[cfg(target_arch = "wasm32")]
582 {
583 (MC8_MAX_ATTN_UNIFORM_CHUNK_SLOTS * MC8_UNIFORM_ALIGN) as etalon::BufferAddress
584 }
585 #[cfg(not(target_arch = "wasm32"))]
586 {
587 std::mem::size_of::<AttentionGpuParams>() as etalon::BufferAddress
588 }
589 };
590 use etalon_inference::gpu_resources::{KvBuffers, KvGeometry, ResourceBudget};
591 let resources = match KvBuffers::build(
592 self.gpu_device(),
593 KvGeometry {
594 cache_bytes: bytes,
595 attention_params_bytes: attn_params_bytes,
596 mask_bytes: (MAX_ATTN_MASK_UPLOAD_WORDS * std::mem::size_of::<u32>()).max(4) as u64,
597 },
598 ResourceBudget::for_device(self.gpu_device()),
599 |_| {
600 #[cfg(not(target_arch = "wasm32"))]
601 {
602 let ledger = crate::gpu_context::global_vram_ledger();
603 let orch = crate::gpu_context::universe_orchestrator();
604 if !ledger.can_allocate_in_universe(
605 &orch,
606 crate::gpu_context::ComputeUniverse::LlmInference,
607 bytes,
608 ) {
609 log::warn!(
610 "LLM_LOAD|kv-cache|denied|U0 budget {:.1} MiB used, need {:.1} MiB (mode {:?})",
611 ledger.universe_used_bytes(
612 crate::gpu_context::ComputeUniverse::LlmInference
613 ) as f64
614 / (1024.0 * 1024.0),
615 bytes as f64 / (1024.0 * 1024.0),
616 orch.active_mode,
617 );
618 return Err(etalon_inference::gpu_resources::ResourceError::PolicyDenied);
619 }
620 }
621 Ok(())
622 },
623 ) {
624 Ok(resources) => resources,
625 Err(error) => {
626 if !matches!(
627 error,
628 etalon_inference::gpu_resources::ResourceError::PolicyDenied
629 ) && (h.architecture == crate::gguf_sharder::ARCH_GEMMA4
630 || h.ssm_conv_kernel > 0
631 || (h.arch_flags & crate::gguf_sharder::ARCH_FLAG_HAS_QK_NORM) != 0
632 || crate::llm_bench::cpu_attention_enabled())
633 {
634 let cpu = vec![0f32; layout.total_f32_elems].into_boxed_slice();
635 self.kv_layout = Some(layout);
636 self.kv_cache_cpu = Some(cpu);
637 self.kv_cache_gpu = None;
638 log::info!(
639 "LLM_LOAD|kv-cache|0.86|Reserved {:.1} MiB host KV cache (context {})",
640 bytes as f64 / (1024.0 * 1024.0),
641 layout.max_context,
642 );
643 return;
644 }
645 log::warn!("inference KV resource admission failed: {error:?}");
646 return;
647 }
648 };
649 let gpu = resources.cache;
650 let cpu = vec![0f32; layout.total_f32_elems].into_boxed_slice();
651 self.attention_params_buf = Some(resources.attention_params);
652 self.attention_mask_buf = Some(resources.mask);
653 self.kv_layout = Some(layout);
654 self.kv_cache_gpu = Some(gpu);
655 self.kv_cache_cpu = Some(cpu);
656 #[cfg(not(target_arch = "wasm32"))]
658 self.upload_dict_atoms();
659 #[cfg(not(target_arch = "wasm32"))]
660 {
661 let ledger = crate::gpu_context::global_vram_ledger();
662 ledger.record_kv_cache(bytes);
663 }
664 log::info!(
665 "LLM_LOAD|kv-cache|0.86|Reserved {:.1} MiB KV cache (GPU + CPU mirror, context {})",
666 bytes as f64 / (1024.0 * 1024.0),
667 layout.max_context,
668 );
669 #[cfg(not(target_arch = "wasm32"))]
670 eprintln!(
671 "[gguf_bridge] KV arena {} slots ({:.1} MiB, {}), context={}",
672 layout.total_f32_elems,
673 bytes as f64 / (1024.0 * 1024.0),
674 if layout.dict_k > 0 {
675 "dict-coded"
676 } else if layout.int8 {
677 "int8+scale"
678 } else {
679 "f32"
680 },
681 layout.max_context,
682 );
683 }
684
685 #[cfg(not(feature = "gpu-runtime"))]
687 pub(crate) fn ensure_kv_cache(&mut self, h: &crate::gguf_sharder::GgufHyperparams) {
688 #[cfg(not(target_arch = "wasm32"))]
689 let candidate = self.dense_qwen_kv_layout(h);
690 #[cfg(target_arch = "wasm32")]
691 let candidate = KvCacheLayout::from_hyperparams(h);
692 let layout = match candidate {
693 Some(l) => l,
694 None => return,
695 };
696 let total = layout.total_f32_elems;
697 let max_context = layout.max_context;
698 let cpu = vec![0f32; total].into_boxed_slice();
699 self.kv_layout = Some(layout);
700 self.kv_cache_cpu = Some(cpu);
701 log::info!(
702 "LLM_LOAD|kv-cache|0.86|Reserved {:.1} MiB CPU KV cache (context {})",
703 (total * std::mem::size_of::<f32>()) as f64 / (1024.0 * 1024.0),
704 max_context,
705 );
706 }
707
708 pub fn reset_kv_cache(&mut self) {
710 self.reset_recurrent_state();
711 let Some(layout) = self.kv_layout.as_ref() else {
712 return;
713 };
714 let n = layout.total_f32_elems;
715 if let Some(cpu) = self.kv_cache_cpu.as_mut() {
716 for v in cpu.iter_mut().take(n) {
717 unsafe { core::ptr::write_volatile(v, 0.0) };
718 }
719 }
720 #[cfg(feature = "gpu-runtime")]
721 if let (Some(cpu), Some(gpu)) = (self.kv_cache_cpu.as_ref(), self.kv_cache_gpu.as_ref()) {
722 self.gpu_queue()
723 .write_buffer(gpu, 0, bytemuck::cast_slice(&cpu[..n]));
724 }
725 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
727 self.upload_dict_atoms();
728 }
729
730 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
733 pub(crate) fn upload_dict_atoms(&self) {
734 let Some(layout) = self.kv_layout.as_ref() else {
735 return;
736 };
737 if layout.dict_k == 0 {
738 return;
739 }
740 let (Some(buf), Some((flat, na, hd))) = (
741 self.kv_cache_gpu.as_ref(),
742 crate::kv_dict_runtime::atoms_flat(),
743 ) else {
744 return;
745 };
746 let code_region = (layout.max_context * 2 * layout.n_kv_head * layout.dict_k) as usize;
747 let per_layer_atoms = 2 * na * hd;
748 let ls = layout.layer_stride as usize;
749 for l in 0..layout.n_layer as usize {
750 let s = l * per_layer_atoms;
751 let e = ((l + 1) * per_layer_atoms).min(flat.len());
752 if e <= s {
753 break;
754 }
755 let dst_word = l * ls + code_region;
756 self.gpu_queue().write_buffer(
757 buf,
758 (dst_word * 4) as u64,
759 bytemuck::cast_slice(&flat[s..e]),
760 );
761 }
762 }
763
764 pub fn get_kv_cache_cpu(&self) -> Option<&[f32]> {
766 self.kv_cache_cpu.as_deref()
767 }
768
769 pub fn set_kv_cache_cpu(&mut self, data: &[f32]) -> bool {
771 let Some(layout) = self.kv_layout.as_ref() else {
772 return false;
773 };
774 let n = layout.total_f32_elems;
775 if data.len() < n {
776 return false;
777 }
778 let Some(cpu) = self.kv_cache_cpu.as_mut() else {
779 return false;
780 };
781 cpu[..n].copy_from_slice(&data[..n]);
782 #[cfg(feature = "gpu-runtime")]
783 if let (Some(cpu), Some(gpu)) = (self.kv_cache_cpu.as_ref(), self.kv_cache_gpu.as_ref()) {
784 self.gpu_queue()
785 .write_buffer(gpu, 0, bytemuck::cast_slice(&cpu[..n]));
786 }
787 true
788 }
789
790 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
795 pub fn read_kv_cache_gpu(&self) -> Option<Vec<f32>> {
796 let gpu = self.kv_cache_gpu.as_ref()?;
797 let layout = self.kv_layout.as_ref()?;
798 if layout.int8 {
799 return None;
800 }
801 let n = layout.total_f32_elems;
802 let size = (n * std::mem::size_of::<f32>()) as etalon::BufferAddress;
803 let staging = self.gpu_device().create_buffer(&etalon::BufferDescriptor {
804 label: Some("KvCacheReadback"),
805 size: size.max(4),
806 usage: etalon::BufferUsages::MAP_READ | etalon::BufferUsages::COPY_DST,
807 mapped_at_creation: false,
808 });
809 let mut encoder =
810 self.gpu_device()
811 .create_command_encoder(&etalon::CommandEncoderDescriptor {
812 label: Some("KvReadback"),
813 });
814 encoder.copy_buffer_to_buffer(gpu, 0, &staging, 0, size);
815 self.submit_commands(Some(encoder.finish()));
816
817 let slice = staging.slice(..);
818 let (tx, rx) = futures_channel::oneshot::channel();
819 slice.map_async(etalon::MapMode::Read, move |v| {
820 let _ = tx.send(v);
821 });
822 self.poll_wait();
823 let handle = tokio::runtime::Handle::try_current().unwrap_or_else(|_| {
824 let rt = Box::leak(Box::new(tokio::runtime::Runtime::new().unwrap()));
825 rt.handle().clone()
826 });
827 if handle.block_on(rx).ok()?.is_err() {
828 return None;
829 }
830 let data = slice
831 .get_mapped_range()
832 .expect("wgpu buffer map_range failed");
833 let out: Vec<f32> = bytemuck::cast_slice(&data).to_vec();
834 drop(data);
835 staging.unmap();
836 Some(out)
837 }
838
839 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
844 pub fn capture_kv_f32(
845 &self,
846 n_tokens: u32,
847 max_per_layer: usize,
848 ) -> Option<crate::kv_capture::KvCapture> {
849 let layout = *self.kv_layout.as_ref()?;
850 let flat = self.read_kv_cache_gpu()?;
851 let n_layer = layout.n_layer as usize;
852 let n_kv = layout.n_kv_head as usize;
853 let head_dim = layout.head_dim as usize;
854 if head_dim == 0 || n_kv == 0 || n_layer == 0 {
855 return None;
856 }
857 let take = (n_tokens as usize).min(layout.max_context as usize);
858 let mut k = vec![Vec::new(); n_layer];
859 let mut v = vec![Vec::new(); n_layer];
860 let read_vec = |base_of: &dyn Fn(u32) -> usize| -> Option<Vec<f32>> {
861 let mut out = Vec::with_capacity(head_dim);
862 for d in 0..head_dim {
863 let idx = base_of(d as u32);
864 if idx >= flat.len() {
865 return None;
866 }
867 out.push(flat[idx]);
868 }
869 Some(out)
870 };
871 for l in 0..n_layer {
872 for pos in 0..take {
873 let slot = layout.ring_slot(pos as u32);
874 for hkv in 0..n_kv {
875 if k[l].len() < max_per_layer {
876 if let Some(vec_k) =
877 read_vec(&|d| layout.k_index(l as u32, slot, hkv as u32, d))
878 {
879 k[l].push(vec_k);
880 }
881 }
882 if v[l].len() < max_per_layer {
883 if let Some(vec_v) =
884 read_vec(&|d| layout.v_index(l as u32, slot, hkv as u32, d))
885 {
886 v[l].push(vec_v);
887 }
888 }
889 }
890 }
891 }
892 Some(crate::kv_capture::KvCapture { head_dim, k, v })
893 }
894
895 #[cfg(feature = "gpu-runtime")]
896 pub(crate) fn ensure_gemm_buffers(&mut self, max_weight_bytes: usize, max_out_dim: u32) {
898 if self.output_topk_pipeline.is_none() {
901 self.init_output_topk();
902 }
903 let need_input = MAX_STACK_GEMM_IN.max(MAX_PREFILL_BATCH_FLOATS);
904 let prefill_bufs_ready = {
905 #[cfg(target_arch = "wasm32")]
906 {
907 self.prefill_scratch_buf.is_some()
908 && self.prefill_work_buf_a.is_some()
909 && self.prefill_work_buf_b.is_some()
910 }
911 #[cfg(not(target_arch = "wasm32"))]
912 {
913 true
914 }
915 };
916 #[cfg(target_arch = "wasm32")]
917 let weight_arena_ready = self.mc8_weight_arena.is_some();
918 #[cfg(not(target_arch = "wasm32"))]
919 let weight_arena_ready = true;
920 if self.gemm_weight_buf.is_some()
921 && max_weight_bytes <= self.max_tensor_bytes
922 && self.gemm_max_input_floats >= need_input
923 && prefill_bufs_ready
924 && weight_arena_ready
925 {
926 return;
927 }
928 use etalon_inference::gpu_resources::{GemmBuffers, GemmGeometry, ResourceBudget};
929 let gemm_params_bytes = {
930 #[cfg(target_arch = "wasm32")]
931 {
932 (MC8_MAX_GEMM_UNIFORM_CHUNK_SLOTS * MC8_UNIFORM_ALIGN) as u64
933 }
934 #[cfg(not(target_arch = "wasm32"))]
935 {
936 std::mem::size_of::<GemmGpuParams>() as u64
937 }
938 };
939 let elem_params_bytes = {
940 #[cfg(target_arch = "wasm32")]
941 {
942 (MC8_MAX_ELEM_UNIFORM_CHUNK_SLOTS * MC8_UNIFORM_ALIGN) as u64
943 }
944 #[cfg(not(target_arch = "wasm32"))]
945 {
946 std::mem::size_of::<ElemGpuParams>() as u64
947 }
948 };
949 let out_bytes = (u64::from(max_out_dim) * 4).max(4);
950 let resources = GemmBuffers::build(
951 self.gpu_device(),
952 GemmGeometry {
953 weight_bytes: max_weight_bytes.max(4) as u64,
954 input_bytes: (need_input * 4) as u64,
955 output_bytes: out_bytes,
956 gemm_params_bytes,
957 elem_params_bytes,
958 norm_bytes: (MAX_HIDDEN_DIM * 4) as u64,
959 staging_bytes: if cfg!(target_arch = "wasm32") {
960 out_bytes.max(65536 * 4)
961 } else {
962 out_bytes
963 },
964 #[cfg(target_arch = "wasm32")]
965 prefill_work_bytes: (PREFILL_CHUNK_SIZE
966 * (MAX_HIDDEN_DIM + 2 * max_out_dim as usize + MAX_HIDDEN_DIM).max(4)
967 * 4)
968 .max(4) as u64,
969 },
970 ResourceBudget::for_device(self.gpu_device()),
971 |_| Ok(()),
972 )
973 .unwrap_or_else(|error| panic!("inference GEMM resources: {error:?}"));
974 self.gemm_input_buf = Some(resources.input);
975 self.gemm_weight_buf = Some(resources.weight);
976 self.gemm_output_buf = Some(resources.output);
977 self.gemm_params_buf = Some(resources.params);
978 self.gemm_output_staging = Some(resources.staging);
979 self.gemm_aux_buf = Some(resources.aux);
980 self.gemm_ffn_buf = Some(resources.ffn);
981 self.elem_params_buf = Some(resources.elem_params);
982 self.norm_weight_buf = Some(resources.norm);
983 #[cfg(target_arch = "wasm32")]
984 {
985 let browser = resources.browser;
986 let [qkv_k, qkv_v, qkv_q, o_proj, gate, up, down] = browser.role_weights;
987 self.mc8_weight_arena = Some(Mc8WeightArenaBufs {
988 qkv_k,
989 qkv_v,
990 qkv_q,
991 o_proj,
992 gate,
993 up,
994 down,
995 });
996 self.gemm_weight_buf_b = Some(browser.weight_b);
997 self.prefill_scratch_buf = Some(browser.prefill_scratch);
998 self.prefill_work_buf_a = Some(browser.work_a);
999 self.prefill_work_buf_b = Some(browser.work_b);
1000 self.mc8_q_proj_buf = Some(browser.q_proj);
1001 self.mc8_k_proj_buf = Some(browser.k_proj);
1002 self.mc8_v_proj_buf = Some(browser.v_proj);
1003 }
1004 self.gemm_max_out_dim = max_out_dim;
1005 self.gemm_max_input_floats = need_input;
1006 self.max_tensor_bytes = max_weight_bytes;
1007 }
1008}
1009
1010#[cfg(not(feature = "gpu-runtime"))]
1012const _: fn() -> super::QTensorEngine = super::QTensorEngine::new;