1#![cfg_attr(not(feature = "gpu-runtime"), allow(dead_code))]
16#![allow(unused_imports, dead_code)]
17
18pub(crate) use crate::NQuin;
21pub(crate) use crate::gguf_sharder::GgufTensorInfo;
22use log;
23#[cfg(not(target_arch = "wasm32"))]
24pub(crate) use memmap2::MmapOptions;
25pub(crate) use std::sync::Arc;
26
27pub use crate::ggml_quants::{ExecutionError, fetch_token_embedding};
28
29pub fn dequantize_token_embedding_into(
31 raw: &[u8],
32 tensor: &GgufTensorInfo,
33 out: &mut [f32],
34) -> Result<usize, ExecutionError> {
35 let n_embd = tensor.dims[0] as usize;
36 if out.len() < n_embd {
37 return Err(ExecutionError::MmapBounds);
38 }
39 crate::ggml_quants::dequantize_row_into(raw, tensor.ggml_type, n_embd, out).map_err(|e| match e
40 {
41 crate::ggml_quants::GgmlDequantError::UnsupportedType => ExecutionError::UnsupportedType,
42 crate::ggml_quants::GgmlDequantError::BufferTooSmall
43 | crate::ggml_quants::GgmlDequantError::TruncatedInput => ExecutionError::MmapBounds,
44 })
45}
46
47#[derive(Debug, Clone)]
49pub struct QTensor {
50 pub shape: Vec<usize>,
51 pub byte_offset: u64,
52 pub is_quantized_q4_k: bool,
53}
54
55impl QTensor {
56 pub fn new(shape: Vec<usize>, byte_offset: u64, is_quantized_q4_k: bool) -> Self {
58 Self {
59 shape,
60 byte_offset,
61 is_quantized_q4_k,
62 }
63 }
64
65 pub fn map_from_pointer_with_index(
69 quin: &NQuin,
70 index: &crate::gguf_sharder::GgufTensorIndex,
71 ) -> Option<Self> {
72 use crate::QuinPointerExt;
73
74 let flag = quin.extract_modality_flag();
75 if flag != crate::MODALITY_FLAG_LLM_TENSOR && flag != crate::MODALITY_FLAG_VISION_TENSOR {
76 return None;
77 }
78
79 let offset = quin.extract_byte_offset();
80 let subject_hash = quin.subject;
81
82 for (name_hash, info) in &index.entries {
83 if *name_hash == subject_hash || info.byte_offset == offset {
84 let n_dims = (info.n_dims as usize).min(4);
85 let shape: Vec<usize> = info.dims[..n_dims].iter().map(|&d| d as usize).collect();
86 let is_quantized_q4_k = info.ggml_type == 12 || info.ggml_type == 112;
87 let abs_offset = index.tensor_data_start.saturating_add(info.byte_offset);
88 return Some(Self::new(shape, abs_offset, is_quantized_q4_k));
89 }
90 }
91 None
92 }
93
94 pub fn map_from_pointer(quin: &NQuin) -> Option<Self> {
98 use crate::QuinPointerExt;
99
100 let flag = quin.extract_modality_flag();
101 if flag != crate::MODALITY_FLAG_LLM_TENSOR && flag != crate::MODALITY_FLAG_VISION_TENSOR {
102 return None;
103 }
104
105 #[cfg(not(target_arch = "wasm32"))]
106 {
107 if let Some(mmap) = crate::resident_model::current_resident_mmap() {
108 let index = crate::gguf_sharder::GgufTensorIndex::from_gguf(&mmap);
109 return Self::map_from_pointer_with_index(quin, &index);
110 }
111 }
112
113 None
114 }
115}
116
117mod gpu_params;
121mod quant_support;
122#[cfg(feature = "gpu-runtime")]
123mod submission;
124pub(crate) use gpu_params::*;
125pub(crate) use quant_support::*;
126
127pub const KV_ATTENTION_MASK_WORDS: usize = crate::compute_universe::KV_ATTENTION_MASK_WORDS;
129
130#[cfg(target_arch = "wasm32")]
134#[inline]
135pub(crate) fn mc8_bg_hash(parts: &[u64]) -> u64 {
136 etalon_inference::browser_prefill::encoding::bg_hash(parts)
137}
138
139#[cfg(target_arch = "wasm32")]
141pub(crate) const MC8_UNIFORM_ALIGN: usize = 256;
142#[cfg(target_arch = "wasm32")]
143pub(crate) const MC8_MAX_GEMM_UNIFORM_SLOTS: usize = 8;
144#[cfg(target_arch = "wasm32")]
145pub(crate) const MC8_MAX_ELEM_UNIFORM_SLOTS: usize = 8;
146#[cfg(target_arch = "wasm32")]
147pub(crate) const MC8_MAX_ATTN_UNIFORM_SLOTS: usize = 8;
148#[cfg(target_arch = "wasm32")]
149pub(crate) const MC8_MAX_ELEM_UNIFORM_LAYER_SLOTS: usize = MC8_MAX_ELEM_UNIFORM_SLOTS;
150#[cfg(target_arch = "wasm32")]
154pub(crate) const MC8_LAYERS_PER_ENCODER: u32 = 64;
155#[cfg(target_arch = "wasm32")]
157pub(crate) const MC8_ATTN_SLOTS_PER_LAYER: usize = 4;
158#[cfg(target_arch = "wasm32")]
159pub(crate) const MC8_ELEM_SLOTS_PER_LAYER: usize = 6;
160#[cfg(target_arch = "wasm32")]
161pub(crate) const MC8_GEMM_SLOTS_PER_LAYER: usize = 8; #[cfg(target_arch = "wasm32")]
163pub(crate) const MC8_MAX_ATTN_UNIFORM_CHUNK_SLOTS: usize =
164 MC8_ATTN_SLOTS_PER_LAYER * MC8_LAYERS_PER_ENCODER as usize;
165#[cfg(target_arch = "wasm32")]
166pub(crate) const MC8_MAX_ELEM_UNIFORM_CHUNK_SLOTS: usize =
167 MC8_ELEM_SLOTS_PER_LAYER * MC8_LAYERS_PER_ENCODER as usize;
168#[cfg(target_arch = "wasm32")]
169pub(crate) const MC8_MAX_GEMM_UNIFORM_CHUNK_SLOTS: usize =
170 MC8_GEMM_SLOTS_PER_LAYER * MC8_LAYERS_PER_ENCODER as usize;
171
172#[cfg(target_arch = "wasm32")]
174pub(crate) use etalon_inference::browser_prefill::types::Mc8ChunkUniformCursors;
175
176#[cfg(target_arch = "wasm32")]
178pub(crate) use etalon_inference::browser_prefill::types::Mc8WeightRole;
179
180#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
182pub(crate) struct Mc8WeightArenaBufs {
183 qkv_k: etalon::Buffer,
184 qkv_v: etalon::Buffer,
185 qkv_q: etalon::Buffer,
186 o_proj: etalon::Buffer,
187 gate: etalon::Buffer,
188 up: etalon::Buffer,
189 down: etalon::Buffer,
190}
191
192#[cfg(target_arch = "wasm32")]
194pub(crate) use etalon_inference::browser_prefill::types::Mc8PrefillLayerUniforms;
195
196#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
198pub(crate) use etalon_inference::browser_prefill::types::Mc8PrefillLayerGeom;
199
200#[cfg(target_arch = "wasm32")]
202pub(crate) use etalon_inference::browser_prefill::types::Mc8UniformArena;
203
204#[cfg(target_arch = "wasm32")]
206pub(crate) use etalon_inference::browser_prefill::types::Mc8ElemUniformArena;
207
208#[cfg(target_arch = "wasm32")]
209pub(crate) use etalon_inference::browser_prefill::types::Mc8AttnUniformArena;
210
211#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
213pub struct WasmGpuPipeline {
214 encoder: etalon::CommandEncoder,
215}
216
217#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
222pub(crate) async fn await_wgpu_map(slice: etalon::BufferSlice<'_>) -> bool {
223 etalon_wasm::readback::await_map(slice, etalon_wasm::readback::browser_worker()).await
224}
225
226#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
227impl WasmGpuPipeline {
228 pub(crate) fn begin(engine: &QTensorEngine) -> Self {
230 Self {
231 encoder: engine
232 .device()
233 .create_command_encoder(&etalon::CommandEncoderDescriptor {
234 label: Some("MC8FusedEncoder"),
235 }),
236 }
237 }
238
239 pub(crate) fn finish(self) -> etalon::CommandBuffer {
241 self.encoder.finish()
242 }
243}
244
245#[cfg(target_arch = "wasm32")]
248mod mc8_wasm;
249
250pub const MAX_CONTEXT_WINDOW: u32 = 1024;
252pub const KV_CACHE_MAX_BYTES: usize = 768 * 1024 * 1024;
254
255#[derive(Debug, Clone, Copy, PartialEq, Eq)]
259pub struct KvCacheLayout {
260 pub max_context: u32,
261 pub n_layer: u32,
262 pub n_kv_head: u32,
263 pub head_dim: u32,
264 pub slot_kv_elems: u32,
265 pub layer_stride: u32,
266 pub total_f32_elems: usize,
267 pub int8: bool,
270 pub dict_k: u32,
274 pub dict_n_atoms: u32,
277}
278
279impl KvCacheLayout {
280 pub fn from_hyperparams(h: &crate::gguf_sharder::GgufHyperparams) -> Option<Self> {
282 #[cfg(not(target_arch = "wasm32"))]
286 let (dict_k, dict_n_atoms) = if crate::llm_bench::kv_dict_enabled() {
287 crate::kv_dict_runtime::installed_meta()
288 .filter(|&(_, hd, _)| hd == h.head_dim() as usize)
289 .map(|(k, _, na)| (k as u32, na as u32))
290 .unwrap_or((0, 0))
291 } else {
292 (0, 0)
293 };
294 #[cfg(target_arch = "wasm32")]
295 let (dict_k, dict_n_atoms) = (0u32, 0u32);
296
297 #[cfg(not(target_arch = "wasm32"))]
300 let want_int8 = dict_k == 0
301 && crate::llm_bench::kv_int8_enabled()
302 && (h.head_dim() % 4 == 0)
303 && h.head_dim() > 0;
304 #[cfg(target_arch = "wasm32")]
305 let want_int8 = false;
306 Self::from_hyperparams_mode(h, want_int8, dict_k, dict_n_atoms)
307 }
308
309 fn from_hyperparams_mode(
310 h: &crate::gguf_sharder::GgufHyperparams,
311 int8: bool,
312 dict_k: u32,
313 dict_n_atoms: u32,
314 ) -> Option<Self> {
315 let n_layer = h.n_layer;
316 let n_kv_head = h.effective_n_kv_head();
317 let head_dim = h.head_dim();
318 if n_layer == 0 || n_kv_head == 0 || head_dim == 0 {
319 return None;
320 }
321 let int8 = int8 && dict_k == 0;
323 let slot_kv_elems = n_kv_head * head_dim;
324 let layer_stride = if dict_k > 0 {
328 MAX_CONTEXT_WINDOW * 2 * n_kv_head * dict_k + 2 * dict_n_atoms * head_dim
331 } else if int8 {
332 MAX_CONTEXT_WINDOW * 2 * n_kv_head * (1 + head_dim / 4)
333 } else {
334 MAX_CONTEXT_WINDOW * slot_kv_elems * 2
335 };
336 let total = (n_layer as usize).checked_mul(layer_stride as usize)?;
337 let bytes = total.checked_mul(std::mem::size_of::<f32>())?;
338 if bytes > KV_CACHE_MAX_BYTES {
339 return None;
340 }
341 Some(Self {
342 max_context: MAX_CONTEXT_WINDOW,
343 n_layer,
344 n_kv_head,
345 head_dim,
346 slot_kv_elems,
347 layer_stride,
348 total_f32_elems: total,
349 int8,
350 dict_k,
351 dict_n_atoms,
352 })
353 }
354
355 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
359 pub(crate) fn dense_device_layout(self) -> Option<Self> {
360 let slot_kv_elems = self.n_kv_head.checked_mul(self.head_dim)?;
361 let layer_stride = self
362 .max_context
363 .checked_mul(slot_kv_elems)?
364 .checked_mul(2)?;
365 let total_f32_elems = (self.n_layer as usize).checked_mul(layer_stride as usize)?;
366 total_f32_elems
367 .checked_mul(std::mem::size_of::<f32>())
368 .filter(|bytes| *bytes <= KV_CACHE_MAX_BYTES)?;
369 Some(Self {
370 slot_kv_elems,
371 layer_stride,
372 total_f32_elems,
373 int8: false,
374 dict_k: 0,
375 dict_n_atoms: 0,
376 ..self
377 })
378 }
379
380 #[inline]
381 pub fn ring_slot(&self, token_idx: u32) -> u32 {
383 token_idx % self.max_context
384 }
385
386 #[inline]
387 pub fn k_index(&self, layer: u32, slot: u32, kv_head: u32, dim: u32) -> usize {
389 let base = layer as usize * self.layer_stride as usize
390 + slot as usize * self.slot_kv_elems as usize * 2;
391 base + kv_head as usize * self.head_dim as usize + dim as usize
392 }
393
394 #[inline]
395 pub fn v_index(&self, layer: u32, slot: u32, kv_head: u32, dim: u32) -> usize {
397 let k_base = layer as usize * self.layer_stride as usize
398 + slot as usize * self.slot_kv_elems as usize * 2;
399 let v_off = self.n_kv_head as usize * self.head_dim as usize;
400 k_base + v_off + kv_head as usize * self.head_dim as usize + dim as usize
401 }
402
403 #[inline]
408 pub fn code_index(&self, layer: u32, slot: u32, kv_head: u32, k_not_v: bool, i: u32) -> usize {
409 let dk = self.dict_k as usize;
410 let per_slot = 2 * self.n_kv_head as usize * dk;
411 let base = layer as usize * self.layer_stride as usize + slot as usize * per_slot;
412 let stream_off = if k_not_v {
413 0
414 } else {
415 self.n_kv_head as usize * dk
416 };
417 base + stream_off + kv_head as usize * dk + i as usize
418 }
419}
420
421const MAX_STACK_GEMM_DIM: usize = 20480;
423const MAX_STACK_GEMM_OUT: usize = MAX_STACK_GEMM_DIM;
424const MAX_STACK_GEMM_IN: usize = MAX_STACK_GEMM_DIM;
425const MAX_HIDDEN_DIM: usize = 4096;
427pub(crate) const RMS_NORM_EPS: f32 = 1e-5;
429pub const PREFILL_CHUNK_SIZE: usize = 64;
431const MAX_ATTN_MASK_UPLOAD_WORDS: usize = PREFILL_CHUNK_SIZE * KV_ATTENTION_MASK_WORDS;
433pub const MAX_PREFILL_BATCH_FLOATS: usize = 10240 * PREFILL_CHUNK_SIZE;
435pub const PREFILL_CHUNK_STACK_FLOATS: usize = 2560 * PREFILL_CHUNK_SIZE;
437const MAX_WGPU_WEIGHT_STAGING: usize = 64 * 1024 * 1024;
439pub const VOCAB_CHUNK_ROWS: usize = 10240;
444
445pub use etalon_inference::projections::StreamingArgmaxResult;
447
448#[derive(Debug, Clone, Copy, PartialEq, Eq)]
449pub struct GgufLoadReport {
450 pub mapped_bytes: u64,
451 pub tensor_data_offset: u64,
452 pub n_layer: u32,
453 pub n_head: u32,
454 pub n_kv_head: u32,
455 pub max_tensor_bytes: usize,
456 pub kv_cache_bytes: u64,
457 pub directml_enabled: bool,
458}
459
460mod cpu_ops;
464pub(crate) use cpu_ops::*;
465#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
466mod pipeline_cache;
467pub mod wasm_cpu;
470#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
471pub(crate) use pipeline_cache::*;
472
473mod async_dispatch;
476mod attention;
477mod attention_adapter;
478#[cfg(target_arch = "wasm32")]
479mod browser;
480#[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
481mod cuda_decode_plan;
482#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
483mod native_attention_adapter;
484#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
485mod native_resident_adapter;
486#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
487mod native_resident_planning_adapter;
488mod projection_adapter;
489#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
490mod projection_gpu_adapter;
491#[cfg(not(target_arch = "wasm32"))]
495pub(crate) const MAX_CUDA_CONTEXT_WINDOW: u32 = 4096;
496mod embedding;
497mod ffn;
498mod forward;
499#[cfg(not(target_arch = "wasm32"))]
500mod gemma4;
501mod gemma4_qwen35;
502mod moe_ffn;
503mod moe_gguf;
504mod ssm_forward;
505pub use etalon_inference::ordinary::ssm_layout::SsmStateError;
506pub const DEFAULT_SSM_STATE_BYTE_BUDGET: usize = 128 * 1024 * 1024;
508#[cfg(not(target_arch = "wasm32"))]
509pub(crate) use forward::PrefillDispatchFailure;
510mod gemm;
511mod init;
512mod load;
513mod output;
514#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
515mod prefill_arena;
516mod prefill_async;
517#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
518mod resident_decode;
519#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
520mod verify_arena;
521#[cfg(target_arch = "wasm32")]
523pub(crate) mod wasm_yield;
524
525#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
527fn probe_max_abs_diff(a: &[f32], b: &[f32], n: usize) -> f32 {
528 let n = n.min(a.len()).min(b.len());
529 let mut m = 0.0f32;
530 for i in 0..n {
531 m = m.max((a[i] - b[i]).abs());
532 }
533 m
534}
535
536#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
537fn probe_log_diff(phase: &str, cpu: &[f32], gpu: &[f32], n: usize) {
538 let n = n.min(8).min(cpu.len()).min(gpu.len());
539 if n == 0 {
540 return;
541 }
542 let err = probe_max_abs_diff(cpu, gpu, n);
543 wlog(&format!(
544 "[MC8 L0 diff] {phase}: cpu[0]={:.6} gpu[0]={:.6} max_abs_err={:.6}",
545 cpu[0], gpu[0], err
546 ));
547}
548
549#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
550fn probe_log_mid_diff(phase: &str, cpu: &[f32], gpu: &[f32], n: usize) {
551 let n = n.min(8).min(cpu.len()).min(gpu.len());
552 if n == 0 {
553 return;
554 }
555 let err = probe_max_abs_diff(cpu, gpu, n);
556 wlog(&format!(
557 "[MC8 L0 mid] {phase}: cpu[0]={:.6} gpu[0]={:.6} max_abs_err={:.6}",
558 cpu[0], gpu[0], err
559 ));
560}
561
562#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
563fn probe_log_ffn_diff(phase: &str, cpu: &[f32], gpu: &[f32], n: usize) {
564 let n = n.min(8).min(cpu.len()).min(gpu.len());
565 if n == 0 {
566 return;
567 }
568 let err = probe_max_abs_diff(cpu, gpu, n);
569 wlog(&format!(
570 "[MC8 L0 ffn] {phase}: cpu[0]={:.6} gpu[0]={:.6} max_abs_err={:.6}",
571 cpu[0], gpu[0], err
572 ));
573}
574
575#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
577async fn mc8_cpu_l0_ffn_stages(
578 engine: &QTensorEngine,
579 index: &crate::gguf_sharder::GgufTensorIndex,
580 mmap: &[u8],
581 layout: &KvCacheLayout,
582 hidden_cpu: &[f32],
583 n_embd: usize,
584 token_idx: u32,
585 post_attn: &mut [f32],
586 ffn_input: &mut [f32],
587 gate: &mut [f32],
588 up: &mut [f32],
589 swiglu: &mut [f32],
590 down: &mut [f32],
591) -> Option<usize> {
592 let tensors = index.get_layer_tensors(0);
593 let mut attn_out = [0f32; MAX_HIDDEN_DIM];
594 let q_dim = mc8_cpu_l0_attn_out(
595 engine,
596 index,
597 mmap,
598 layout,
599 hidden_cpu,
600 n_embd,
601 token_idx,
602 &mut attn_out,
603 )
604 .await?;
605 let out_info = tensors.attn_output.as_ref()?;
606 let o_raw =
607 crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, out_info).ok()?;
608 let (o_in, _) = QTensorEngine::matmul_dims(out_info);
609 if o_in > q_dim {
610 return None;
611 }
612 let mut o_proj = [0f32; MAX_HIDDEN_DIM];
613 if !stack_gemm_quant(
614 o_raw,
615 out_info,
616 &attn_out[..q_dim],
617 &mut o_proj[..n_embd],
618 o_in,
619 n_embd,
620 ) {
621 return None;
622 }
623 for i in 0..n_embd {
624 post_attn[i] = hidden_cpu[i] + o_proj[i];
625 }
626 let mut norm_w = [0f32; MAX_HIDDEN_DIM];
627 let mut ffn_norm_scratch = [0f32; MAX_HIDDEN_DIM];
628 let normed = prepare_pre_norm_input(
629 &post_attn[..n_embd],
630 n_embd,
631 tensors.ffn_norm.as_ref(),
632 Some(mmap),
633 index.tensor_data_start,
634 &mut ffn_norm_scratch,
635 &mut norm_w,
636 );
637 ffn_input[..n_embd].copy_from_slice(normed);
638 let gate_info = tensors.ffn_gate.as_ref()?;
639 let up_info = tensors.ffn_up.as_ref()?;
640 let down_info = tensors.ffn_down.as_ref()?;
641 let gate_raw =
642 crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, gate_info).ok()?;
643 let up_raw =
644 crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, up_info).ok()?;
645 let down_raw =
646 crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, down_info).ok()?;
647 let (gate_in, n_ffn) = QTensorEngine::matmul_dims(gate_info);
648 let (up_in, up_out) = QTensorEngine::matmul_dims(up_info);
649 let (dn_in, dn_out) = QTensorEngine::matmul_dims(down_info);
650 if gate_in > n_embd
651 || up_in != gate_in
652 || up_out != n_ffn
653 || dn_in != n_ffn
654 || n_ffn > gate.len()
655 || dn_out < n_embd
656 {
657 return None;
658 }
659 if !stack_gemm_quant(
660 gate_raw,
661 gate_info,
662 normed,
663 &mut gate[..n_ffn],
664 gate_in,
665 n_ffn,
666 ) {
667 return None;
668 }
669 if !stack_gemm_quant(up_raw, up_info, normed, &mut up[..n_ffn], up_in, n_ffn) {
670 return None;
671 }
672 for i in 0..n_ffn {
673 let g = gate[i];
674 let silu = g / (1.0 + (-g).exp());
675 swiglu[i] = silu * up[i];
676 }
677 if !stack_gemm_quant(
678 down_raw,
679 down_info,
680 &swiglu[..dn_in],
681 &mut down[..n_embd],
682 dn_in,
683 n_embd,
684 ) {
685 return None;
686 }
687 Some(n_ffn)
688}
689
690#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
692fn read_kv_cpu_head(
693 layout: &KvCacheLayout,
694 kv: &[f32],
695 layer: u32,
696 token_pos: u32,
697 kv_h: u32,
698 head_dim: usize,
699 k_not_v: bool,
700 out: &mut [f32],
701) -> bool {
702 if head_dim == 0 || head_dim > out.len() {
703 return false;
704 }
705 let slot = layout.ring_slot(token_pos);
706 for d in 0..head_dim {
707 let idx = if k_not_v {
708 layout.k_index(layer, slot, kv_h, d as u32)
709 } else {
710 layout.v_index(layer, slot, kv_h, d as u32)
711 };
712 if idx >= kv.len() {
713 return false;
714 }
715 out[d] = kv[idx];
716 }
717 true
718}
719
720#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
721fn probe_log_prefill_diff(phase: &str, cpu: &[f32], gpu: &[f32], n: usize) {
722 let n = n.min(8).min(cpu.len()).min(gpu.len());
723 if n == 0 {
724 return;
725 }
726 let err = probe_max_abs_diff(cpu, gpu, n);
727 wlog(&format!(
728 "[MC8 prefill] {phase}: cpu[0]={:.6} gpu[0]={:.6} max_abs_err={:.6}",
729 cpu[0], gpu[0], err
730 ));
731}
732
733#[cfg(all(target_arch = "wasm32", feature = "wasm-llm-diagnostics"))]
735async fn mc8_cpu_l0_attn_out(
736 engine: &QTensorEngine,
737 index: &crate::gguf_sharder::GgufTensorIndex,
738 mmap: &[u8],
739 layout: &KvCacheLayout,
740 hidden_cpu: &[f32],
741 n_embd: usize,
742 token_idx: u32,
743 out: &mut [f32],
744) -> Option<usize> {
745 let h = &index.hyperparams;
746 let tensors = index.get_layer_tensors(0);
747 let head_dim = h.head_dim() as usize;
748 let n_head = h.n_head as usize;
749 let q_heads_per_kv = h.q_heads_per_kv() as usize;
750 if head_dim == 0 || n_head == 0 || q_heads_per_kv == 0 {
751 return None;
752 }
753 let q_dim = n_head * head_dim;
754 if q_dim > out.len() {
755 return None;
756 }
757 let mut norm_w = [0f32; MAX_HIDDEN_DIM];
758 let mut norm_cpu = [0f32; MAX_HIDDEN_DIM];
759 let mut proj = [0f32; MAX_STACK_GEMM_OUT];
760 let normed = prepare_pre_norm_input(
761 &hidden_cpu[..n_embd],
762 n_embd,
763 tensors.attn_norm.as_ref(),
764 Some(mmap),
765 index.tensor_data_start,
766 &mut norm_cpu,
767 &mut norm_w,
768 );
769 let q_info = tensors.attn_q.as_ref()?;
770 let q_raw =
771 crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, q_info).ok()?;
772 let (q_in, q_out) = QTensorEngine::matmul_dims(q_info);
773 if q_out != q_dim || !stack_gemm_quant(q_raw, q_info, normed, &mut proj[..q_out], q_in, q_out) {
774 return None;
775 }
776 rope_inplace(
777 &mut proj[..q_out],
778 n_head,
779 head_dim,
780 token_idx,
781 h.effective_rope_freq_base(),
782 h.effective_rope_scale(),
783 );
784 let scale = 1.0f32 / (head_dim as f32).sqrt();
785 let mut k_slot = [0f32; 128];
786 let mut v_slot = [0f32; 128];
787 if head_dim > k_slot.len() {
788 return None;
789 }
790 for qh in 0..n_head {
791 let kv_h = qh / q_heads_per_kv;
792 let q_off = qh * head_dim;
793 let mut att_scores = [0f32; MAX_CONTEXT_WINDOW as usize];
794 let mut max_score = f32::NEG_INFINITY;
795 for past_pos in 0..=token_idx {
796 let past_slot = layout.ring_slot(past_pos);
797 if !engine
798 .pipeline_read_kv_head(
799 layout,
800 0,
801 past_slot,
802 kv_h as u32,
803 head_dim,
804 true,
805 &mut k_slot,
806 )
807 .await
808 {
809 return None;
810 }
811 let mut dot = 0.0f32;
812 for d in 0..head_dim {
813 dot += proj[q_off + d] * k_slot[d];
814 }
815 let score = dot * scale;
816 att_scores[past_pos as usize] = score;
817 max_score = max_score.max(score);
818 }
819 let mut sum_exp = 0.0f32;
820 for past_pos in 0..=token_idx {
821 let exp_val = (att_scores[past_pos as usize] - max_score).exp();
822 att_scores[past_pos as usize] = exp_val;
823 sum_exp += exp_val;
824 }
825 if sum_exp > 0.0 {
826 for past_pos in 0..=token_idx {
827 let prob = att_scores[past_pos as usize] / sum_exp;
828 let past_slot = layout.ring_slot(past_pos);
829 if !engine
830 .pipeline_read_kv_head(
831 layout,
832 0,
833 past_slot,
834 kv_h as u32,
835 head_dim,
836 false,
837 &mut v_slot,
838 )
839 .await
840 {
841 return None;
842 }
843 for d in 0..head_dim {
844 out[q_off + d] += v_slot[d] * prob;
845 }
846 }
847 }
848 }
849 Some(q_dim)
850}
851
852#[cfg(target_arch = "wasm32")]
854#[inline]
855pub(crate) fn wlog(s: &str) {
857 web_sys::console::log_1(&wasm_bindgen::JsValue::from_str(s));
858}
859#[cfg(not(target_arch = "wasm32"))]
860#[inline]
861pub(crate) fn wlog(s: &str) {
863 log::warn!("{s}");
864}
865
866pub static GPU_WAIT_COUNT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
870
871#[inline]
873pub fn gpu_wait_count() -> u64 {
874 GPU_WAIT_COUNT.load(std::sync::atomic::Ordering::Relaxed)
875}
876
877#[inline]
879pub fn reset_gpu_wait_count() {
880 GPU_WAIT_COUNT.store(0, std::sync::atomic::Ordering::Relaxed);
881}
882
883use etalon_inference::ordinary::rope::rope_inplace;
887
888#[cfg(test)]
892mod rope_stem_parity_tests {
893 #[test]
894 fn rope_kernel_is_the_stem_rotation() {
895 let n_heads = 2usize;
896 let head_dim = 8usize;
897 let (pos, base, scale) = (7u32, 10000.0f32, 1.0f32);
898 let xs: Vec<f32> = (0..n_heads * head_dim)
899 .map(|i| (i as f32 - 8.0) * 0.25)
900 .collect();
901
902 let mut got = xs.clone();
903 super::rope_inplace(&mut got, n_heads, head_dim, pos, base, scale);
904
905 let mut want: Vec<f64> = xs.iter().map(|&v| v as f64).collect();
906 crate::solvers::rope::rope_interleaved(
907 &mut want,
908 n_heads,
909 head_dim,
910 pos as f64,
911 base as f64,
912 scale as f64,
913 );
914
915 for i in 0..xs.len() {
916 assert!(
917 (got[i] as f64 - want[i]).abs() < 1e-4,
918 "RoPE kernel diverges from solvers::rope at {i}: {} vs {}",
919 got[i],
920 want[i]
921 );
922 }
923 }
924}
925
926pub(crate) fn stack_gemm_quant(
929 raw: &[u8],
930 info: &GgufTensorInfo,
931 input: &[f32],
932 out: &mut [f32],
933 n_in: usize,
934 n_out: usize,
935) -> bool {
936 if n_in > input.len() || n_out > out.len() || n_in > MAX_STACK_GEMM_IN {
937 wlog(&format!(
938 "[stack_gemm] GUARD tripped n_in={n_in} n_out={n_out} input={} out={} MAX_IN={MAX_STACK_GEMM_IN}",
939 input.len(),
940 out.len()
941 ));
942 return false;
943 }
944 #[cfg(target_arch = "wasm32")]
945 if info.ggml_type == crate::ggml_quants::GGML_TYPE_Q8_0 {
946 return wasm_cpu::q8_0_gemv_into(raw, input, out, n_in, n_out);
947 }
948 let mut row = [0f32; MAX_STACK_GEMM_IN];
949 for i in 0..n_out {
950 if crate::ggml_quants::dequant_matrix_row_into(raw, info, i, &mut row[..n_in]).unwrap_or(0)
951 < n_in
952 {
953 return false;
954 }
955 out[i] = row[..n_in]
956 .iter()
957 .zip(&input[..n_in])
958 .map(|(w, x)| w * x)
959 .sum();
960 }
961 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
962 etalon_agent::provider_scope::record_route(1);
963 true
964}
965
966mod state;
967pub use state::QTensorEngine;
968#[cfg(not(target_arch = "wasm32"))]
969mod provider;
970#[cfg(not(target_arch = "wasm32"))]
971pub use provider::{NativeProviderScope, NativeProviderSnapshot};
972mod model_bytes;
973#[cfg(not(target_arch = "wasm32"))]
974pub mod qwen35;
975#[cfg(not(target_arch = "wasm32"))]
976pub(crate) use model_bytes::ModelByteHandle;
977#[cfg(not(target_arch = "wasm32"))]
978pub(crate) use model_bytes::NativeModelSources;
979pub(crate) mod execution_env;
980
981#[cfg(target_arch = "wasm32")]
982thread_local! {
983 pub static WASM_ENGINE_INSTANCE: std::cell::RefCell<Option<QTensorEngine>> = std::cell::RefCell::new(None);
984}
985
986#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
993pub async fn initialize_webgpu_engine(model_data: std::sync::Arc<[u8]>) -> Result<(), String> {
994 let stamp = etalon_wasm::readback::begin_browser_model()
995 .map_err(|e| format!("Browser model context: {e:?}"))?;
996 initialize_webgpu_engine_with_stamp(model_data, stamp).await
997}
998
999#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
1000pub(crate) async fn initialize_webgpu_engine_with_stamp(
1001 model_data: std::sync::Arc<[u8]>,
1002 stamp: etalon_wasm::worker_steward::CompletionStamp,
1003) -> Result<(), String> {
1004 use wasm_yield::{clear_init_status, phase, set_init_status};
1005
1006 {
1007 let worker = etalon_wasm::readback::browser_worker();
1008 let worker = worker.lock().unwrap();
1009 if worker.completion_stamp() != stamp || !worker.is_ready() {
1010 return Err("Browser model initialization superseded or cancelled".into());
1011 }
1012 WASM_ENGINE_INSTANCE.with(|slot| *slot.borrow_mut() = None);
1013 }
1014
1015 phase("Requesting WebGPU adapter + device…").await;
1020 let mut engine = QTensorEngine::try_new().await?;
1021 phase("WebGPU device + pipelines ready — loading model (weights may take 1–3 min)…").await;
1022
1023 if crate::p64_weight::has_p64_magic(&model_data) {
1026 set_init_status(format!(
1027 "Loading P64 + uploading GPU weights ({:.0} MB)…",
1028 model_data.len() as f64 / (1024.0 * 1024.0)
1029 ));
1030 wasm_yield::yield_to_browser().await;
1032 engine.adopt_resident_p64(model_data)?;
1033 } else {
1034 set_init_status(format!(
1035 "Loading GGUF + uploading GPU weights ({:.0} MB)…",
1036 model_data.len() as f64 / (1024.0 * 1024.0)
1037 ));
1038 wasm_yield::yield_to_browser().await;
1039 engine.adopt_resident_mmap(model_data)?;
1040 }
1041 phase("Engine resident — ready").await;
1042 let worker = etalon_wasm::readback::browser_worker();
1043 let current = worker.lock().unwrap();
1044 if current.completion_stamp() != stamp || !current.is_ready() {
1045 return Err("Browser model initialization superseded or cancelled".into());
1046 }
1047 WASM_ENGINE_INSTANCE.with(|g| *g.borrow_mut() = Some(engine));
1048 clear_init_status();
1049 Ok(())
1050}
1051
1052#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
1053impl QTensorEngine {
1054 pub fn gemm_parity_probe(
1061 &mut self,
1062 info: &GgufTensorInfo,
1063 raw: &[u8],
1064 input: &[f32],
1065 gpu_out: &mut [f32],
1066 cpu_out: &mut [f32],
1067 n_in: usize,
1068 n_out: usize,
1069 ) -> bool {
1070 self.ensure_gemm_buffers(raw.len().max(1), n_out as u32);
1071 let gpu_ok = self.dispatch_gemm_raw_into(info, raw, input, gpu_out, n_in, n_out);
1072 let cpu_ok = stack_gemm_quant(raw, info, input, cpu_out, n_in, n_out);
1073 gpu_ok && cpu_ok
1074 }
1075}
1076
1077mod cpu_attention_adapter;
1078
1079#[cfg(target_arch = "wasm32")]
1080mod async_attention_adapter;
1081
1082#[cfg(target_arch = "wasm32")]
1083mod prefill_adapter;
1084
1085#[cfg(target_arch = "wasm32")]
1086mod browser_projection_adapter;