pub struct QTensorEngine {
pub pipeline: ComputePipeline,
pub embedding_pipeline: ComputePipeline,
pub is_initialized: bool,
pub gguf_mmap: Option<Arc<Mmap>>,
pub p64_index: Option<P64TensorIndex>,
pub tensor_index_cache: Option<GgufTensorIndex>,
pub ftw_package: Option<Arc<FtwModelPackage>>,
pub tensor_data_offset: u64,
pub hyperparams: GgufHyperparams,
pub max_tensor_bytes: usize,
/* private fields */
}Fields§
§pipeline: ComputePipeline§embedding_pipeline: ComputePipelineGPU-side Q6_K embedding dequant + matmul (zero CPU dequant).
is_initialized: bool§gguf_mmap: Option<Arc<Mmap>>Memory-mapped GGUF file (set after load_gguf).
p64_index: Option<P64TensorIndex>Cached P64 index after adopt_resident_p64_* — decode must not re-CRC the container.
tensor_index_cache: Option<GgufTensorIndex>Cached synthetic GGUF index built from p64_index (or from GGUF parse).
ftw_package: Option<Arc<FtwModelPackage>>FTW multi-shard package for native MoE execution.
tensor_data_offset: u64Byte offset into the mmap where tensor data begins.
hyperparams: GgufHyperparams§max_tensor_bytes: usizeImplementations§
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn dispatch_output_logits_into(
&self,
index: &GgufTensorIndex,
hidden: &[f32],
emb_dim: usize,
logits_out: &mut [f32],
) -> usize
pub fn dispatch_output_logits_into( &self, index: &GgufTensorIndex, hidden: &[f32], emb_dim: usize, logits_out: &mut [f32], ) -> usize
Final logits via chunked projection into logits_out (fills min(vocab, buf) rows).
Sourcepub fn decode_lexicon_bound(
&self,
_logits: &[f32],
valid_lexicon_ids: &[u64],
) -> u64
pub fn decode_lexicon_bound( &self, _logits: &[f32], valid_lexicon_ids: &[u64], ) -> u64
Decodes lexicon_bound; fails closed on malformed input.
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn dispatch_forge_causal_attention(
&mut self,
query: &[f32],
keys: &[f32],
values: &[f32],
scale: f32,
shape: AttentionShape,
out: &mut [f32],
) -> Result<(), ForgeError>
pub fn dispatch_forge_causal_attention( &mut self, query: &[f32], keys: &[f32], values: &[f32], scale: f32, shape: AttentionShape, out: &mut [f32], ) -> Result<(), ForgeError>
Certified Naga-validated WGSL Forge causal attention device dispatch during decode.
Native + wgsl-forge only — wasm profiles do not compile the Forge module.
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn try_native_projection_block(
&self,
tensor: &ProjectionTensor<'_>,
input: &[f32],
) -> Result<Option<Vec<f32>>, &'static str>
pub fn try_native_projection_block( &self, tensor: &ProjectionTensor<'_>, input: &[f32], ) -> Result<Option<Vec<f32>>, &'static str>
Fallible native projection route. Governed callers may configure the admitted budget with set_projection_cold_byte_budget; ordinary preparation inherits the native host/device policy.
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn dispatch_quantized_token_embedding(
&self,
raw_embd: &[u8],
ggml_type: u32,
n_embd: u32,
weight_tensor: &QTensor,
) -> Option<Vec<f32>>
pub fn dispatch_quantized_token_embedding( &self, raw_embd: &[u8], ggml_type: u32, n_embd: u32, weight_tensor: &QTensor, ) -> Option<Vec<f32>>
Upload raw quantized embedding bytes to the GPU and matmul without CPU dequant.
Returns None when the GGML type has no WGSL kernel (caller uses CPU fallback).
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn dispatch_prefill_chunk(
&mut self,
index: &GgufTensorIndex,
batch_hidden: &mut [f32],
emb_dim: usize,
n_tokens: u32,
batch_start_token_idx: u32,
scratch_a: &mut [f32],
scratch_b: &mut [f32],
max_layers: u32,
) -> Result<(), PrefillDispatchFailure>
pub fn dispatch_prefill_chunk( &mut self, index: &GgufTensorIndex, batch_hidden: &mut [f32], emb_dim: usize, n_tokens: u32, batch_start_token_idx: u32, scratch_a: &mut [f32], scratch_b: &mut [f32], max_layers: u32, ) -> Result<(), PrefillDispatchFailure>
Chunked prefill: populate KV arena for n_tokens prompt positions starting at batch_start.
Sourcepub fn dispatch_transformer_layer(
&mut self,
index: &GgufTensorIndex,
layer: u32,
token_idx: u32,
hidden: &mut [f32],
emb_dim: usize,
scratch_a: &mut [f32],
scratch_b: &mut [f32],
) -> bool
pub fn dispatch_transformer_layer( &mut self, index: &GgufTensorIndex, layer: u32, token_idx: u32, hidden: &mut [f32], emb_dim: usize, scratch_a: &mut [f32], scratch_b: &mut [f32], ) -> bool
One transformer block using real mmap tensor offsets (stack buffers only).
Sourcepub fn dispatch_transformer_forward(
&mut self,
index: &GgufTensorIndex,
hidden: &mut [f32],
emb_dim: usize,
scratch_a: &mut [f32],
scratch_b: &mut [f32],
token_idx: u32,
max_layers: u32,
) -> u32
pub fn dispatch_transformer_forward( &mut self, index: &GgufTensorIndex, hidden: &mut [f32], emb_dim: usize, scratch_a: &mut [f32], scratch_b: &mut [f32], token_idx: u32, max_layers: u32, ) -> u32
Sequential layer-by-layer forward (one tensor payload in VRAM at a time).
max_layers: 0 runs all blocks; otherwise caps how many layers execute.
Sourcepub fn verify_topology_draft_batch(
&mut self,
index: &GgufTensorIndex,
ctx: &mut Vec<u32>,
draft: &TopologyDraftBatch,
emb_dim: usize,
emb_buf: &mut [f32],
scratch_a: &mut [f32],
scratch_b: &mut [f32],
max_layers: u32,
max_vocab_chunks: u32,
) -> u32
pub fn verify_topology_draft_batch( &mut self, index: &GgufTensorIndex, ctx: &mut Vec<u32>, draft: &TopologyDraftBatch, emb_dim: usize, emb_buf: &mut [f32], scratch_a: &mut [f32], scratch_b: &mut [f32], max_layers: u32, max_vocab_chunks: u32, ) -> u32
Topological speculative verify — accept longest draft prefix (B3.1d).
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn dispatch_gemma4_token(
&mut self,
index: &GgufTensorIndex,
token: u32,
position: u32,
hidden: &mut [f32],
scratch_a: &mut [f32],
scratch_b: &mut [f32],
) -> u32
pub fn dispatch_gemma4_token( &mut self, index: &GgufTensorIndex, token: u32, position: u32, hidden: &mut [f32], scratch_a: &mut [f32], scratch_b: &mut [f32], ) -> u32
Text-only Gemma4 token step. The owner reads the actual retained embedding and PLE token row, so hidden-only callers cannot silently substitute token0. Returns all layers or zero; an interrupted/failed token requires reset.
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn adopt_ftw_package(
&mut self,
dir: &Path,
) -> Result<GgufLoadReport, String>
pub fn adopt_ftw_package( &mut self, dir: &Path, ) -> Result<GgufLoadReport, String>
Adopt an FTW multi-shard model directory for native MoE execution.
Sourcepub fn adopt_ftw_package_with_budget(
&mut self,
dir: &Path,
max_context: usize,
model_state_byte_budget: usize,
) -> Result<GgufLoadReport, String>
pub fn adopt_ftw_package_with_budget( &mut self, dir: &Path, max_context: usize, model_state_byte_budget: usize, ) -> Result<GgufLoadReport, String>
Select the external whole-model state budget and context capacity for admitted Qwen execution. This is separate from the legacy SSM-only budget and the 42 MiB Sentinel arena; the default reserves at most 128 MiB.
Sourcepub fn fetch_tensor_raw_bytes<'a>(
&'a self,
info: &GgufTensorInfo,
) -> Option<&'a [u8]>
pub fn fetch_tensor_raw_bytes<'a>( &'a self, info: &GgufTensorInfo, ) -> Option<&'a [u8]>
Read raw bytes for a tensor from either the FTW package shards or the primary GGUF mmap.
Sourcepub fn dispatch_moe_step_routed<F>(
&mut self,
emb_dim: usize,
router_weights: &[f32],
num_experts: usize,
topk: usize,
expert_fetcher: F,
shared_expert: Option<&ExpertWeightView<'_>>,
shared_gate_weight: f32,
ffn_input: &[f32],
scratch_a: &mut [f32],
) -> bool
pub fn dispatch_moe_step_routed<F>( &mut self, emb_dim: usize, router_weights: &[f32], num_experts: usize, topk: usize, expert_fetcher: F, shared_expert: Option<&ExpertWeightView<'_>>, shared_gate_weight: f32, ffn_input: &[f32], scratch_a: &mut [f32], ) -> bool
Dispatches MoE layer using the zero-heap dispatch_moe_step driver with expert fetcher (EOS-070).
Sourcepub fn dispatch_clustered_moe_operator(
&mut self,
op: &ClusteredMoEOperator,
routed_experts: &[RoutedExpert],
ffn_input: &[f32],
output: &mut [f32],
scratch: &mut ClusteredMoeScratch<'_>,
) -> bool
pub fn dispatch_clustered_moe_operator( &mut self, op: &ClusteredMoEOperator, routed_experts: &[RoutedExpert], ffn_input: &[f32], output: &mut [f32], scratch: &mut ClusteredMoeScratch<'_>, ) -> bool
Dispatches MoE layer using the clustered shared-down MoE operator (EOS-071).
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn set_ssm_state_byte_budget(&mut self, byte_budget: usize)
pub fn set_ssm_state_byte_budget(&mut self, byte_budget: usize)
Select the external recurrent-state budget for subsequent cold model loads. This arena is separate from the 42 MiB Sentinel arena.
pub fn init_ssm_state_with_budget( &mut self, index: &GgufTensorIndex, byte_budget: usize, ) -> Result<usize, SsmStateError>
Sourcepub fn init_ssm_state(&mut self, index: &GgufTensorIndex) -> usize
pub fn init_ssm_state(&mut self, index: &GgufTensorIndex) -> usize
Compatibility entry point. Loaders use fallible admission; a refused arena remains empty and hybrid dispatch fails closed.
pub fn reset_recurrent_state(&mut self)
Sourcepub fn snapshot_recurrent_state_into(&self, out: &mut [f32]) -> Option<usize>
pub fn snapshot_recurrent_state_into(&self, out: &mut [f32]) -> Option<usize>
Export a zero-heap snapshot of the current hybrid SSM / GatedDeltaNet recurrent state
(ssm_recurrent_state) into caller-supplied out (f32 slice).
Returns Some(self.ssm_recurrent_state.len()) when out has sufficient capacity.
Sourcepub fn restore_recurrent_state_from_slice(&mut self, src: &[f32]) -> bool
pub fn restore_recurrent_state_from_slice(&mut self, src: &[f32]) -> bool
Restore a hybrid SSM / GatedDeltaNet recurrent state snapshot from a .p64 neural
extension cartridge or pre-warmed scene seed into self.ssm_recurrent_state (zero-heap).
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn set_projection_cold_byte_budget(&mut self, bytes: usize)
pub fn set_projection_cold_byte_budget(&mut self, bytes: usize)
Set the admitted cold native-projection/readback payload budget. This legacy Vec-returning entrypoint reports native admission exhaustion by panic instead of choosing another layout.
Sourcepub fn set_resident_projection_byte_budget(&mut self, bytes: u64)
pub fn set_resident_projection_byte_budget(&mut self, bytes: u64)
Set aggregate projection-weight allowance. Admission misses retain uncached GPU/CPU fallback.
Sourcepub fn dispatch_gemm_into(
&self,
index: &GgufTensorIndex,
info: &GgufTensorInfo,
input: &[f32],
out: &mut [f32],
n_in: usize,
n_out: usize,
) -> bool
pub fn dispatch_gemm_into( &self, index: &GgufTensorIndex, info: &GgufTensorInfo, input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool
Quantized GEMM into caller out using reused GPU buffers (Q6_K) or CPU dequant fallback.
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub async fn try_new() -> Result<Self, String>
pub async fn try_new() -> Result<Self, String>
Tries to build the engine; error when GPU init fails.
Sourcepub fn reset_kv_cache(&mut self)
pub fn reset_kv_cache(&mut self)
Zero the static KV arena at the start of a new decode context (zero heap in decode).
Sourcepub fn get_kv_cache_cpu(&self) -> Option<&[f32]>
pub fn get_kv_cache_cpu(&self) -> Option<&[f32]>
Returns kv_cache_cpu.
Sourcepub fn set_kv_cache_cpu(&mut self, data: &[f32]) -> bool
pub fn set_kv_cache_cpu(&mut self, data: &[f32]) -> bool
Sets kv_cache_cpu.
Sourcepub fn read_kv_cache_gpu(&self) -> Option<Vec<f32>>
pub fn read_kv_cache_gpu(&self) -> Option<Vec<f32>>
Read the entire GPU KV-cache arena back to host as f32 (native, cold path — forge KV capture,
not the decode hot path). The returned flat buffer is interpretable via KvCacheLayout:: k_index/v_index only when the layout is f32 (int8 KV disabled at load); with an int8
layout the bytes are packed i8 lanes + scales and this returns None.
Sourcepub fn capture_kv_f32(
&self,
n_tokens: u32,
max_per_layer: usize,
) -> Option<KvCapture>
pub fn capture_kv_f32( &self, n_tokens: u32, max_per_layer: usize, ) -> Option<KvCapture>
Decode the current f32 KV cache into per-layer K and V vectors for token positions 0..n_tokens
(capped at max_per_layer per layer per stream). The GPU-readback capture route for the
sparse-KV-dictionary go/no-go — reads the real decode-path K/V straight from VRAM, no CPU
reference forward. Returns None on an int8 layout or if readback fails.
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn kv_layout(&self) -> Option<KvCacheLayout>
pub fn kv_layout(&self) -> Option<KvCacheLayout>
KV-cache layout when the engine allocates one.
Sourcepub fn kv_cache_bytes(&self) -> u64
pub fn kv_cache_bytes(&self) -> u64
KV-cache size in bytes.
Sourcepub fn load_gguf_checked(
&mut self,
path: &str,
) -> Result<GgufLoadReport, String>
pub fn load_gguf_checked( &mut self, path: &str, ) -> Result<GgufLoadReport, String>
Loads gguf-checked.
Sourcepub fn load_gguf(&mut self, path: &str)
pub fn load_gguf(&mut self, path: &str)
Memory-map a GGUF file so tensor bytes are accessible without heap allocation.
Call this once after new(), before the first dispatch_fused_transformer_block.
Sourcepub fn load_model_checked(
&mut self,
path: &str,
) -> Result<GgufLoadReport, String>
pub fn load_model_checked( &mut self, path: &str, ) -> Result<GgufLoadReport, String>
Memory-map and auto-detect a supported local model container.
Canonical P64 is detected by its exact p64\0 magic and adopted through
the P64 validation path. All other inputs are passed to the GGUF parser,
which rejects malformed or unsupported data.
Sourcepub fn adopt_resident_safetensor_mmap(
&mut self,
mmap: Arc<Mmap>,
config_json: Option<&str>,
) -> Result<GgufLoadReport, String>
pub fn adopt_resident_safetensor_mmap( &mut self, mmap: Arc<Mmap>, config_json: Option<&str>, ) -> Result<GgufLoadReport, String>
Boot directly from an already-mapped Safetensors model (including BF16).
Sourcepub fn load_model(&mut self, path: &str)
pub fn load_model(&mut self, path: &str)
Fail-soft wrapper retained for the agent decode path.
Sourcepub fn adopt_resident_p64_mmap(
&mut self,
mmap: Arc<Mmap>,
) -> Result<GgufLoadReport, String>
pub fn adopt_resident_p64_mmap( &mut self, mmap: Arc<Mmap>, ) -> Result<GgufLoadReport, String>
Boot from an already-mapped P64 weight container (native). Mirrors the GGUF
adopt_resident_mmap but for the P64 format: validates + builds a synthetic GGUF index
from the manifest, points the byte source at the P64 bytes (tensor_data_start = 0,
absolute blob offsets), reserves the GEMM/KV arenas, makes the (verbatim) output projection
resident, and builds the resident 2-bit ternary-FFN dispatcher from the FFN blobs. The
attention/norm/embed tensors stay at source precision and run the standard GGUF hot path.
Sourcepub fn adopt_resident_q42_mmap(
&mut self,
mmap: Arc<Mmap>,
) -> Result<GgufLoadReport, String>
👎Deprecated: use adopt_resident_p64_mmap
pub fn adopt_resident_q42_mmap( &mut self, mmap: Arc<Mmap>, ) -> Result<GgufLoadReport, String>
use adopt_resident_p64_mmap
Compatibility alias for the historical pre-P64 API name.
Sourcepub fn ternary_ffn_resident_len(&self) -> usize
pub fn ternary_ffn_resident_len(&self) -> usize
A1b: number of resident ternary FFN tensors (0 unless a ternary P64 was adopted). Lets a test confirm the GPU resident path is actually populated (not a silent CPU-only fallback).
Sourcepub fn adopt_resident_mmap(
&mut self,
mmap: Arc<Mmap>,
) -> Result<GgufLoadReport, String>
pub fn adopt_resident_mmap( &mut self, mmap: Arc<Mmap>, ) -> Result<GgufLoadReport, String>
Attach an already-mapped resident GGUF (shared with orchestrator slot).
Sourcepub fn bench_empty_submit_roundtrip(&self, n: u32) -> u64
pub fn bench_empty_submit_roundtrip(&self, n: u32) -> u64
Decode-profiler: wall-clock for n EMPTY submit → poll(Maintain::Wait) round-trips (no
compute dispatched). Isolates the fixed CPU↔GPU fence latency: if a token’s forward time ≈
(its round-trip count × this per-round-trip cost), the bottleneck is synchronization, not
math; if forward ≫ that, the kernels themselves are slow. Does NOT touch GPU_WAIT_COUNT.
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn dispatch_output_argmax_chunked(
&self,
index: &GgufTensorIndex,
hidden: &[f32],
emb_dim: usize,
chunk_logits: &mut [f32],
max_chunks: u32,
sieve_mask: Option<&SieveStateMask>,
) -> Option<StreamingArgmaxResult>
pub fn dispatch_output_argmax_chunked( &self, index: &GgufTensorIndex, hidden: &[f32], emb_dim: usize, chunk_logits: &mut [f32], max_chunks: u32, sieve_mask: Option<&SieveStateMask>, ) -> Option<StreamingArgmaxResult>
Chunked vocabulary projection with streaming argmax (zero heap, stack chunk buffer only).
max_chunks: 0 sweeps the full vocabulary; otherwise caps chunk iterations (tests).
Sourcepub fn dispatch_output_top1_chunked(
&self,
index: &GgufTensorIndex,
hidden: &[f32],
emb_dim: usize,
) -> Option<StreamingArgmaxResult>
pub fn dispatch_output_top1_chunked( &self, index: &GgufTensorIndex, hidden: &[f32], emb_dim: usize, ) -> Option<StreamingArgmaxResult>
Native decode fast path: output projection plus GPU block argmax (k=1) with one tiny
candidate readback after all vocab chunks have been submitted. This avoids the full
chunk-logit readback in Self::dispatch_output_argmax_chunked and avoids heap allocation
in the decode loop.
Sourcepub fn dispatch_output_topk_chunked(
&self,
index: &GgufTensorIndex,
hidden: &[f32],
emb_dim: usize,
k: usize,
) -> Option<Vec<TopKItem>>
pub fn dispatch_output_topk_chunked( &self, index: &GgufTensorIndex, hidden: &[f32], emb_dim: usize, k: usize, ) -> Option<Vec<TopKItem>>
A1a: GPU top-k over the output projection — the logits stay on-GPU (gemm_output_buf), the
top-k kernel reduces them per chunk, and only K (id, logit) candidates are read back (vs the
196 KB/token full-logit readback + CPU argmax in dispatch_output_argmax_chunked). Returns the
merged global top-K, or None to signal the caller to fall back to the argmax path. v1: no
sieve coupling (caller routes here only when no sieve mask is active).
Sourcepub fn apply_output_norm_inplace(
&self,
index: &GgufTensorIndex,
hidden: &mut [f32],
emb_dim: usize,
) -> bool
pub fn apply_output_norm_inplace( &self, index: &GgufTensorIndex, hidden: &mut [f32], emb_dim: usize, ) -> bool
Final output_norm RMSNorm in-place before vocabulary projection (Pre-Norm LLM tail).
REQUIRED on all targets — native previously skipped it → logits from an un-normed hidden.
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn resident_dispatches_per_token(&self) -> Option<u32>
pub fn resident_dispatches_per_token(&self) -> Option<u32>
Exact steady-state compute dispatch count for the prepared resident plan.
Sourcepub fn resident_readback_bytes_per_token(&self) -> Option<u32>
pub fn resident_readback_bytes_per_token(&self) -> Option<u32>
Exact candidate staging copy size for one resident greedy token.
Sourcepub fn dispatch_token_forward_resident(
&mut self,
index: &GgufTensorIndex,
emb: &[f32],
token_idx: u32,
) -> Option<StreamingArgmaxResult>
pub fn dispatch_token_forward_resident( &mut self, index: &GgufTensorIndex, emb: &[f32], token_idx: u32, ) -> Option<StreamingArgmaxResult>
Single-fence resident-token decode (greedy top-1). Returns the argmax token, or None
on any ineligibility (caller falls back to the legacy per-layer path).
Sampler-compatible resident forward: same single-fence layer stack + output
RMSNorm, then read back the normed hidden into out_hidden so the caller can
project full logits + sample on CPU without the legacy ~107-fence path.
emb is the token embedding input; out_hidden receives post-output-norm state
(may be the same logical buffer only if the caller copies input first — they must
not alias while the upload of emb is live; pass distinct slices).
Source§impl QTensorEngine
impl QTensorEngine
pub fn capture_native_providers(&self) -> Result<NativeProviderSnapshot, String>
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn prepare_qwen35_linear_layer(
&self,
layer: u32,
profile: Profile,
byte_budget: usize,
) -> Result<PreparedLinearLayer, Error>
pub fn prepare_qwen35_linear_layer( &self, layer: u32, profile: Profile, byte_budget: usize, ) -> Result<PreparedLinearLayer, Error>
Explicit CPU lab operation; does not grant native Webizen inference permissions, install fallback policy, or admit the full architecture. Budget covers newly owned buffers and this owner; payload is retained, never copied. Existing source-loader memory is caller-owned separately.
Sourcepub fn prepare_qwen35_linear_from_leases(
&self,
layer: u32,
profile: Profile,
byte_budget: usize,
primary: Arc<ModelLease>,
shards: &[Arc<ModelLease>],
) -> Result<PreparedLinearLayer, Error>
pub fn prepare_qwen35_linear_from_leases( &self, layer: u32, profile: Profile, byte_budget: usize, primary: Arc<ModelLease>, shards: &[Arc<ModelLease>], ) -> Result<PreparedLinearLayer, Error>
Explicit lab custody: independently selected full-file identities must already have been verified by ModelLease. FTW leases use manifest order. Monolithic GGUF/P64 leases must already be bound by native preparation; this method never reparses an unbounded replacement directory. Does not activate the model or allocate a full-model KV/recurrent arena.
Source§impl QTensorEngine
impl QTensorEngine
Sourcepub fn gemm_parity_probe(
&mut self,
info: &GgufTensorInfo,
raw: &[u8],
input: &[f32],
gpu_out: &mut [f32],
cpu_out: &mut [f32],
n_in: usize,
n_out: usize,
) -> bool
pub fn gemm_parity_probe( &mut self, info: &GgufTensorInfo, raw: &[u8], input: &[f32], gpu_out: &mut [f32], cpu_out: &mut [f32], n_in: usize, n_out: usize, ) -> bool
W3 kernel-parity probe (test/diagnostic): run the GPU GEMM (dispatch_gemm_raw_into) and the
CPU reference (stack_gemm_quant) on the SAME quantized weights + input, writing each into a
caller-provided buffer. Ensures the GEMM buffers exist first, so a fresh engine (no model
loaded) can be probed directly. Returns true only if both ran; the caller compares the two
outputs with crate::llm_kernel_parity. Enable crate::llm_gpu_profiler around the call
to witness that the GPU path actually executed rather than silently falling back to the CPU.
Trait Implementations§
Source§impl AttentionProvider for QTensorEngine
impl AttentionProvider for QTensorEngine
type Index = TensorIndex<GgufHyperparams>
type Layout = KvCacheLayout
fn kv_layout(&self) -> Option<Self::Layout>
fn hyperparams(index: &Self::Index) -> ModelHyperparams
fn tensor_data_start(index: &Self::Index) -> u64
fn model_bytes(&self) -> Option<&[u8]>
fn debug_decode(&self) -> bool
fn prefer_cuda(&self) -> bool
fn attention_preproject(&self) -> bool
fn attention_o_fuse(&self) -> bool
fn dispatch_gemm_into( &self, index: &Self::Index, info: &GgufTensorInfo, input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool
fn dispatch_ffn_block_pre_norm( &mut self, index: &Self::Index, hidden: &mut [f32], emb_dim: usize, tensors: &LayerTensors, scratch_a: &mut [f32], scratch_b: &mut [f32], ) -> bool
fn dispatch_attention_pass( &self, hidden: &[f32], n_embd: usize, num_tokens_in_batch: u32, batch_start_token_idx: u32, layout: &Self::Layout, layer: u32, token_idx: u32, h: &ModelHyperparams, info: &GgufTensorInfo, raw_weights: &[u8], proj_kind: u32, n_workgroups: u32, norm_weight: Option<&[f32]>, readback_out: Option<&mut [f32]>, ) -> bool
fn try_cuda_soa_attention_layer( &self, hidden_input: &[f32], n_embd: usize, q_dim: usize, kv_dim: usize, n_head: usize, n_kv: usize, head_dim: usize, layout: &Self::Layout, layer: u32, token_idx: u32, h: &ModelHyperparams, q_raw: &[u8], k_raw: &[u8], v_raw: &[u8], out_info: Option<&GgufTensorInfo>, mmap: &[u8], tensor_data_start: u64, scratch_a: &mut [f32], scratch_b: &mut [f32], ) -> bool
fn dispatch_attention_kv_preproject_fused( &self, hidden: &[f32], n_embd: usize, layout: &Self::Layout, layer: u32, token_idx: u32, h: &ModelHyperparams, k_info: &GgufTensorInfo, k_raw: &[u8], v_info: &GgufTensorInfo, v_raw: &[u8], kv_dim: usize, ) -> bool
fn dispatch_attention_q_o_fused( &self, hidden: &[f32], n_embd: usize, layout: &Self::Layout, layer: u32, token_idx: u32, h: &ModelHyperparams, q_info: &GgufTensorInfo, q_raw: &[u8], o_info: &GgufTensorInfo, o_raw: &[u8], out: &mut [f32], o_in: usize, o_out: usize, ) -> bool
Source§impl ForgeAttentionHost for QTensorEngine
Available on crate feature wgsl-forge and non-WebAssembly only.
impl ForgeAttentionHost for QTensorEngine
wgsl-forge and non-WebAssembly only.Source§impl GpuAttentionProvider for QTensorEngine
impl GpuAttentionProvider for QTensorEngine
type Layout = KvCacheLayout
type Mask = [u32; 32]
fn attention_resources(&self) -> AttentionResources<'_>
fn resident_weights_enabled(&self) -> bool
fn coop_gemv_enabled(&self) -> bool
fn gpu_device(&self) -> &Device
fn gpu_queue(&self) -> &Queue
fn native_gemm_bind_layout(&self, coop: bool) -> &BindGroupLayout
fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<Buffer>
fn submit(&self, command: CommandBuffer) -> bool
fn pass_writes_begin(&self) -> Option<ComputePassTimestampWrites<'static>>
fn pass_writes_end(&self) -> Option<ComputePassTimestampWrites<'static>>
fn resolve(&self, encoder: &mut CommandEncoder)
fn accumulate_attention(&self)
fn layer_stride(layout: &Self::Layout) -> u32
fn attention_kv_mask_for_dispatch( layout: &Self::Layout, token_idx: u32, proj_kind: u32, ) -> (Self::Mask, u32, u32)
fn attention_gpu_params( h: &ModelHyperparams, layout: &Self::Layout, layer: u32, token_idx: u32, info: &GgufTensorInfo, raw_len: usize, proj_kind: u32, num_tokens_in_batch: u32, batch_start_token_idx: u32, mask_active: u32, mask_word_count: u32, out_stride_elems: u32, ) -> AttentionGpuParams
fn readback(&self, staging: &Buffer, out_bytes: u64, out: &mut [f32]) -> bool
fn device(&self) -> &Device
Source§impl LayerProvider for QTensorEngine
impl LayerProvider for QTensorEngine
type Index = TensorIndex<GgufHyperparams>
fn hyperparams(index: &Self::Index) -> ModelHyperparams
fn moe_routing(index: &Self::Index) -> MoeRoutingSettings
fn data_start(index: &Self::Index) -> u64
fn tensors(index: &Self::Index, layer: u32) -> LayerTensors
fn model_bytes(&self) -> Option<&[u8]>
fn state(&self) -> &[f32]
fn state_mut(&mut self) -> &mut [f32]
fn gemm( &self, index: &Self::Index, info: &GgufTensorInfo, input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool
fn norm(&self, info: &GgufTensorInfo, out: &mut [f32]) -> usize
fn raw(&self, info: &GgufTensorInfo) -> Option<&[u8]>
fn event(&self, event: LayerEvent)
fn moe( &mut self, index: &Self::Index, emb_dim: usize, tensors: &LayerTensors, out: &mut [f32], input: &[f32], ) -> bool
fn record_ffn_input(&self, input: &[f32])
fn attention( &mut self, index: &Self::Index, layer: u32, token: u32, hidden: &[f32], emb_dim: usize, tensors: &LayerTensors, a: &mut [f32], b: &mut [f32], ) -> Option<usize>
fn ffn_dispatch( &mut self, index: &Self::Index, hidden: &mut [f32], emb_dim: usize, tensors: &LayerTensors, a: &mut [f32], b: &mut [f32], ) -> bool
fn profile_attention(&self, ns: u64)
fn profile_ffn(&self, ns: u64)
fn layer_diagnostic( &self, layer: u32, attn_ok: bool, tensors: &LayerTensors, attention: &[f32], hidden: &[f32], )
fn expert_source(&self) -> Option<&dyn ExpertSource>
Source§impl ProjectionGpuProvider for QTensorEngine
impl ProjectionGpuProvider for QTensorEngine
fn tensor_data_offset(&self) -> u64
fn backend_fused_block( &self, tensor: &ProjectionTensor<'_>, input: &[f32], ) -> Option<Vec<f32>>
fn record_operations(&self, count: usize)
fn readback_owned( &self, buffer: &Buffer, bytes: u64, panic_on_error: bool, ) -> Option<Vec<f32>>
fn projection_resources(&self) -> ProjectionResources<'_>
fn gpu_device(&self) -> &Device
fn gpu_queue(&self) -> &Queue
fn coop_gemv_enabled(&self) -> bool
fn coop_gemv_workgroups(&self, rows: u32) -> u32
fn resident_weights_enabled(&self) -> bool
fn native_gemm_bind_layout(&self, coop: bool) -> &BindGroupLayout
fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<Buffer>
fn write_weight_words(&self, raw: &[u8], max_bytes: usize)
fn submit(&self, commands: CommandBuffer)
fn map_read(&self, buffer: &Buffer, bytes: u64) -> bool
fn readback(&self, buffer: &Buffer, bytes: u64, out: &mut [f32]) -> bool
fn pass_writes( &self, profile: Profile, ) -> Option<ComputePassTimestampWrites<'static>>
fn resolve(&self, encoder: &mut CommandEncoder)
fn accumulate(&self, phase: Phase)
fn cpu_gemm( &self, raw: &[u8], info: &GgufTensorInfo, input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool
fn cuda_gemm( &self, info: &GgufTensorInfo, raw: &[u8], input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool
fn device(&self) -> &Device
Source§impl ProjectionProvider for QTensorEngine
impl ProjectionProvider for QTensorEngine
type Index = TensorIndex<GgufHyperparams>
type Mask = SieveStateMask
fn logits(index: &Self::Index) -> Option<GgufTensorInfo>
fn output_norm(index: &Self::Index) -> Option<GgufTensorInfo>
fn data_start(index: &Self::Index) -> u64
fn embedding_dim(index: &Self::Index) -> usize
fn model_bytes(&self) -> Option<&[u8]>
fn nvfp4_output(&self) -> Option<(&[u8], &[u8], &[u8])>
fn fetch_tensor_raw_bytes(&self, info: &GgufTensorInfo) -> Option<&[u8]>
fn gemm( &self, info: &GgufTensorInfo, raw: &[u8], input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool
fn allows(mask: &Self::Mask, token: u32) -> bool
fn scrub(&self, values: &mut [f32], n: usize)
fn update_argmax( &self, logits: &[f32], rows: usize, chunk: usize, mask: Option<&Self::Mask>, best: &mut u32, max: &mut f32, )
fn update_argmax_unmasked( &self, logits: &[f32], rows: usize, chunk: usize, best: &mut u32, max: &mut f32, )
Source§impl ResidentPlanProvider for QTensorEngine
impl ResidentPlanProvider for QTensorEngine
type Index = TensorIndex<GgufHyperparams>
type ModelLease = PlanningModelLease
type Layout = KvCacheLayout
fn model_lease(&self) -> Option<Self::ModelLease>
fn hyperparams(index: &Self::Index) -> ModelHyperparams
fn data_start(index: &Self::Index) -> u64
fn tensors(index: &Self::Index, layer: u32) -> LayerTensors
fn logits_info(index: &Self::Index) -> Option<GgufTensorInfo>
fn output_norm_info(index: &Self::Index) -> Option<GgufTensorInfo>
fn tensor_bytes<'a>( bytes: &'a [u8], start: u64, info: &GgufTensorInfo, ) -> Option<&'a [u8]>
fn norm_row( bytes: &[u8], start: u64, info: &GgufTensorInfo, out: &mut [f32], ) -> usize
fn plan_layout(&self) -> Option<Self::Layout>
fn max_context(layout: &Self::Layout) -> u32
fn layer_stride(layout: &Self::Layout) -> u32
fn plan_options(&self) -> ResidentPlanOptions
fn plan_resources(&self) -> ResidentPlanResources<'_>
fn gpu_device(&self) -> &Device
fn gpu_queue(&self) -> &Queue
fn native_gemm_bind_layout(&self, coop: bool) -> &BindGroupLayout
fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<Buffer>
fn promote_matrix_to_f16_resident( &self, info: &GgufTensorInfo, raw: &[u8], ) -> Option<(Buffer, u32, u32, u32)>
fn prewarm_layer( &self, layer: u32, matrices: &[(&GgufTensorInfo, &[u8], usize, usize)], )
fn set_ffn_fusion_in_resident(&self, fused: bool)
fn event(&self, event: PlanEvent)
fn attention_gpu_params( h: &ModelHyperparams, layout: &Self::Layout, layer: u32, token_idx: u32, info: &GgufTensorInfo, raw_len: usize, proj_kind: u32, num_tokens_in_batch: u32, batch_start_token_idx: u32, mask_active: u32, mask_word_count: u32, out_stride_elems: u32, ) -> AttentionGpuParams
Source§impl ResidentProvider for QTensorEngine
impl ResidentProvider for QTensorEngine
type Mask = [u32; 32]
fn resident_resources(&self) -> ResidentResources<'_>
fn gpu_device(&self) -> &Device
fn gpu_queue(&self) -> &Queue
fn submit(&self, command: CommandBuffer) -> bool
fn poll_wait(&self) -> bool
fn attention_kv_mask(&self, token: u32, capacity: u32) -> (Self::Mask, u32)
fn dual_multirow(&self) -> bool
fn coop_gemv_workgroups(&self, rows: u32) -> u32
fn pass_writes(&self) -> Option<ComputePassTimestampWrites<'static>>
fn resolve(&self, encoder: &mut CommandEncoder)
fn accumulate_fused(&self)
Auto Trait Implementations§
impl !Freeze for QTensorEngine
impl !RefUnwindSafe for QTensorEngine
impl !UnwindSafe for QTensorEngine
impl Send for QTensorEngine
impl Sync for QTensorEngine
impl Unpin for QTensorEngine
impl UnsafeUnpin for QTensorEngine
Blanket Implementations§
§impl<S, A> Aggregate<Result<S, Error>> for Awhere
A: Aggregate<S>,
impl<S, A> Aggregate<Result<S, Error>> for Awhere
A: Aggregate<S>,
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
§impl<T> Instrument for T
impl<T> Instrument for T
§fn instrument(self, span: Span) -> Instrumented<Self> ⓘ
fn instrument(self, span: Span) -> Instrumented<Self> ⓘ
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more