Skip to main content

QTensorEngine

Struct QTensorEngine 

Source
pub struct QTensorEngine {
    pub pipeline: ComputePipeline,
    pub embedding_pipeline: ComputePipeline,
    pub is_initialized: bool,
    pub gguf_mmap: Option<Arc<Mmap>>,
    pub p64_index: Option<P64TensorIndex>,
    pub tensor_index_cache: Option<GgufTensorIndex>,
    pub ftw_package: Option<Arc<FtwModelPackage>>,
    pub tensor_data_offset: u64,
    pub hyperparams: GgufHyperparams,
    pub max_tensor_bytes: usize,
    /* private fields */
}

Fields§

§pipeline: ComputePipeline§embedding_pipeline: ComputePipeline

GPU-side Q6_K embedding dequant + matmul (zero CPU dequant).

§is_initialized: bool§gguf_mmap: Option<Arc<Mmap>>

Memory-mapped GGUF file (set after load_gguf).

§p64_index: Option<P64TensorIndex>

Cached P64 index after adopt_resident_p64_* — decode must not re-CRC the container.

§tensor_index_cache: Option<GgufTensorIndex>

Cached synthetic GGUF index built from p64_index (or from GGUF parse).

§ftw_package: Option<Arc<FtwModelPackage>>

FTW multi-shard package for native MoE execution.

§tensor_data_offset: u64

Byte offset into the mmap where tensor data begins.

§hyperparams: GgufHyperparams§max_tensor_bytes: usize

Implementations§

Source§

impl QTensorEngine

Source

pub fn dispatch_output_logits_into( &self, index: &GgufTensorIndex, hidden: &[f32], emb_dim: usize, logits_out: &mut [f32], ) -> usize

Final logits via chunked projection into logits_out (fills min(vocab, buf) rows).

Source

pub fn decode_lexicon_bound( &self, _logits: &[f32], valid_lexicon_ids: &[u64], ) -> u64

Decodes lexicon_bound; fails closed on malformed input.

Source§

impl QTensorEngine

Source

pub fn dispatch_forge_causal_attention( &mut self, query: &[f32], keys: &[f32], values: &[f32], scale: f32, shape: AttentionShape, out: &mut [f32], ) -> Result<(), ForgeError>

Certified Naga-validated WGSL Forge causal attention device dispatch during decode. Native + wgsl-forge only — wasm profiles do not compile the Forge module.

Source§

impl QTensorEngine

Source

pub fn try_native_projection_block( &self, tensor: &ProjectionTensor<'_>, input: &[f32], ) -> Result<Option<Vec<f32>>, &'static str>

Fallible native projection route. Governed callers may configure the admitted budget with set_projection_cold_byte_budget; ordinary preparation inherits the native host/device policy.

Source§

impl QTensorEngine

Source

pub fn dispatch_quantized_token_embedding( &self, raw_embd: &[u8], ggml_type: u32, n_embd: u32, weight_tensor: &QTensor, ) -> Option<Vec<f32>>

Upload raw quantized embedding bytes to the GPU and matmul without CPU dequant. Returns None when the GGML type has no WGSL kernel (caller uses CPU fallback).

Source

pub fn dispatch_fused_transformer_block( &self, tensor: &QTensor, input_activations: &[f32], ) -> Vec<f32>

Async dispatch: fused-transformer-block.

Source§

impl QTensorEngine

Source

pub fn dispatch_prefill_chunk( &mut self, index: &GgufTensorIndex, batch_hidden: &mut [f32], emb_dim: usize, n_tokens: u32, batch_start_token_idx: u32, scratch_a: &mut [f32], scratch_b: &mut [f32], max_layers: u32, ) -> Result<(), PrefillDispatchFailure>

Chunked prefill: populate KV arena for n_tokens prompt positions starting at batch_start.

Source

pub fn dispatch_transformer_layer( &mut self, index: &GgufTensorIndex, layer: u32, token_idx: u32, hidden: &mut [f32], emb_dim: usize, scratch_a: &mut [f32], scratch_b: &mut [f32], ) -> bool

One transformer block using real mmap tensor offsets (stack buffers only).

Source

pub fn dispatch_transformer_forward( &mut self, index: &GgufTensorIndex, hidden: &mut [f32], emb_dim: usize, scratch_a: &mut [f32], scratch_b: &mut [f32], token_idx: u32, max_layers: u32, ) -> u32

Sequential layer-by-layer forward (one tensor payload in VRAM at a time). max_layers: 0 runs all blocks; otherwise caps how many layers execute.

Source

pub fn verify_topology_draft_batch( &mut self, index: &GgufTensorIndex, ctx: &mut Vec<u32>, draft: &TopologyDraftBatch, emb_dim: usize, emb_buf: &mut [f32], scratch_a: &mut [f32], scratch_b: &mut [f32], max_layers: u32, max_vocab_chunks: u32, ) -> u32

Topological speculative verify — accept longest draft prefix (B3.1d).

Source§

impl QTensorEngine

Source

pub fn dispatch_gemma4_token( &mut self, index: &GgufTensorIndex, token: u32, position: u32, hidden: &mut [f32], scratch_a: &mut [f32], scratch_b: &mut [f32], ) -> u32

Text-only Gemma4 token step. The owner reads the actual retained embedding and PLE token row, so hidden-only callers cannot silently substitute token0. Returns all layers or zero; an interrupted/failed token requires reset.

Source§

impl QTensorEngine

Source

pub fn adopt_ftw_package( &mut self, dir: &Path, ) -> Result<GgufLoadReport, String>

Adopt an FTW multi-shard model directory for native MoE execution.

Source

pub fn adopt_ftw_package_with_budget( &mut self, dir: &Path, max_context: usize, model_state_byte_budget: usize, ) -> Result<GgufLoadReport, String>

Select the external whole-model state budget and context capacity for admitted Qwen execution. This is separate from the legacy SSM-only budget and the 42 MiB Sentinel arena; the default reserves at most 128 MiB.

Source

pub fn fetch_tensor_raw_bytes<'a>( &'a self, info: &GgufTensorInfo, ) -> Option<&'a [u8]>

Read raw bytes for a tensor from either the FTW package shards or the primary GGUF mmap.

Source

pub fn dispatch_moe_step_routed<F>( &mut self, emb_dim: usize, router_weights: &[f32], num_experts: usize, topk: usize, expert_fetcher: F, shared_expert: Option<&ExpertWeightView<'_>>, shared_gate_weight: f32, ffn_input: &[f32], scratch_a: &mut [f32], ) -> bool
where F: FnMut(u16) -> Option<ExpertWeightView<'static>>,

Dispatches MoE layer using the zero-heap dispatch_moe_step driver with expert fetcher (EOS-070).

Source

pub fn dispatch_clustered_moe_operator( &mut self, op: &ClusteredMoEOperator, routed_experts: &[RoutedExpert], ffn_input: &[f32], output: &mut [f32], scratch: &mut ClusteredMoeScratch<'_>, ) -> bool

Dispatches MoE layer using the clustered shared-down MoE operator (EOS-071).

Source§

impl QTensorEngine

Source

pub fn set_ssm_state_byte_budget(&mut self, byte_budget: usize)

Select the external recurrent-state budget for subsequent cold model loads. This arena is separate from the 42 MiB Sentinel arena.

Source

pub fn init_ssm_state_with_budget( &mut self, index: &GgufTensorIndex, byte_budget: usize, ) -> Result<usize, SsmStateError>

Source

pub fn init_ssm_state(&mut self, index: &GgufTensorIndex) -> usize

Compatibility entry point. Loaders use fallible admission; a refused arena remains empty and hybrid dispatch fails closed.

Source

pub fn reset_recurrent_state(&mut self)

Source

pub fn snapshot_recurrent_state_into(&self, out: &mut [f32]) -> Option<usize>

Export a zero-heap snapshot of the current hybrid SSM / GatedDeltaNet recurrent state (ssm_recurrent_state) into caller-supplied out (f32 slice). Returns Some(self.ssm_recurrent_state.len()) when out has sufficient capacity.

Source

pub fn restore_recurrent_state_from_slice(&mut self, src: &[f32]) -> bool

Restore a hybrid SSM / GatedDeltaNet recurrent state snapshot from a .p64 neural extension cartridge or pre-warmed scene seed into self.ssm_recurrent_state (zero-heap).

Source§

impl QTensorEngine

Source

pub fn set_projection_cold_byte_budget(&mut self, bytes: usize)

Set the admitted cold native-projection/readback payload budget. This legacy Vec-returning entrypoint reports native admission exhaustion by panic instead of choosing another layout.

Source

pub fn set_resident_projection_byte_budget(&mut self, bytes: u64)

Set aggregate projection-weight allowance. Admission misses retain uncached GPU/CPU fallback.

Source

pub fn dispatch_gemm_into( &self, index: &GgufTensorIndex, info: &GgufTensorInfo, input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool

Quantized GEMM into caller out using reused GPU buffers (Q6_K) or CPU dequant fallback.

Source§

impl QTensorEngine

Source

pub fn device(&self) -> &Device

Shared process-wide wgpu device (LLM + render coexistence).

Source

pub fn queue(&self) -> &Queue

Shared process-wide wgpu queue.

Source

pub async fn try_new() -> Result<Self, String>

Tries to build the engine; error when GPU init fails.

Source

pub fn new() -> Self

Builds the engine; panics when GPU init fails.

Source

pub fn reset_kv_cache(&mut self)

Zero the static KV arena at the start of a new decode context (zero heap in decode).

Source

pub fn get_kv_cache_cpu(&self) -> Option<&[f32]>

Returns kv_cache_cpu.

Source

pub fn set_kv_cache_cpu(&mut self, data: &[f32]) -> bool

Sets kv_cache_cpu.

Source

pub fn read_kv_cache_gpu(&self) -> Option<Vec<f32>>

Read the entire GPU KV-cache arena back to host as f32 (native, cold path — forge KV capture, not the decode hot path). The returned flat buffer is interpretable via KvCacheLayout:: k_index/v_index only when the layout is f32 (int8 KV disabled at load); with an int8 layout the bytes are packed i8 lanes + scales and this returns None.

Source

pub fn capture_kv_f32( &self, n_tokens: u32, max_per_layer: usize, ) -> Option<KvCapture>

Decode the current f32 KV cache into per-layer K and V vectors for token positions 0..n_tokens (capped at max_per_layer per layer per stream). The GPU-readback capture route for the sparse-KV-dictionary go/no-go — reads the real decode-path K/V straight from VRAM, no CPU reference forward. Returns None on an int8 layout or if readback fails.

Source§

impl QTensorEngine

Source

pub fn kv_layout(&self) -> Option<KvCacheLayout>

KV-cache layout when the engine allocates one.

Source

pub fn kv_cache_bytes(&self) -> u64

KV-cache size in bytes.

Source

pub fn load_gguf_checked( &mut self, path: &str, ) -> Result<GgufLoadReport, String>

Loads gguf-checked.

Source

pub fn load_gguf(&mut self, path: &str)

Memory-map a GGUF file so tensor bytes are accessible without heap allocation. Call this once after new(), before the first dispatch_fused_transformer_block.

Source

pub fn load_model_checked( &mut self, path: &str, ) -> Result<GgufLoadReport, String>

Memory-map and auto-detect a supported local model container.

Canonical P64 is detected by its exact p64\0 magic and adopted through the P64 validation path. All other inputs are passed to the GGUF parser, which rejects malformed or unsupported data.

Source

pub fn adopt_resident_safetensor_mmap( &mut self, mmap: Arc<Mmap>, config_json: Option<&str>, ) -> Result<GgufLoadReport, String>

Boot directly from an already-mapped Safetensors model (including BF16).

Source

pub fn load_model(&mut self, path: &str)

Fail-soft wrapper retained for the agent decode path.

Source

pub fn adopt_resident_p64_mmap( &mut self, mmap: Arc<Mmap>, ) -> Result<GgufLoadReport, String>

Boot from an already-mapped P64 weight container (native). Mirrors the GGUF adopt_resident_mmap but for the P64 format: validates + builds a synthetic GGUF index from the manifest, points the byte source at the P64 bytes (tensor_data_start = 0, absolute blob offsets), reserves the GEMM/KV arenas, makes the (verbatim) output projection resident, and builds the resident 2-bit ternary-FFN dispatcher from the FFN blobs. The attention/norm/embed tensors stay at source precision and run the standard GGUF hot path.

Source

pub fn adopt_resident_q42_mmap( &mut self, mmap: Arc<Mmap>, ) -> Result<GgufLoadReport, String>

👎Deprecated:

use adopt_resident_p64_mmap

Compatibility alias for the historical pre-P64 API name.

Source

pub fn ternary_ffn_resident_len(&self) -> usize

A1b: number of resident ternary FFN tensors (0 unless a ternary P64 was adopted). Lets a test confirm the GPU resident path is actually populated (not a silent CPU-only fallback).

Source

pub fn adopt_resident_mmap( &mut self, mmap: Arc<Mmap>, ) -> Result<GgufLoadReport, String>

Attach an already-mapped resident GGUF (shared with orchestrator slot).

Source

pub fn bench_empty_submit_roundtrip(&self, n: u32) -> u64

Decode-profiler: wall-clock for n EMPTY submit → poll(Maintain::Wait) round-trips (no compute dispatched). Isolates the fixed CPU↔GPU fence latency: if a token’s forward time ≈ (its round-trip count × this per-round-trip cost), the bottleneck is synchronization, not math; if forward ≫ that, the kernels themselves are slow. Does NOT touch GPU_WAIT_COUNT.

Source§

impl QTensorEngine

Source

pub fn dispatch_output_argmax_chunked( &self, index: &GgufTensorIndex, hidden: &[f32], emb_dim: usize, chunk_logits: &mut [f32], max_chunks: u32, sieve_mask: Option<&SieveStateMask>, ) -> Option<StreamingArgmaxResult>

Chunked vocabulary projection with streaming argmax (zero heap, stack chunk buffer only). max_chunks: 0 sweeps the full vocabulary; otherwise caps chunk iterations (tests).

Source

pub fn dispatch_output_top1_chunked( &self, index: &GgufTensorIndex, hidden: &[f32], emb_dim: usize, ) -> Option<StreamingArgmaxResult>

Native decode fast path: output projection plus GPU block argmax (k=1) with one tiny candidate readback after all vocab chunks have been submitted. This avoids the full chunk-logit readback in Self::dispatch_output_argmax_chunked and avoids heap allocation in the decode loop.

Source

pub fn dispatch_output_topk_chunked( &self, index: &GgufTensorIndex, hidden: &[f32], emb_dim: usize, k: usize, ) -> Option<Vec<TopKItem>>

A1a: GPU top-k over the output projection — the logits stay on-GPU (gemm_output_buf), the top-k kernel reduces them per chunk, and only K (id, logit) candidates are read back (vs the 196 KB/token full-logit readback + CPU argmax in dispatch_output_argmax_chunked). Returns the merged global top-K, or None to signal the caller to fall back to the argmax path. v1: no sieve coupling (caller routes here only when no sieve mask is active).

Source

pub fn apply_output_norm_inplace( &self, index: &GgufTensorIndex, hidden: &mut [f32], emb_dim: usize, ) -> bool

Final output_norm RMSNorm in-place before vocabulary projection (Pre-Norm LLM tail). REQUIRED on all targets — native previously skipped it → logits from an un-normed hidden.

Source§

impl QTensorEngine

Source

pub fn resident_dispatches_per_token(&self) -> Option<u32>

Exact steady-state compute dispatch count for the prepared resident plan.

Source

pub fn resident_readback_bytes_per_token(&self) -> Option<u32>

Exact candidate staging copy size for one resident greedy token.

Source

pub fn dispatch_token_forward_resident( &mut self, index: &GgufTensorIndex, emb: &[f32], token_idx: u32, ) -> Option<StreamingArgmaxResult>

Single-fence resident-token decode (greedy top-1). Returns the argmax token, or None on any ineligibility (caller falls back to the legacy per-layer path).

Source

pub fn dispatch_token_forward_resident_hidden( &mut self, index: &GgufTensorIndex, emb: &[f32], token_idx: u32, out_hidden: &mut [f32], ) -> bool

Sampler-compatible resident forward: same single-fence layer stack + output RMSNorm, then read back the normed hidden into out_hidden so the caller can project full logits + sample on CPU without the legacy ~107-fence path.

emb is the token embedding input; out_hidden receives post-output-norm state (may be the same logical buffer only if the caller copies input first — they must not alias while the upload of emb is live; pass distinct slices).

Source§

impl QTensorEngine

Source§

impl QTensorEngine

Source

pub fn prepare_qwen35_linear_layer( &self, layer: u32, profile: Profile, byte_budget: usize, ) -> Result<PreparedLinearLayer, Error>

Explicit CPU lab operation; does not grant native Webizen inference permissions, install fallback policy, or admit the full architecture. Budget covers newly owned buffers and this owner; payload is retained, never copied. Existing source-loader memory is caller-owned separately.

Source

pub fn prepare_qwen35_linear_from_leases( &self, layer: u32, profile: Profile, byte_budget: usize, primary: Arc<ModelLease>, shards: &[Arc<ModelLease>], ) -> Result<PreparedLinearLayer, Error>

Explicit lab custody: independently selected full-file identities must already have been verified by ModelLease. FTW leases use manifest order. Monolithic GGUF/P64 leases must already be bound by native preparation; this method never reparses an unbounded replacement directory. Does not activate the model or allocate a full-model KV/recurrent arena.

Source§

impl QTensorEngine

Source

pub fn gemm_parity_probe( &mut self, info: &GgufTensorInfo, raw: &[u8], input: &[f32], gpu_out: &mut [f32], cpu_out: &mut [f32], n_in: usize, n_out: usize, ) -> bool

W3 kernel-parity probe (test/diagnostic): run the GPU GEMM (dispatch_gemm_raw_into) and the CPU reference (stack_gemm_quant) on the SAME quantized weights + input, writing each into a caller-provided buffer. Ensures the GEMM buffers exist first, so a fresh engine (no model loaded) can be probed directly. Returns true only if both ran; the caller compares the two outputs with crate::llm_kernel_parity. Enable crate::llm_gpu_profiler around the call to witness that the GPU path actually executed rather than silently falling back to the CPU.

Trait Implementations§

Source§

impl AttentionProvider for QTensorEngine

Source§

type Index = TensorIndex<GgufHyperparams>

Source§

type Layout = KvCacheLayout

Source§

fn kv_layout(&self) -> Option<Self::Layout>

Source§

fn hyperparams(index: &Self::Index) -> ModelHyperparams

Source§

fn tensor_data_start(index: &Self::Index) -> u64

Source§

fn model_bytes(&self) -> Option<&[u8]>

Source§

fn debug_decode(&self) -> bool

Source§

fn prefer_cuda(&self) -> bool

Source§

fn attention_preproject(&self) -> bool

Source§

fn attention_o_fuse(&self) -> bool

Source§

fn dispatch_gemm_into( &self, index: &Self::Index, info: &GgufTensorInfo, input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool

Source§

fn dispatch_ffn_block_pre_norm( &mut self, index: &Self::Index, hidden: &mut [f32], emb_dim: usize, tensors: &LayerTensors, scratch_a: &mut [f32], scratch_b: &mut [f32], ) -> bool

Source§

fn dispatch_attention_pass( &self, hidden: &[f32], n_embd: usize, num_tokens_in_batch: u32, batch_start_token_idx: u32, layout: &Self::Layout, layer: u32, token_idx: u32, h: &ModelHyperparams, info: &GgufTensorInfo, raw_weights: &[u8], proj_kind: u32, n_workgroups: u32, norm_weight: Option<&[f32]>, readback_out: Option<&mut [f32]>, ) -> bool

Source§

fn try_cuda_soa_attention_layer( &self, hidden_input: &[f32], n_embd: usize, q_dim: usize, kv_dim: usize, n_head: usize, n_kv: usize, head_dim: usize, layout: &Self::Layout, layer: u32, token_idx: u32, h: &ModelHyperparams, q_raw: &[u8], k_raw: &[u8], v_raw: &[u8], out_info: Option<&GgufTensorInfo>, mmap: &[u8], tensor_data_start: u64, scratch_a: &mut [f32], scratch_b: &mut [f32], ) -> bool

Source§

fn dispatch_attention_kv_preproject_fused( &self, hidden: &[f32], n_embd: usize, layout: &Self::Layout, layer: u32, token_idx: u32, h: &ModelHyperparams, k_info: &GgufTensorInfo, k_raw: &[u8], v_info: &GgufTensorInfo, v_raw: &[u8], kv_dim: usize, ) -> bool

Source§

fn dispatch_attention_q_o_fused( &self, hidden: &[f32], n_embd: usize, layout: &Self::Layout, layer: u32, token_idx: u32, h: &ModelHyperparams, q_info: &GgufTensorInfo, q_raw: &[u8], o_info: &GgufTensorInfo, o_raw: &[u8], out: &mut [f32], o_in: usize, o_out: usize, ) -> bool

Source§

impl ForgeAttentionHost for QTensorEngine

Available on crate feature wgsl-forge and non-WebAssembly only.
Source§

fn device(&self) -> &Device

Source§

fn submit(&self, command: CommandBuffer)

Source§

fn poll_wait(&self)

Source§

fn wait_for_map(&self, receiver: Receiver<Result<(), BufferAsyncError>>) -> bool

Source§

impl GpuAttentionProvider for QTensorEngine

Source§

type Layout = KvCacheLayout

Source§

type Mask = [u32; 32]

Source§

fn attention_resources(&self) -> AttentionResources<'_>

Source§

fn resident_weights_enabled(&self) -> bool

Source§

fn coop_gemv_enabled(&self) -> bool

Source§

fn gpu_device(&self) -> &Device

Source§

fn gpu_queue(&self) -> &Queue

Source§

fn native_gemm_bind_layout(&self, coop: bool) -> &BindGroupLayout

Source§

fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<Buffer>

Source§

fn submit(&self, command: CommandBuffer) -> bool

Source§

fn pass_writes_begin(&self) -> Option<ComputePassTimestampWrites<'static>>

Source§

fn pass_writes_end(&self) -> Option<ComputePassTimestampWrites<'static>>

Source§

fn resolve(&self, encoder: &mut CommandEncoder)

Source§

fn accumulate_attention(&self)

Source§

fn layer_stride(layout: &Self::Layout) -> u32

Source§

fn attention_kv_mask_for_dispatch( layout: &Self::Layout, token_idx: u32, proj_kind: u32, ) -> (Self::Mask, u32, u32)

Source§

fn attention_gpu_params( h: &ModelHyperparams, layout: &Self::Layout, layer: u32, token_idx: u32, info: &GgufTensorInfo, raw_len: usize, proj_kind: u32, num_tokens_in_batch: u32, batch_start_token_idx: u32, mask_active: u32, mask_word_count: u32, out_stride_elems: u32, ) -> AttentionGpuParams

Source§

fn readback(&self, staging: &Buffer, out_bytes: u64, out: &mut [f32]) -> bool

§

fn device(&self) -> &Device

Source§

impl LayerProvider for QTensorEngine

Source§

type Index = TensorIndex<GgufHyperparams>

Source§

fn hyperparams(index: &Self::Index) -> ModelHyperparams

Source§

fn moe_routing(index: &Self::Index) -> MoeRoutingSettings

Source§

fn data_start(index: &Self::Index) -> u64

Source§

fn tensors(index: &Self::Index, layer: u32) -> LayerTensors

Source§

fn model_bytes(&self) -> Option<&[u8]>

Source§

fn state(&self) -> &[f32]

Source§

fn state_mut(&mut self) -> &mut [f32]

Source§

fn gemm( &self, index: &Self::Index, info: &GgufTensorInfo, input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool

Source§

fn norm(&self, info: &GgufTensorInfo, out: &mut [f32]) -> usize

Source§

fn raw(&self, info: &GgufTensorInfo) -> Option<&[u8]>

Source§

fn event(&self, event: LayerEvent)

Source§

fn moe( &mut self, index: &Self::Index, emb_dim: usize, tensors: &LayerTensors, out: &mut [f32], input: &[f32], ) -> bool

Source§

fn record_ffn_input(&self, input: &[f32])

Source§

fn attention( &mut self, index: &Self::Index, layer: u32, token: u32, hidden: &[f32], emb_dim: usize, tensors: &LayerTensors, a: &mut [f32], b: &mut [f32], ) -> Option<usize>

Source§

fn ffn_dispatch( &mut self, index: &Self::Index, hidden: &mut [f32], emb_dim: usize, tensors: &LayerTensors, a: &mut [f32], b: &mut [f32], ) -> bool

Source§

fn profile_attention(&self, ns: u64)

Source§

fn profile_ffn(&self, ns: u64)

Source§

fn layer_diagnostic( &self, layer: u32, attn_ok: bool, tensors: &LayerTensors, attention: &[f32], hidden: &[f32], )

Source§

fn expert_source(&self) -> Option<&dyn ExpertSource>

Source§

impl ProjectionGpuProvider for QTensorEngine

Source§

fn tensor_data_offset(&self) -> u64

Source§

fn backend_fused_block( &self, tensor: &ProjectionTensor<'_>, input: &[f32], ) -> Option<Vec<f32>>

Source§

fn record_operations(&self, count: usize)

Source§

fn readback_owned( &self, buffer: &Buffer, bytes: u64, panic_on_error: bool, ) -> Option<Vec<f32>>

Source§

fn projection_resources(&self) -> ProjectionResources<'_>

Source§

fn gpu_device(&self) -> &Device

Source§

fn gpu_queue(&self) -> &Queue

Source§

fn coop_gemv_enabled(&self) -> bool

Source§

fn coop_gemv_workgroups(&self, rows: u32) -> u32

Source§

fn resident_weights_enabled(&self) -> bool

Source§

fn native_gemm_bind_layout(&self, coop: bool) -> &BindGroupLayout

Source§

fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<Buffer>

Source§

fn write_weight_words(&self, raw: &[u8], max_bytes: usize)

Source§

fn submit(&self, commands: CommandBuffer)

Source§

fn map_read(&self, buffer: &Buffer, bytes: u64) -> bool

Source§

fn readback(&self, buffer: &Buffer, bytes: u64, out: &mut [f32]) -> bool

Source§

fn pass_writes( &self, profile: Profile, ) -> Option<ComputePassTimestampWrites<'static>>

Source§

fn resolve(&self, encoder: &mut CommandEncoder)

Source§

fn accumulate(&self, phase: Phase)

Source§

fn cpu_gemm( &self, raw: &[u8], info: &GgufTensorInfo, input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool

Source§

fn cuda_gemm( &self, info: &GgufTensorInfo, raw: &[u8], input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool

§

fn device(&self) -> &Device

Source§

impl ProjectionProvider for QTensorEngine

Source§

type Index = TensorIndex<GgufHyperparams>

Source§

type Mask = SieveStateMask

Source§

fn logits(index: &Self::Index) -> Option<GgufTensorInfo>

Source§

fn output_norm(index: &Self::Index) -> Option<GgufTensorInfo>

Source§

fn data_start(index: &Self::Index) -> u64

Source§

fn embedding_dim(index: &Self::Index) -> usize

Source§

fn model_bytes(&self) -> Option<&[u8]>

Source§

fn nvfp4_output(&self) -> Option<(&[u8], &[u8], &[u8])>

Source§

fn fetch_tensor_raw_bytes(&self, info: &GgufTensorInfo) -> Option<&[u8]>

Source§

fn gemm( &self, info: &GgufTensorInfo, raw: &[u8], input: &[f32], out: &mut [f32], n_in: usize, n_out: usize, ) -> bool

Source§

fn allows(mask: &Self::Mask, token: u32) -> bool

Source§

fn scrub(&self, values: &mut [f32], n: usize)

§

fn update_argmax( &self, logits: &[f32], rows: usize, chunk: usize, mask: Option<&Self::Mask>, best: &mut u32, max: &mut f32, )

§

fn update_argmax_unmasked( &self, logits: &[f32], rows: usize, chunk: usize, best: &mut u32, max: &mut f32, )

Source§

impl ResidentPlanProvider for QTensorEngine

Source§

type Index = TensorIndex<GgufHyperparams>

Source§

type ModelLease = PlanningModelLease

Source§

type Layout = KvCacheLayout

Source§

fn model_lease(&self) -> Option<Self::ModelLease>

Source§

fn hyperparams(index: &Self::Index) -> ModelHyperparams

Source§

fn data_start(index: &Self::Index) -> u64

Source§

fn tensors(index: &Self::Index, layer: u32) -> LayerTensors

Source§

fn logits_info(index: &Self::Index) -> Option<GgufTensorInfo>

Source§

fn output_norm_info(index: &Self::Index) -> Option<GgufTensorInfo>

Source§

fn tensor_bytes<'a>( bytes: &'a [u8], start: u64, info: &GgufTensorInfo, ) -> Option<&'a [u8]>

Source§

fn norm_row( bytes: &[u8], start: u64, info: &GgufTensorInfo, out: &mut [f32], ) -> usize

Source§

fn plan_layout(&self) -> Option<Self::Layout>

Source§

fn max_context(layout: &Self::Layout) -> u32

Source§

fn layer_stride(layout: &Self::Layout) -> u32

Source§

fn plan_options(&self) -> ResidentPlanOptions

Source§

fn plan_resources(&self) -> ResidentPlanResources<'_>

Source§

fn gpu_device(&self) -> &Device

Source§

fn gpu_queue(&self) -> &Queue

Source§

fn native_gemm_bind_layout(&self, coop: bool) -> &BindGroupLayout

Source§

fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<Buffer>

Source§

fn promote_matrix_to_f16_resident( &self, info: &GgufTensorInfo, raw: &[u8], ) -> Option<(Buffer, u32, u32, u32)>

Source§

fn prewarm_layer( &self, layer: u32, matrices: &[(&GgufTensorInfo, &[u8], usize, usize)], )

Source§

fn set_ffn_fusion_in_resident(&self, fused: bool)

Source§

fn event(&self, event: PlanEvent)

Source§

fn attention_gpu_params( h: &ModelHyperparams, layout: &Self::Layout, layer: u32, token_idx: u32, info: &GgufTensorInfo, raw_len: usize, proj_kind: u32, num_tokens_in_batch: u32, batch_start_token_idx: u32, mask_active: u32, mask_word_count: u32, out_stride_elems: u32, ) -> AttentionGpuParams

Source§

impl ResidentProvider for QTensorEngine

Source§

type Mask = [u32; 32]

Source§

fn resident_resources(&self) -> ResidentResources<'_>

Source§

fn gpu_device(&self) -> &Device

Source§

fn gpu_queue(&self) -> &Queue

Source§

fn submit(&self, command: CommandBuffer) -> bool

Source§

fn poll_wait(&self) -> bool

Source§

fn attention_kv_mask(&self, token: u32, capacity: u32) -> (Self::Mask, u32)

Source§

fn dual_multirow(&self) -> bool

Source§

fn coop_gemv_workgroups(&self, rows: u32) -> u32

Source§

fn pass_writes(&self) -> Option<ComputePassTimestampWrites<'static>>

Source§

fn resolve(&self, encoder: &mut CommandEncoder)

Source§

fn accumulate_fused(&self)

Auto Trait Implementations§

Blanket Implementations§

§

impl<S, A> Aggregate<Result<S, Error>> for A
where A: Aggregate<S>,

§

fn from_shares<T>(iter: T) -> Result<A, Error>
where T: IntoIterator<Item = Result<S, Error>>,

Aggregate shares in an MPC protocol.
Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
§

impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
where ST: ?Sized, DT: ?Sized,

§

impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
where ST: ?Sized, DT: ?Sized,

§

impl<T> Downcast<T> for T

§

fn downcast(&self) -> &T

Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

§

impl<T> Instrument for T

§

fn instrument(self, span: Span) -> Instrumented<Self> ⓘ

Instruments this type with the provided [Span], returning an Instrumented wrapper. Read more
§

fn in_current_span(self) -> Instrumented<Self> ⓘ

Instruments this type with the current Span, returning an Instrumented wrapper. Read more
Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoEither for T

Source§

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
§

impl<T> Pointable for T

§

const ALIGN: usize

The alignment of pointer.
§

type Init = T

The type for initializers.
§

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more
§

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more
§

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more
§

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more
§

impl<T> PolicyExt for T
where T: ?Sized,

§

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] only if self and other return Action::Follow. Read more
§

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] if either self or other returns Action::Follow. Read more
§

impl<T> Read<Exclusive, BecauseExclusive> for T
where T: ?Sized,

Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
§

impl<T> Upcast<T> for T

§

fn upcast(&self) -> Option<&T>

§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

§

fn vzip(self) -> V

§

impl<T> WasmNotSend for T
where T: Send,

§

impl<T> WasmNotSendSync for T
where T: WasmNotSend + WasmNotSync,

§

impl<T> WasmNotSync for T
where T: Sync,

§

impl<T> WithSubscriber for T

§

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> ⓘ
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a [WithDispatch] wrapper. Read more
§

fn with_current_subscriber(self) -> WithDispatch<Self> ⓘ

Attaches the current default Subscriber to this type, returning a [WithDispatch] wrapper. Read more