qualia_core_db/inference/cuda_lane/
mod.rs

1//! CUDA inference lane — dense batch GEMM via persistent WMMA (mode=`cuda`).
2//!
3//! # Goal
4//! Prefer tensor-core dense matmul for **batch prefill-shaped** GEMMs when:
5//! - `InferenceMode::CudaTc` is active (`prefer_tensor_core_gemm`)
6//! - dims can be padded to multiples of 16
7//! - a dense f32 weight matrix is available (f16 p64 expand, or one-shot dequant)
8//!
9//! Weight rows are **cached on the CUDA slab** by content fingerprint so subsequent
10//! chunks do not re-upload the same matrix (no host thrash for repeated layers).
11//!
12//! # Honest limits
13//! - Not a fused Q4_K dequant-GEMV on-device (llama.cpp-class) — that remains M2b.
14//! - Quantized weights must be dequantized once to dense f32 before first cache insert.
15//! - Slab is finite (256 MiB); LRU-ish eviction of oldest entries when full.
16
17#![cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
18
19mod attention;
20mod device;
21mod gemv;
22mod mega_pass;
23mod paged_attention;
24mod q8;
25mod tuning;
26mod weight_cache;
27
28pub use attention::try_q4k_soa_attention_device;
29pub(crate) use device::{
30    decode_graph_h2d_bytes_per_token, decode_graph_key, decode_graph_node_count,
31};
32pub use device::{
33    device_kv_ready, ensure_device_kv_cache, preload_q4k_soa_weights, preload_resident_blob,
34    q4k_device_weight_count, q4k_weight_resident, warm_cuda_context,
35};
36pub use gemv::{
37    try_q4k_soa_ffn_block, try_q4k_soa_ffn_block_residual, try_q4k_soa_fused_swiglu,
38    try_q4k_soa_gemv, try_q4k_soa_qkv,
39};
40pub(crate) use mega_pass::try_cuda_mega_pass_with_token;
41pub use mega_pass::{
42    prepare_mega_pass_kernels, try_cuda_mega_pass, MegaPassLayerDims, MegaPassLayerWeights,
43    MegaPassPlanView, MegaPassWeightLayout,
44};
45pub use q8::{q8_0_gemv_oracle_into, try_q8_0_cuda_gemv, Q8_0_BLOCK_BYTES, Q8_0_BLOCK_ELEMS};
46pub(crate) use tuning::cuda_q8_tuning_for_model;
47pub use weight_cache::{
48    cache_dense_weight, cache_dense_weight_direct, clear_weight_cache, dense_weight_cached,
49    try_cuda_batch_gemv, try_cuda_batch_gemv_cached, try_cuda_batch_gemv_cached_only,
50    weight_cache_len, weight_fingerprint, MAX_DENSE_ELEMS,
51};
qualia_core_db/inference/cuda_lane/mod.rs

qualia_core_db/inference/cuda_lane/
mod.rs
