Skip to main content

qualia_core_db/inference/
etalon_tensor_bridge.rs

1//! Bridge between `etalon-tensor` accelerated operators and QualiaDB inference.
2//!
3//! Exposes unified forward execution arenas, FlashAttention compute pipelines,
4//! in-shader top-k reduction, and DirectML / CUDA tensor execution lanes.
5
6pub use etalon_tensor::{
7    online_softmax_rescale, AttentionParams, CapturedCudaMegaPass, CudaMegaPassExecutor,
8    DirectMlComputeLane, DmlGemmOp, DmlGemmParams, DmlLane, ForwardArena, ForwardMode,
9    FusedAttentionPipeline, KvCacheLayout, LogitMaskHeader, PagedKvBlockTable, QuantScheme,
10    TensorView, TopkParams, TopkReductionPipeline, FUSED_ATTENTION_WGSL, TOPK_REDUCTION_WGSL,
11};
12
13/// Construct a unified forward execution arena on the given GPU device.
14///
15/// Sizes the arena's intermediate buffers to accommodate the maximum dimension
16/// across `hidden_dim`, `intermediate_dim`, and `vocab_size` for `max_batch` elements.
17pub fn build_forward_arena_on(
18    device: &etalon::Device,
19    hidden_dim: u32,
20    intermediate_dim: u32,
21    vocab_size: u32,
22    max_batch: u32,
23) -> ForwardArena {
24    let effective_dim = hidden_dim.max(intermediate_dim).max(vocab_size);
25    let mut arena = ForwardArena::new(device, max_batch, effective_dim, 1);
26    arena.hidden_dim = hidden_dim;
27    arena
28}
29
30/// Construct a compiled fused FlashAttention compute pipeline on the given device.
31pub fn build_fused_attention_on(device: &etalon::Device) -> FusedAttentionPipeline {
32    FusedAttentionPipeline::new(device)
33}
34
35/// Construct a compiled GPU block top-k reduction compute pipeline on the given device.
36pub fn build_topk_pipeline_on(device: &etalon::Device) -> TopkReductionPipeline {
37    TopkReductionPipeline::new(device)
38}
39
40/// Attempt to construct a unified forward execution arena using the process-wide shared GPU device.
41#[cfg(not(target_arch = "wasm32"))]
42pub fn try_shared_forward_arena(
43    hidden_dim: u32,
44    intermediate_dim: u32,
45    vocab_size: u32,
46    max_batch: u32,
47) -> Option<ForwardArena> {
48    crate::gpu_context::try_shared_gpu().map(|ctx| {
49        build_forward_arena_on(
50            &ctx.device,
51            hidden_dim,
52            intermediate_dim,
53            vocab_size,
54            max_batch,
55        )
56    })
57}
58
59/// Attempt to construct a fused FlashAttention compute pipeline using the process-wide shared GPU device.
60#[cfg(not(target_arch = "wasm32"))]
61pub fn try_shared_fused_attention() -> Option<FusedAttentionPipeline> {
62    crate::gpu_context::try_shared_gpu().map(|ctx| build_fused_attention_on(&ctx.device))
63}
64
65/// Attempt to construct a top-k reduction compute pipeline using the process-wide shared GPU device.
66#[cfg(not(target_arch = "wasm32"))]
67pub fn try_shared_topk_pipeline() -> Option<TopkReductionPipeline> {
68    crate::gpu_context::try_shared_gpu().map(|ctx| build_topk_pipeline_on(&ctx.device))
69}
70
71/// Dispatch fused attention on an active compute pass.
72#[inline]
73pub fn dispatch_fused_attention<'a>(
74    pipeline: &'a FusedAttentionPipeline,
75    cpass: &mut etalon::ComputePass<'a>,
76    bind_group: &'a etalon::BindGroup,
77    num_heads: u32,
78) {
79    pipeline.dispatch(cpass, bind_group, num_heads);
80}
81
82/// Dispatch top-k block reduction on an active compute pass.
83#[inline]
84pub fn dispatch_topk_reduction<'a>(
85    pipeline: &'a TopkReductionPipeline,
86    cpass: &mut etalon::ComputePass<'a>,
87    bind_group: &'a etalon::BindGroup,
88    num_blocks: u32,
89) {
90    pipeline.dispatch(cpass, bind_group, num_blocks);
91}
92
93/// Record a copy of token candidates into the forward arena readback stage.
94#[inline]
95pub fn record_readback_copy(
96    arena: &ForwardArena,
97    encoder: &mut etalon::CommandEncoder,
98    token_src: &etalon::Buffer,
99    mode: ForwardMode,
100) {
101    arena.record_readback_copy(encoder, token_src, mode);
102}
103
104/// Blocking readback of a single decoded token from the forward arena staging buffer.
105#[inline]
106pub fn read_token_blocking(arena: &ForwardArena, device: &etalon::Device) -> Option<u32> {
107    arena.read_token_blocking(device)
108}
109
110/// Upload attention uniform parameters to a uniform buffer using the device queue.
111#[inline]
112pub fn upload_attention_params(
113    queue: &etalon::Queue,
114    buffer: &etalon::Buffer,
115    offset: etalon::BufferAddress,
116    params: &AttentionParams,
117) {
118    queue.write_buffer(buffer, offset, bytemuck::bytes_of(params));
119}
120
121/// Upload top-k uniform parameters to a uniform buffer using the device queue.
122#[inline]
123pub fn upload_topk_params(
124    queue: &etalon::Queue,
125    buffer: &etalon::Buffer,
126    offset: etalon::BufferAddress,
127    params: &TopkParams,
128) {
129    queue.write_buffer(buffer, offset, bytemuck::bytes_of(params));
130}
131
132/// Upload logit mask header to a uniform/storage buffer using the device queue.
133#[inline]
134pub fn upload_logit_mask_header(
135    queue: &etalon::Queue,
136    buffer: &etalon::Buffer,
137    offset: etalon::BufferAddress,
138    header: &LogitMaskHeader,
139) {
140    queue.write_buffer(buffer, offset, bytemuck::bytes_of(header));
141}
142
143#[cfg(test)]
144mod tests {
145    use super::*;
146
147    #[test]
148    fn test_online_softmax_rescale_all_masked_guard() {
149        let (rescale, weight) = online_softmax_rescale(f32::NEG_INFINITY, f32::NEG_INFINITY);
150        assert_eq!(rescale, 0.0);
151        assert_eq!(weight, 0.0);
152
153        let (rescale, weight) = online_softmax_rescale(10.0, f32::MIN);
154        assert_eq!(rescale, 0.0);
155        assert_eq!(weight, 0.0);
156
157        let (rescale, weight) = online_softmax_rescale(f32::MIN, 5.0);
158        assert_eq!(rescale, 0.0);
159        assert_eq!(weight, 1.0);
160
161        let (rescale, weight) = online_softmax_rescale(4.0, 4.0);
162        assert!((rescale - 1.0).abs() < 1e-6);
163        assert_eq!(weight, 1.0);
164
165        let (rescale, weight) = online_softmax_rescale(2.0, 4.0);
166        let expected = (2.0f32 - 4.0f32).exp();
167        assert!((rescale - expected).abs() < 1e-6);
168        assert_eq!(weight, 1.0);
169    }
170
171    #[test]
172    fn test_kv_cache_layout_and_paged_table() {
173        assert_eq!(KvCacheLayout::PagedBlock, KvCacheLayout::PagedBlock);
174        assert_ne!(KvCacheLayout::Fp32, KvCacheLayout::Int8Quantized);
175
176        let mut table = PagedKvBlockTable::new(64, 32, 512);
177        assert_eq!(table.block_size, 64);
178        assert_eq!(table.num_layers, 32);
179        assert_eq!(table.max_blocks, 512);
180        assert!(table.block_entries.is_empty());
181
182        table.block_entries.push(0x1234);
183        table.block_entries.push(0x5678);
184        assert_eq!(table.block_entries.len(), 2);
185        assert_eq!(table.block_entries[0], 0x1234);
186    }
187
188    #[test]
189    fn test_logit_mask_header_alignment_and_size() {
190        assert_eq!(core::mem::size_of::<LogitMaskHeader>(), 80);
191        assert_eq!(core::mem::align_of::<LogitMaskHeader>(), 16);
192
193        let header = LogitMaskHeader::default();
194        assert_eq!(header.mask_words.len(), 16);
195        assert_eq!(header.mask_words[0], u32::MAX);
196        assert_eq!(header.temperature, 1.0);
197        assert_eq!(header.top_p, 1.0);
198        assert_eq!(header.min_p, 0.0);
199        assert_eq!(header.presence_penalty, 0.0);
200
201        let bytes = bytemuck::bytes_of(&header);
202        assert_eq!(bytes.len(), 80);
203    }
204
205    #[test]
206    fn test_attention_and_topk_params_layout() {
207        assert_eq!(core::mem::size_of::<AttentionParams>(), 48);
208        assert_eq!(core::mem::align_of::<AttentionParams>(), 16);
209
210        assert_eq!(core::mem::size_of::<TopkParams>(), 16);
211        assert_eq!(core::mem::align_of::<TopkParams>(), 16);
212    }
213}