qualia_core_db/inference/
etalon_tensor_bridge.rs1pub use etalon_tensor::{
7 online_softmax_rescale, AttentionParams, CapturedCudaMegaPass, CudaMegaPassExecutor,
8 DirectMlComputeLane, DmlGemmOp, DmlGemmParams, DmlLane, ForwardArena, ForwardMode,
9 FusedAttentionPipeline, KvCacheLayout, LogitMaskHeader, PagedKvBlockTable, QuantScheme,
10 TensorView, TopkParams, TopkReductionPipeline, FUSED_ATTENTION_WGSL, TOPK_REDUCTION_WGSL,
11};
12
13pub fn build_forward_arena_on(
18 device: &etalon::Device,
19 hidden_dim: u32,
20 intermediate_dim: u32,
21 vocab_size: u32,
22 max_batch: u32,
23) -> ForwardArena {
24 let effective_dim = hidden_dim.max(intermediate_dim).max(vocab_size);
25 let mut arena = ForwardArena::new(device, max_batch, effective_dim, 1);
26 arena.hidden_dim = hidden_dim;
27 arena
28}
29
30pub fn build_fused_attention_on(device: &etalon::Device) -> FusedAttentionPipeline {
32 FusedAttentionPipeline::new(device)
33}
34
35pub fn build_topk_pipeline_on(device: &etalon::Device) -> TopkReductionPipeline {
37 TopkReductionPipeline::new(device)
38}
39
40#[cfg(not(target_arch = "wasm32"))]
42pub fn try_shared_forward_arena(
43 hidden_dim: u32,
44 intermediate_dim: u32,
45 vocab_size: u32,
46 max_batch: u32,
47) -> Option<ForwardArena> {
48 crate::gpu_context::try_shared_gpu().map(|ctx| {
49 build_forward_arena_on(
50 &ctx.device,
51 hidden_dim,
52 intermediate_dim,
53 vocab_size,
54 max_batch,
55 )
56 })
57}
58
59#[cfg(not(target_arch = "wasm32"))]
61pub fn try_shared_fused_attention() -> Option<FusedAttentionPipeline> {
62 crate::gpu_context::try_shared_gpu().map(|ctx| build_fused_attention_on(&ctx.device))
63}
64
65#[cfg(not(target_arch = "wasm32"))]
67pub fn try_shared_topk_pipeline() -> Option<TopkReductionPipeline> {
68 crate::gpu_context::try_shared_gpu().map(|ctx| build_topk_pipeline_on(&ctx.device))
69}
70
71#[inline]
73pub fn dispatch_fused_attention<'a>(
74 pipeline: &'a FusedAttentionPipeline,
75 cpass: &mut etalon::ComputePass<'a>,
76 bind_group: &'a etalon::BindGroup,
77 num_heads: u32,
78) {
79 pipeline.dispatch(cpass, bind_group, num_heads);
80}
81
82#[inline]
84pub fn dispatch_topk_reduction<'a>(
85 pipeline: &'a TopkReductionPipeline,
86 cpass: &mut etalon::ComputePass<'a>,
87 bind_group: &'a etalon::BindGroup,
88 num_blocks: u32,
89) {
90 pipeline.dispatch(cpass, bind_group, num_blocks);
91}
92
93#[inline]
95pub fn record_readback_copy(
96 arena: &ForwardArena,
97 encoder: &mut etalon::CommandEncoder,
98 token_src: &etalon::Buffer,
99 mode: ForwardMode,
100) {
101 arena.record_readback_copy(encoder, token_src, mode);
102}
103
104#[inline]
106pub fn read_token_blocking(arena: &ForwardArena, device: &etalon::Device) -> Option<u32> {
107 arena.read_token_blocking(device)
108}
109
110#[inline]
112pub fn upload_attention_params(
113 queue: &etalon::Queue,
114 buffer: &etalon::Buffer,
115 offset: etalon::BufferAddress,
116 params: &AttentionParams,
117) {
118 queue.write_buffer(buffer, offset, bytemuck::bytes_of(params));
119}
120
121#[inline]
123pub fn upload_topk_params(
124 queue: &etalon::Queue,
125 buffer: &etalon::Buffer,
126 offset: etalon::BufferAddress,
127 params: &TopkParams,
128) {
129 queue.write_buffer(buffer, offset, bytemuck::bytes_of(params));
130}
131
132#[inline]
134pub fn upload_logit_mask_header(
135 queue: &etalon::Queue,
136 buffer: &etalon::Buffer,
137 offset: etalon::BufferAddress,
138 header: &LogitMaskHeader,
139) {
140 queue.write_buffer(buffer, offset, bytemuck::bytes_of(header));
141}
142
143#[cfg(test)]
144mod tests {
145 use super::*;
146
147 #[test]
148 fn test_online_softmax_rescale_all_masked_guard() {
149 let (rescale, weight) = online_softmax_rescale(f32::NEG_INFINITY, f32::NEG_INFINITY);
150 assert_eq!(rescale, 0.0);
151 assert_eq!(weight, 0.0);
152
153 let (rescale, weight) = online_softmax_rescale(10.0, f32::MIN);
154 assert_eq!(rescale, 0.0);
155 assert_eq!(weight, 0.0);
156
157 let (rescale, weight) = online_softmax_rescale(f32::MIN, 5.0);
158 assert_eq!(rescale, 0.0);
159 assert_eq!(weight, 1.0);
160
161 let (rescale, weight) = online_softmax_rescale(4.0, 4.0);
162 assert!((rescale - 1.0).abs() < 1e-6);
163 assert_eq!(weight, 1.0);
164
165 let (rescale, weight) = online_softmax_rescale(2.0, 4.0);
166 let expected = (2.0f32 - 4.0f32).exp();
167 assert!((rescale - expected).abs() < 1e-6);
168 assert_eq!(weight, 1.0);
169 }
170
171 #[test]
172 fn test_kv_cache_layout_and_paged_table() {
173 assert_eq!(KvCacheLayout::PagedBlock, KvCacheLayout::PagedBlock);
174 assert_ne!(KvCacheLayout::Fp32, KvCacheLayout::Int8Quantized);
175
176 let mut table = PagedKvBlockTable::new(64, 32, 512);
177 assert_eq!(table.block_size, 64);
178 assert_eq!(table.num_layers, 32);
179 assert_eq!(table.max_blocks, 512);
180 assert!(table.block_entries.is_empty());
181
182 table.block_entries.push(0x1234);
183 table.block_entries.push(0x5678);
184 assert_eq!(table.block_entries.len(), 2);
185 assert_eq!(table.block_entries[0], 0x1234);
186 }
187
188 #[test]
189 fn test_logit_mask_header_alignment_and_size() {
190 assert_eq!(core::mem::size_of::<LogitMaskHeader>(), 80);
191 assert_eq!(core::mem::align_of::<LogitMaskHeader>(), 16);
192
193 let header = LogitMaskHeader::default();
194 assert_eq!(header.mask_words.len(), 16);
195 assert_eq!(header.mask_words[0], u32::MAX);
196 assert_eq!(header.temperature, 1.0);
197 assert_eq!(header.top_p, 1.0);
198 assert_eq!(header.min_p, 0.0);
199 assert_eq!(header.presence_penalty, 0.0);
200
201 let bytes = bytemuck::bytes_of(&header);
202 assert_eq!(bytes.len(), 80);
203 }
204
205 #[test]
206 fn test_attention_and_topk_params_layout() {
207 assert_eq!(core::mem::size_of::<AttentionParams>(), 48);
208 assert_eq!(core::mem::align_of::<AttentionParams>(), 16);
209
210 assert_eq!(core::mem::size_of::<TopkParams>(), 16);
211 assert_eq!(core::mem::align_of::<TopkParams>(), 16);
212 }
213}