qualia_core_db/gguf_bridge/
native_attention_adapter.rs1use super::*;
2use etalon_inference::native_attention::{AttentionResources, GpuAttentionProvider};
3use etalon_model::inference::ModelHyperparams;
4impl GpuAttentionProvider for QTensorEngine {
5 type Layout = KvCacheLayout;
6 type Mask = [u32; KV_ATTENTION_MASK_WORDS];
7 fn attention_resources(&self) -> AttentionResources<'_> {
8 AttentionResources {
9 gemm_input_buf: self.gemm_input_buf.as_ref(),
10 gemm_output_buf: self.gemm_output_buf.as_ref(),
11 gemm_aux_buf: self.gemm_aux_buf.as_ref(),
12 gemm_output_staging: self.gemm_output_staging.as_ref(),
13 attention_params_buf: self.attention_params_buf.as_ref(),
14 attention_mask_buf: self.attention_mask_buf.as_ref(),
15 gemm_params_buf: self.gemm_params_buf.as_ref(),
16 kv_cache_gpu: self.kv_cache_gpu.as_ref(),
17 attention_kv_gemm_params: self.attention_kv_gemm_params.as_ref(),
18 attention_kv_params: self.attention_kv_params.as_ref(),
19 attention_bind_layout: &self.attention_bind_layout,
20 attention_pipeline: &self.attention_pipeline,
21 coop_gemv_pipeline: &self.coop_gemv_pipeline,
22 pipeline: &self.pipeline,
23 gemm_max_input_floats: self.gemm_max_input_floats,
24 gemm_max_out_dim: self.gemm_max_out_dim,
25 max_tensor_bytes: self.max_tensor_bytes,
26 }
27 }
28 fn resident_weights_enabled(&self) -> bool {
29 crate::llm_bench::resident_weights_enabled()
30 }
31 fn coop_gemv_enabled(&self) -> bool {
32 crate::llm_bench::coop_gemv_enabled()
33 }
34 fn gpu_device(&self) -> &etalon::Device {
35 QTensorEngine::gpu_device(self)
36 }
37 fn gpu_queue(&self) -> &etalon::Queue {
38 QTensorEngine::gpu_queue(self)
39 }
40 fn native_gemm_bind_layout(&self, coop: bool) -> &etalon::BindGroupLayout {
41 QTensorEngine::native_gemm_bind_layout(self, coop)
42 }
43 fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<etalon::Buffer> {
44 QTensorEngine::resident_weight_buffer(self, key, raw)
45 }
46 fn submit(&self, command: etalon::CommandBuffer) -> bool {
47 self.submit_commands([command]).is_some()
48 }
49 fn pass_writes_begin(&self) -> Option<etalon::ComputePassTimestampWrites<'static>> {
50 crate::llm_gpu_profiler::pass_writes_begin()
51 }
52 fn pass_writes_end(&self) -> Option<etalon::ComputePassTimestampWrites<'static>> {
53 crate::llm_gpu_profiler::pass_writes_end()
54 }
55 fn resolve(&self, encoder: &mut etalon::CommandEncoder) {
56 crate::llm_gpu_profiler::resolve(encoder)
57 }
58 fn accumulate_attention(&self) {
59 crate::llm_gpu_profiler::accumulate(crate::llm_gpu_profiler::Phase::Attention)
60 }
61 fn layer_stride(layout: &Self::Layout) -> u32 {
62 layout.layer_stride
63 }
64 fn attention_kv_mask_for_dispatch(
65 layout: &Self::Layout,
66 token_idx: u32,
67 proj_kind: u32,
68 ) -> (Self::Mask, u32, u32) {
69 QTensorEngine::attention_kv_mask_for_dispatch(layout, token_idx, proj_kind)
70 }
71 fn attention_gpu_params(
72 h: &ModelHyperparams,
73 layout: &Self::Layout,
74 layer: u32,
75 token_idx: u32,
76 info: &GgufTensorInfo,
77 raw_len: usize,
78 proj_kind: u32,
79 num_tokens_in_batch: u32,
80 batch_start_token_idx: u32,
81 mask_active: u32,
82 mask_word_count: u32,
83 out_stride_elems: u32,
84 ) -> AttentionGpuParams {
85 QTensorEngine::attention_gpu_params(
86 &(*h).into(),
87 layout,
88 layer,
89 token_idx,
90 info,
91 raw_len,
92 proj_kind,
93 num_tokens_in_batch,
94 batch_start_token_idx,
95 mask_active,
96 mask_word_count,
97 out_stride_elems,
98 )
99 }
100 fn readback(&self, staging: &etalon::Buffer, out_bytes: u64, out: &mut [f32]) -> bool {
101 let slice = staging.slice(..out_bytes);
102 let (tx, rx) = futures_channel::oneshot::channel();
103 slice.map_async(etalon::MapMode::Read, move |r| {
104 let _ = tx.send(r);
105 });
106 self.poll_wait();
107 if let Ok(handle) = tokio::runtime::Handle::try_current() {
108 if handle
109 .block_on(rx)
110 .ok()
111 .map(|mapped| mapped.is_ok())
112 .unwrap_or(false)
113 {
114 let data = slice
115 .get_mapped_range()
116 .expect("wgpu buffer map_range failed");
117 let floats: &[f32] = bytemuck::cast_slice(&data);
118 out.copy_from_slice(&floats[..out.len()]);
119 drop(data);
120 staging.unmap();
121 return true;
122 }
123 }
124 let _ = staging.unmap();
125 false
126 }
127}