Skip to main content

qualia_core_db/gguf_bridge/
native_attention_adapter.rs

1use super::*;
2use etalon_inference::native_attention::{AttentionResources, GpuAttentionProvider};
3use etalon_model::inference::ModelHyperparams;
4impl GpuAttentionProvider for QTensorEngine {
5    type Layout = KvCacheLayout;
6    type Mask = [u32; KV_ATTENTION_MASK_WORDS];
7    fn attention_resources(&self) -> AttentionResources<'_> {
8        AttentionResources {
9            gemm_input_buf: self.gemm_input_buf.as_ref(),
10            gemm_output_buf: self.gemm_output_buf.as_ref(),
11            gemm_aux_buf: self.gemm_aux_buf.as_ref(),
12            gemm_output_staging: self.gemm_output_staging.as_ref(),
13            attention_params_buf: self.attention_params_buf.as_ref(),
14            attention_mask_buf: self.attention_mask_buf.as_ref(),
15            gemm_params_buf: self.gemm_params_buf.as_ref(),
16            kv_cache_gpu: self.kv_cache_gpu.as_ref(),
17            attention_kv_gemm_params: self.attention_kv_gemm_params.as_ref(),
18            attention_kv_params: self.attention_kv_params.as_ref(),
19            attention_bind_layout: &self.attention_bind_layout,
20            attention_pipeline: &self.attention_pipeline,
21            coop_gemv_pipeline: &self.coop_gemv_pipeline,
22            pipeline: &self.pipeline,
23            gemm_max_input_floats: self.gemm_max_input_floats,
24            gemm_max_out_dim: self.gemm_max_out_dim,
25            max_tensor_bytes: self.max_tensor_bytes,
26        }
27    }
28    fn resident_weights_enabled(&self) -> bool {
29        crate::llm_bench::resident_weights_enabled()
30    }
31    fn coop_gemv_enabled(&self) -> bool {
32        crate::llm_bench::coop_gemv_enabled()
33    }
34    fn gpu_device(&self) -> &etalon::Device {
35        QTensorEngine::gpu_device(self)
36    }
37    fn gpu_queue(&self) -> &etalon::Queue {
38        QTensorEngine::gpu_queue(self)
39    }
40    fn native_gemm_bind_layout(&self, coop: bool) -> &etalon::BindGroupLayout {
41        QTensorEngine::native_gemm_bind_layout(self, coop)
42    }
43    fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<etalon::Buffer> {
44        QTensorEngine::resident_weight_buffer(self, key, raw)
45    }
46    fn submit(&self, command: etalon::CommandBuffer) -> bool {
47        self.submit_commands([command]).is_some()
48    }
49    fn pass_writes_begin(&self) -> Option<etalon::ComputePassTimestampWrites<'static>> {
50        crate::llm_gpu_profiler::pass_writes_begin()
51    }
52    fn pass_writes_end(&self) -> Option<etalon::ComputePassTimestampWrites<'static>> {
53        crate::llm_gpu_profiler::pass_writes_end()
54    }
55    fn resolve(&self, encoder: &mut etalon::CommandEncoder) {
56        crate::llm_gpu_profiler::resolve(encoder)
57    }
58    fn accumulate_attention(&self) {
59        crate::llm_gpu_profiler::accumulate(crate::llm_gpu_profiler::Phase::Attention)
60    }
61    fn layer_stride(layout: &Self::Layout) -> u32 {
62        layout.layer_stride
63    }
64    fn attention_kv_mask_for_dispatch(
65        layout: &Self::Layout,
66        token_idx: u32,
67        proj_kind: u32,
68    ) -> (Self::Mask, u32, u32) {
69        QTensorEngine::attention_kv_mask_for_dispatch(layout, token_idx, proj_kind)
70    }
71    fn attention_gpu_params(
72        h: &ModelHyperparams,
73        layout: &Self::Layout,
74        layer: u32,
75        token_idx: u32,
76        info: &GgufTensorInfo,
77        raw_len: usize,
78        proj_kind: u32,
79        num_tokens_in_batch: u32,
80        batch_start_token_idx: u32,
81        mask_active: u32,
82        mask_word_count: u32,
83        out_stride_elems: u32,
84    ) -> AttentionGpuParams {
85        QTensorEngine::attention_gpu_params(
86            &(*h).into(),
87            layout,
88            layer,
89            token_idx,
90            info,
91            raw_len,
92            proj_kind,
93            num_tokens_in_batch,
94            batch_start_token_idx,
95            mask_active,
96            mask_word_count,
97            out_stride_elems,
98        )
99    }
100    fn readback(&self, staging: &etalon::Buffer, out_bytes: u64, out: &mut [f32]) -> bool {
101        let slice = staging.slice(..out_bytes);
102        let (tx, rx) = futures_channel::oneshot::channel();
103        slice.map_async(etalon::MapMode::Read, move |r| {
104            let _ = tx.send(r);
105        });
106        self.poll_wait();
107        if let Ok(handle) = tokio::runtime::Handle::try_current() {
108            if handle
109                .block_on(rx)
110                .ok()
111                .map(|mapped| mapped.is_ok())
112                .unwrap_or(false)
113            {
114                let data = slice
115                    .get_mapped_range()
116                    .expect("wgpu buffer map_range failed");
117                let floats: &[f32] = bytemuck::cast_slice(&data);
118                out.copy_from_slice(&floats[..out.len()]);
119                drop(data);
120                staging.unmap();
121                return true;
122            }
123        }
124        let _ = staging.unmap();
125        false
126    }
127}