qualia_core_db/gguf_bridge/
projection_gpu_adapter.rs1#![allow(unused_variables, unused_mut)]
2use super::*;
3use etalon_inference::projections::{
4 Phase, Profile, ProjectionGpuProvider, ProjectionResources, ProjectionTensor,
5};
6impl QTensorEngine {
7 pub fn try_native_projection_block(
10 &self,
11 tensor: &ProjectionTensor<'_>,
12 input: &[f32],
13 ) -> Result<Option<Vec<f32>>, &'static str> {
14 if self.model_bytes().is_some() {
17 let rows = tensor.shape.first().copied().unwrap_or(4096);
18 let cols = tensor.shape.get(1).copied().unwrap_or(4096);
19 let payload = rows
20 .checked_mul(cols)
21 .and_then(|n| n.checked_mul(2))
22 .and_then(|n| n.checked_add(input.len()))
23 .and_then(|n| {
24 rows.checked_mul(input.len().checked_div(cols)?)?
25 .checked_mul(3)?
26 .checked_add(n)
27 })
28 .and_then(|n| n.checked_mul(4));
29 if !payload.is_some_and(|n| n <= self.projection_cold_byte_budget) {
30 return Err("native projection payload exceeds caller budget or shape overflowed");
31 }
32 }
33 #[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios")))]
34 let result: Option<Vec<f32>> = None;
35 #[cfg(target_os = "windows")]
36 let result = self.dml.as_ref().and_then(|device| {
37 etalon_inference::projections::backend_q4::directml(
38 device,
39 self.model_bytes()?,
40 self.tensor_data_offset,
41 tensor,
42 input,
43 self.projection_cold_byte_budget,
44 )
45 });
46 #[cfg(any(target_os = "macos", target_os = "ios"))]
47 let result = self.model_bytes().and_then(|model| {
48 etalon_inference::projections::backend_q4::accelerate(
49 model,
50 self.tensor_data_offset,
51 tensor,
52 input,
53 self.projection_cold_byte_budget,
54 )
55 });
56 if result.is_some() {
57 let rows = tensor.shape.first().copied().unwrap_or(4096);
58 let cols = tensor.shape.get(1).copied().unwrap_or(4096);
59 self.record_operations(rows.saturating_mul(cols));
60 }
61 Ok(result)
62 }
63}
64impl ProjectionGpuProvider for QTensorEngine {
65 fn tensor_data_offset(&self) -> u64 {
66 self.tensor_data_offset
67 }
68 fn backend_fused_block(
69 &self,
70 tensor: &ProjectionTensor<'_>,
71 input: &[f32],
72 ) -> Option<Vec<f32>> {
73 self.try_native_projection_block(tensor, input)
74 .unwrap_or_else(|error| panic!("{error}"))
75 }
76 fn record_operations(&self, count: usize) {
77 crate::telemetry::SIEVE_OPS_COUNT.fetch_add(count, std::sync::atomic::Ordering::Relaxed);
78 }
79 fn readback_owned(
80 &self,
81 buffer: &etalon::Buffer,
82 bytes: u64,
83 panic_on_error: bool,
84 ) -> Option<Vec<f32>> {
85 let result = etalon_inference::projections::readback::readback_owned(
86 &self.projection_readback,
87 buffer,
88 bytes,
89 self.projection_cold_byte_budget,
90 || self.poll_wait(),
91 );
92 if panic_on_error && result.is_none() {
93 panic!("fused block readback failed or exceeded readback budget");
94 }
95 result
96 }
97 fn projection_resources(&self) -> ProjectionResources<'_> {
98 ProjectionResources {
99 gemm_input_buf: &self.gemm_input_buf,
100 gemm_weight_buf: &self.gemm_weight_buf,
101 gemm_output_buf: &self.gemm_output_buf,
102 gemm_params_buf: &self.gemm_params_buf,
103 gemm_output_staging: &self.gemm_output_staging,
104 topk_params_buf: &self.topk_params_buf,
105 topk_cand_val_buf: &self.topk_cand_val_buf,
106 topk_cand_idx_buf: &self.topk_cand_idx_buf,
107 topk_cand_staging: &self.topk_cand_staging,
108 mc8_logits_resident_buf: &self.mc8_logits_resident_buf,
109 coop_gemv_pipeline: &self.coop_gemv_pipeline,
110 coop_gemv_mr_pipeline: &self.coop_gemv_mr_pipeline,
111 pipeline: &self.pipeline,
112 embedding_pipeline: &self.embedding_pipeline,
113 mock_pipeline: &self.mock_pipeline,
114 embedding_bind_layout: &self.embedding_bind_layout,
115 pipeline_bind_layout: &self.pipeline_bind_layout,
116 output_topk_pipeline: &self.output_topk_pipeline,
117 output_topk_bind_layout: &self.output_topk_bind_layout,
118 gemm_max_out_dim: self.gemm_max_out_dim,
119 max_tensor_bytes: self.max_tensor_bytes,
120 mc8_logits_row_bytes: self.mc8_logits_row_bytes,
121 }
122 }
123 fn gpu_device(&self) -> &etalon::Device {
124 QTensorEngine::gpu_device(self)
125 }
126 fn gpu_queue(&self) -> &etalon::Queue {
127 QTensorEngine::gpu_queue(self)
128 }
129 fn coop_gemv_enabled(&self) -> bool {
130 crate::llm_bench::coop_gemv_enabled()
131 }
132 fn coop_gemv_workgroups(&self, rows: u32) -> u32 {
133 crate::llm_bench::coop_gemv_workgroups(rows)
134 }
135 fn resident_weights_enabled(&self) -> bool {
136 crate::llm_bench::resident_weights_enabled()
137 }
138 fn native_gemm_bind_layout(&self, coop: bool) -> &etalon::BindGroupLayout {
139 QTensorEngine::native_gemm_bind_layout(self, coop)
140 }
141 fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<etalon::Buffer> {
142 QTensorEngine::resident_weight_buffer(self, key, raw)
143 }
144 fn write_weight_words(&self, raw: &[u8], max_bytes: usize) {
145 QTensorEngine::write_weight_words(self, raw, max_bytes)
146 }
147 fn submit(&self, commands: etalon::CommandBuffer) {
148 self.submit_commands(Some(commands));
149 }
150 fn map_read(&self, buffer: &etalon::Buffer, bytes: u64) -> bool {
151 etalon_inference::projections::readback::map_read(
152 &self.projection_readback,
153 buffer,
154 bytes,
155 || self.poll_wait(),
156 )
157 }
158 fn readback(&self, buffer: &etalon::Buffer, bytes: u64, out: &mut [f32]) -> bool {
159 etalon_inference::projections::readback::readback_into(
160 &self.projection_readback,
161 buffer,
162 bytes,
163 out,
164 || self.poll_wait(),
165 )
166 }
167 fn pass_writes(&self, profile: Profile) -> Option<etalon::ComputePassTimestampWrites<'static>> {
168 match profile {
169 Profile::Both => crate::llm_gpu_profiler::pass_writes_both(),
170 Profile::Begin => crate::llm_gpu_profiler::pass_writes_begin(),
171 Profile::End => crate::llm_gpu_profiler::pass_writes_end(),
172 }
173 }
174 fn resolve(&self, encoder: &mut etalon::CommandEncoder) {
175 crate::llm_gpu_profiler::resolve(encoder)
176 }
177 fn accumulate(&self, phase: Phase) {
178 crate::llm_gpu_profiler::accumulate(match phase {
179 Phase::Gemm => crate::llm_gpu_profiler::Phase::Gemm,
180 Phase::OutputTopk => crate::llm_gpu_profiler::Phase::OutputTopk,
181 Phase::Embedding => crate::llm_gpu_profiler::Phase::Embedding,
182 Phase::FusedBlock => crate::llm_gpu_profiler::Phase::FusedBlock,
183 })
184 }
185 fn cpu_gemm(
186 &self,
187 raw: &[u8],
188 info: &GgufTensorInfo,
189 input: &[f32],
190 out: &mut [f32],
191 n_in: usize,
192 n_out: usize,
193 ) -> bool {
194 stack_gemm_quant(raw, info, input, out, n_in, n_out)
195 }
196 fn cuda_gemm(
197 &self,
198 info: &GgufTensorInfo,
199 raw: &[u8],
200 input: &[f32],
201 out: &mut [f32],
202 n_in: usize,
203 n_out: usize,
204 ) -> bool {
205 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
207 {
208 if crate::prefer_tensor_core_gemm() {
209 if info.ggml_type == crate::ggml_quants::GGML_TYPE_Q4_K_SOA
212 && crate::try_q4k_soa_gemv(n_in, n_out, &input[..n_in], raw, &mut out[..n_out])
213 {
214 return true;
215 }
216 let densify_tc_decode = matches!(
222 crate::gguf_bridge::execution_env::var("QUALIA_LLM_CUDA_TC_DECODE")
223 .ok()
224 .as_deref(),
225 Some("1") | Some("true")
226 );
227 if densify_tc_decode
228 && n_in >= 16
229 && n_out >= 16
230 && n_in.saturating_mul(n_out) <= crate::inference::cuda_lane::MAX_DENSE_ELEMS
231 {
232 let key = crate::weight_fingerprint(raw, n_in, n_out);
233 if crate::try_cuda_batch_gemv_cached_only(
234 key,
235 &input[..n_in],
236 1,
237 &mut out[..n_out],
238 ) {
239 return true;
240 }
241 if let Some(dense) = Self::dequant_weight_dense_f32(info, raw, n_in, n_out) {
242 if crate::try_cuda_batch_gemv_cached(
243 key,
244 &input[..n_in],
245 1,
246 n_in,
247 n_out,
248 &dense,
249 &mut out[..n_out],
250 ) {
251 return true;
252 }
253 }
254 }
255 }
256 }
257
258 false
259 }
260}