Skip to main content

qualia_core_db/gguf_bridge/
projection_gpu_adapter.rs

1#![allow(unused_variables, unused_mut)]
2use super::*;
3use etalon_inference::projections::{
4    Phase, Profile, ProjectionGpuProvider, ProjectionResources, ProjectionTensor,
5};
6impl QTensorEngine {
7    /// Fallible native projection route. Governed callers may configure the admitted budget with
8    /// set_projection_cold_byte_budget; ordinary preparation inherits the native host/device policy.
9    pub fn try_native_projection_block(
10        &self,
11        tensor: &ProjectionTensor<'_>,
12        input: &[f32],
13    ) -> Result<Option<Vec<f32>>, &'static str> {
14        // Admission exhaustion must not reinterpret legacy native Q4 bytes through a
15        // different fallback layout. The existing Vec API has no recoverable error channel.
16        if self.model_bytes().is_some() {
17            let rows = tensor.shape.first().copied().unwrap_or(4096);
18            let cols = tensor.shape.get(1).copied().unwrap_or(4096);
19            let payload = rows
20                .checked_mul(cols)
21                .and_then(|n| n.checked_mul(2))
22                .and_then(|n| n.checked_add(input.len()))
23                .and_then(|n| {
24                    rows.checked_mul(input.len().checked_div(cols)?)?
25                        .checked_mul(3)?
26                        .checked_add(n)
27                })
28                .and_then(|n| n.checked_mul(4));
29            if !payload.is_some_and(|n| n <= self.projection_cold_byte_budget) {
30                return Err("native projection payload exceeds caller budget or shape overflowed");
31            }
32        }
33        #[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios")))]
34        let result: Option<Vec<f32>> = None;
35        #[cfg(target_os = "windows")]
36        let result = self.dml.as_ref().and_then(|device| {
37            etalon_inference::projections::backend_q4::directml(
38                device,
39                self.model_bytes()?,
40                self.tensor_data_offset,
41                tensor,
42                input,
43                self.projection_cold_byte_budget,
44            )
45        });
46        #[cfg(any(target_os = "macos", target_os = "ios"))]
47        let result = self.model_bytes().and_then(|model| {
48            etalon_inference::projections::backend_q4::accelerate(
49                model,
50                self.tensor_data_offset,
51                tensor,
52                input,
53                self.projection_cold_byte_budget,
54            )
55        });
56        if result.is_some() {
57            let rows = tensor.shape.first().copied().unwrap_or(4096);
58            let cols = tensor.shape.get(1).copied().unwrap_or(4096);
59            self.record_operations(rows.saturating_mul(cols));
60        }
61        Ok(result)
62    }
63}
64impl ProjectionGpuProvider for QTensorEngine {
65    fn tensor_data_offset(&self) -> u64 {
66        self.tensor_data_offset
67    }
68    fn backend_fused_block(
69        &self,
70        tensor: &ProjectionTensor<'_>,
71        input: &[f32],
72    ) -> Option<Vec<f32>> {
73        self.try_native_projection_block(tensor, input)
74            .unwrap_or_else(|error| panic!("{error}"))
75    }
76    fn record_operations(&self, count: usize) {
77        crate::telemetry::SIEVE_OPS_COUNT.fetch_add(count, std::sync::atomic::Ordering::Relaxed);
78    }
79    fn readback_owned(
80        &self,
81        buffer: &etalon::Buffer,
82        bytes: u64,
83        panic_on_error: bool,
84    ) -> Option<Vec<f32>> {
85        let result = etalon_inference::projections::readback::readback_owned(
86            &self.projection_readback,
87            buffer,
88            bytes,
89            self.projection_cold_byte_budget,
90            || self.poll_wait(),
91        );
92        if panic_on_error && result.is_none() {
93            panic!("fused block readback failed or exceeded readback budget");
94        }
95        result
96    }
97    fn projection_resources(&self) -> ProjectionResources<'_> {
98        ProjectionResources {
99            gemm_input_buf: &self.gemm_input_buf,
100            gemm_weight_buf: &self.gemm_weight_buf,
101            gemm_output_buf: &self.gemm_output_buf,
102            gemm_params_buf: &self.gemm_params_buf,
103            gemm_output_staging: &self.gemm_output_staging,
104            topk_params_buf: &self.topk_params_buf,
105            topk_cand_val_buf: &self.topk_cand_val_buf,
106            topk_cand_idx_buf: &self.topk_cand_idx_buf,
107            topk_cand_staging: &self.topk_cand_staging,
108            mc8_logits_resident_buf: &self.mc8_logits_resident_buf,
109            coop_gemv_pipeline: &self.coop_gemv_pipeline,
110            coop_gemv_mr_pipeline: &self.coop_gemv_mr_pipeline,
111            pipeline: &self.pipeline,
112            embedding_pipeline: &self.embedding_pipeline,
113            mock_pipeline: &self.mock_pipeline,
114            embedding_bind_layout: &self.embedding_bind_layout,
115            pipeline_bind_layout: &self.pipeline_bind_layout,
116            output_topk_pipeline: &self.output_topk_pipeline,
117            output_topk_bind_layout: &self.output_topk_bind_layout,
118            gemm_max_out_dim: self.gemm_max_out_dim,
119            max_tensor_bytes: self.max_tensor_bytes,
120            mc8_logits_row_bytes: self.mc8_logits_row_bytes,
121        }
122    }
123    fn gpu_device(&self) -> &etalon::Device {
124        QTensorEngine::gpu_device(self)
125    }
126    fn gpu_queue(&self) -> &etalon::Queue {
127        QTensorEngine::gpu_queue(self)
128    }
129    fn coop_gemv_enabled(&self) -> bool {
130        crate::llm_bench::coop_gemv_enabled()
131    }
132    fn coop_gemv_workgroups(&self, rows: u32) -> u32 {
133        crate::llm_bench::coop_gemv_workgroups(rows)
134    }
135    fn resident_weights_enabled(&self) -> bool {
136        crate::llm_bench::resident_weights_enabled()
137    }
138    fn native_gemm_bind_layout(&self, coop: bool) -> &etalon::BindGroupLayout {
139        QTensorEngine::native_gemm_bind_layout(self, coop)
140    }
141    fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<etalon::Buffer> {
142        QTensorEngine::resident_weight_buffer(self, key, raw)
143    }
144    fn write_weight_words(&self, raw: &[u8], max_bytes: usize) {
145        QTensorEngine::write_weight_words(self, raw, max_bytes)
146    }
147    fn submit(&self, commands: etalon::CommandBuffer) {
148        self.submit_commands(Some(commands));
149    }
150    fn map_read(&self, buffer: &etalon::Buffer, bytes: u64) -> bool {
151        etalon_inference::projections::readback::map_read(
152            &self.projection_readback,
153            buffer,
154            bytes,
155            || self.poll_wait(),
156        )
157    }
158    fn readback(&self, buffer: &etalon::Buffer, bytes: u64, out: &mut [f32]) -> bool {
159        etalon_inference::projections::readback::readback_into(
160            &self.projection_readback,
161            buffer,
162            bytes,
163            out,
164            || self.poll_wait(),
165        )
166    }
167    fn pass_writes(&self, profile: Profile) -> Option<etalon::ComputePassTimestampWrites<'static>> {
168        match profile {
169            Profile::Both => crate::llm_gpu_profiler::pass_writes_both(),
170            Profile::Begin => crate::llm_gpu_profiler::pass_writes_begin(),
171            Profile::End => crate::llm_gpu_profiler::pass_writes_end(),
172        }
173    }
174    fn resolve(&self, encoder: &mut etalon::CommandEncoder) {
175        crate::llm_gpu_profiler::resolve(encoder)
176    }
177    fn accumulate(&self, phase: Phase) {
178        crate::llm_gpu_profiler::accumulate(match phase {
179            Phase::Gemm => crate::llm_gpu_profiler::Phase::Gemm,
180            Phase::OutputTopk => crate::llm_gpu_profiler::Phase::OutputTopk,
181            Phase::Embedding => crate::llm_gpu_profiler::Phase::Embedding,
182            Phase::FusedBlock => crate::llm_gpu_profiler::Phase::FusedBlock,
183        })
184    }
185    fn cpu_gemm(
186        &self,
187        raw: &[u8],
188        info: &GgufTensorInfo,
189        input: &[f32],
190        out: &mut [f32],
191        n_in: usize,
192        n_out: usize,
193    ) -> bool {
194        stack_gemm_quant(raw, info, input, out, n_in, n_out)
195    }
196    fn cuda_gemm(
197        &self,
198        info: &GgufTensorInfo,
199        raw: &[u8],
200        input: &[f32],
201        out: &mut [f32],
202        n_in: usize,
203        n_out: usize,
204    ) -> bool {
205        // Mode=cuda acceleration (fail closed → wgpu path below).
206        #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
207        {
208            if crate::prefer_tensor_core_gemm() {
209                // M2b: on-device Q4_K SoA dequant-GEMV (no host densify) — preferred for .soa.p64.
210                // This path has CPU differential tests; keep it on for mode=cuda.
211                if info.ggml_type == crate::ggml_quants::GGML_TYPE_Q4_K_SOA
212                    && crate::try_q4k_soa_gemv(n_in, n_out, &input[..n_in], raw, &mut out[..n_out])
213                {
214                    return true;
215                }
216                // Dense densify + CUDA GEMM for F16/Q8/etc. was measured **incoherent** on
217                // single-token decode (pad-to-16 + f16-WMMA reduced path → garbage tokens,
218                // 2026-07-24 SmolLM f16 package). Default OFF for decode GEMV (always batch=1
219                // here). Opt-in: QUALIA_LLM_CUDA_TC_DECODE=1 (lab only). Prefill should use
220                // multi-token dispatch or the resident mega-pass, not this broken default.
221                let densify_tc_decode = matches!(
222                    crate::gguf_bridge::execution_env::var("QUALIA_LLM_CUDA_TC_DECODE")
223                        .ok()
224                        .as_deref(),
225                    Some("1") | Some("true")
226                );
227                if densify_tc_decode
228                    && n_in >= 16
229                    && n_out >= 16
230                    && n_in.saturating_mul(n_out) <= crate::inference::cuda_lane::MAX_DENSE_ELEMS
231                {
232                    let key = crate::weight_fingerprint(raw, n_in, n_out);
233                    if crate::try_cuda_batch_gemv_cached_only(
234                        key,
235                        &input[..n_in],
236                        1,
237                        &mut out[..n_out],
238                    ) {
239                        return true;
240                    }
241                    if let Some(dense) = Self::dequant_weight_dense_f32(info, raw, n_in, n_out) {
242                        if crate::try_cuda_batch_gemv_cached(
243                            key,
244                            &input[..n_in],
245                            1,
246                            n_in,
247                            n_out,
248                            &dense,
249                            &mut out[..n_out],
250                        ) {
251                            return true;
252                        }
253                    }
254                }
255            }
256        }
257
258        false
259    }
260}