1use super::*;
4
5#[cfg(not(target_arch = "wasm32"))]
7const F16_PROMOTE_KEY_TAG: u64 = 0xF16E_F16E_0000_00F1;
8
9#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
12pub(super) struct ResidentProjectionCache {
13 source: Option<super::model_bytes::ModelByteHandle>,
14 package: Option<Arc<crate::inference::moe::ftw_loader::FtwModelPackage>>,
15 buffers: etalon_inference::projections::weight_resources::ResidentWeightCache,
16 byte_budget: u64,
17}
18#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
19impl Default for ResidentProjectionCache {
20 fn default() -> Self {
21 Self {
22 source: None,
23 package: None,
24 buffers: Default::default(),
25 byte_budget: crate::gpu_context::global_vram_ledger().budget(),
28 }
29 }
30}
31
32impl QTensorEngine {
33 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
36 pub(crate) fn dequant_weight_dense_f32(
37 info: &GgufTensorInfo,
38 raw: &[u8],
39 n_in: usize,
40 n_out: usize,
41 ) -> Option<Vec<f32>> {
42 etalon_inference::projections::weights::dense_f32(
43 info,
44 raw,
45 n_in,
46 n_out,
47 crate::inference::cuda_lane::MAX_DENSE_ELEMS * 4,
48 )
49 }
50
51 #[cfg(all(not(target_arch = "wasm32"), feature = "cuda"))]
57 pub(crate) fn prewarm_cuda_weight(
58 info: &GgufTensorInfo,
59 raw: &[u8],
60 n_in: usize,
61 n_out: usize,
62 ) -> bool {
63 if !crate::prefer_tensor_core_gemm() {
64 return false;
65 }
66 if n_in < 16 || n_out < 16 {
67 return false;
68 }
69 if n_in.saturating_mul(n_out).saturating_mul(4) > 64 * 1024 * 1024 {
71 return false;
72 }
73 let key = crate::weight_fingerprint(raw, n_in, n_out);
74 if crate::dense_weight_cached(key) {
75 return true;
76 }
77 crate::inference::cuda_lane::cache_dense_weight_direct(key, n_in, n_out, |buf| {
81 etalon_inference::projections::weights::dequant_dense_into(info, raw, n_in, buf)
82 })
83 }
84
85 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
89 pub(crate) fn promote_matrix_to_f16_resident(
90 &self,
91 info: &GgufTensorInfo,
92 raw: &[u8],
93 ) -> Option<(etalon::Buffer, u32, u32, u32)> {
94 if !crate::llm_bench::ffn_f16_enabled() {
95 return None;
96 }
97 let f16_bytes =
98 etalon_inference::projections::weights::promote_f16(info, raw, 512 * 1024 * 1024)?;
99 let nbytes = f16_bytes.len();
100 let n0 = info.dims[0] as usize;
101 let key = (raw.as_ptr() as u64) ^ F16_PROMOTE_KEY_TAG;
102 let buf = self.resident_weight_buffer(key, &f16_bytes)?;
103 Some((
104 buf,
105 crate::ggml_quants::GGML_TYPE_F16,
106 nbytes as u32,
107 n0 as u32,
108 ))
109 }
110
111 #[cfg(feature = "gpu-runtime")]
112 pub(crate) fn write_weight_words(&self, raw: &[u8], max_bytes: usize) {
114 etalon_inference::projections::weight_resources::write_weight_words(
115 self.gpu_queue(),
116 self.gemm_weight_buf.as_ref().expect("gemm weight buf"),
117 raw,
118 max_bytes,
119 )
120 }
121
122 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
129 pub(crate) fn resident_weight_buffer(&self, key: u64, raw: &[u8]) -> Option<etalon::Buffer> {
130 let source = self.clone_model_bytes()?;
131 let mut cache = self.gemm_resident_weights.lock().ok()?;
132 let same_package = match (cache.package.as_ref(), self.ftw_package.as_ref()) {
133 (Some(a), Some(b)) => Arc::ptr_eq(a, b),
134 (None, None) => true,
135 _ => false,
136 };
137 if !same_package
138 || !cache
139 .source
140 .as_ref()
141 .is_some_and(|old| old.same_owner(&source))
142 {
143 cache.buffers.clear();
144 cache.source = Some(source);
145 cache.package = self.ftw_package.clone();
146 }
147 let mut budget =
148 etalon_inference::gpu_resources::ResourceBudget::for_device(self.gpu_device());
149 budget.max_buffer_bytes = cache.byte_budget;
150 etalon_inference::projections::weight_resources::resident_weight_buffer(
151 self.gpu_device(),
152 self.gpu_queue(),
153 &mut cache.buffers,
154 key,
155 raw,
156 budget,
157 )
158 }
159
160 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
163 pub fn set_projection_cold_byte_budget(&mut self, bytes: usize) {
164 self.projection_cold_byte_budget = bytes;
165 }
166
167 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
169 pub fn set_resident_projection_byte_budget(&mut self, bytes: u64) {
170 let cache = self
171 .gemm_resident_weights
172 .get_mut()
173 .unwrap_or_else(|e| e.into_inner());
174 cache.byte_budget = bytes;
175 if cache.buffers.retained_bytes() > bytes {
176 cache.buffers.clear();
177 }
178 }
179
180 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
181 pub(super) fn invalidate_resident_projection_weights(&mut self) {
182 let cache = self
183 .gemm_resident_weights
184 .get_mut()
185 .unwrap_or_else(|e| e.into_inner());
186 cache.buffers.clear();
187 cache.source = None;
188 cache.package = None;
189 }
190
191 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
193 pub(crate) fn dispatch_gemm_raw_into(
194 &self,
195 info: &GgufTensorInfo,
196 raw: &[u8],
197 input: &[f32],
198 out: &mut [f32],
199 n_in: usize,
200 n_out: usize,
201 ) -> bool {
202 etalon_inference::projections::ProjectionGpuEngine::new(self)
203 .dispatch_gemm_raw_into(info, raw, input, out, n_in, n_out)
204 }
205
206 #[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
208 pub(crate) fn dispatch_gemm_raw_into(
209 &self,
210 info: &GgufTensorInfo,
211 raw: &[u8],
212 input: &[f32],
213 out: &mut [f32],
214 n_in: usize,
215 n_out: usize,
216 ) -> bool {
217 if n_in > input.len() || n_out > out.len() {
218 return false;
219 }
220 stack_gemm_quant(raw, info, input, out, n_in, n_out)
221 }
222
223 #[cfg(target_arch = "wasm32")]
227 pub(crate) fn dispatch_gemm_raw_into(
228 &self,
229 info: &GgufTensorInfo,
230 raw: &[u8],
231 input: &[f32],
232 out: &mut [f32],
233 n_in: usize,
234 n_out: usize,
235 ) -> bool {
236 if n_in > input.len() || n_out > out.len() {
237 return false;
238 }
239 stack_gemm_quant(raw, info, input, out, n_in, n_out)
240 }
241
242 pub(crate) fn dispatch_ternary_ffn(
246 &self,
247 info: &GgufTensorInfo,
248 input: &[f32],
249 out: &mut [f32],
250 n_in: usize,
251 n_out: usize,
252 ) -> bool {
253 if n_in > input.len() || n_out > out.len() {
254 return false;
255 }
256 #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
258 if crate::llm_bench::ternary_ffn_enabled() {
259 if let Some(res) = self.ternary_ffn.as_ref() {
260 if res.gemv(
261 self.gpu_device(),
262 self.gpu_queue(),
263 info.byte_offset,
264 input,
265 out,
266 n_in,
267 n_out,
268 ) {
269 return true;
270 }
271 }
272 }
273 let mmap = match self.model_bytes() {
276 Some(m) => m,
277 None => return false,
278 };
279 let raw = match crate::ggml_quants::fetch_tensor_bytes(mmap, self.tensor_data_offset, info)
280 {
281 Ok(s) => s,
282 Err(_) => return false,
283 };
284 if raw.len() < 4 {
285 return false;
286 }
287 let scale = f32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]);
288 crate::ternary::ternary_gemm_cpu(
289 &input[..n_in],
290 &raw[4..],
291 scale,
292 n_in,
293 n_out,
294 1,
295 0,
296 0,
297 &mut out[..n_out],
298 );
299 true
300 }
301
302 pub fn dispatch_gemm_into(
304 &self,
305 index: &crate::gguf_sharder::GgufTensorIndex,
306 info: &GgufTensorInfo,
307 input: &[f32],
308 out: &mut [f32],
309 n_in: usize,
310 n_out: usize,
311 ) -> bool {
312 if n_in > input.len() || n_out > out.len() {
313 wlog(&format!(
314 "[gemm_into] GUARD n_in={n_in} n_out={n_out} input={} out={}",
315 input.len(),
316 out.len()
317 ));
318 return false;
319 }
320 if info.ggml_type == crate::ternary::GGML_TYPE_TERNARY_158 {
323 return self.dispatch_ternary_ffn(info, input, out, n_in, n_out);
324 }
325 #[cfg(not(target_arch = "wasm32"))]
326 let raw = match self.fetch_tensor_raw_bytes(info) {
327 Some(s) => s,
328 None => {
329 let mmap = match self.model_bytes() {
330 Some(m) => m,
331 None => return false,
332 };
333 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, info) {
334 Ok(s) => s,
335 Err(_) => return false,
336 }
337 }
338 };
339 #[cfg(target_arch = "wasm32")]
340 let raw = {
341 let mmap = match self.model_bytes() {
342 Some(m) => m,
343 None => return false,
344 };
345 match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, info) {
346 Ok(s) => s,
347 Err(_) => return false,
348 }
349 };
350 self.dispatch_gemm_raw_into(info, raw, input, out, n_in, n_out)
351 }
352}
353
354#[cfg(test)]
364mod substrate_parity_tests {
365 use crate::gguf_sharder::GgufTensorInfo;
366 use crate::solvers::linear_algebra::gemm::{matvec, Transpose};
367
368 fn run(n_in: usize, n_out: usize, seed: u64) -> (f32, f32) {
373 let mut s = seed | 1;
375 let mut rng = move || -> f32 {
376 s = s
377 .wrapping_mul(6364136223846793005)
378 .wrapping_add(1442695040888963407);
379 ((s >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0
380 };
381
382 let row_bytes = crate::llm_kernel_parity::q8_0_bytes(n_in);
383 let mut raw = vec![0u8; row_bytes * n_out];
384 let mut w_orig = vec![0f32; n_in * n_out]; let mut row_f32 = vec![0f32; n_in];
386 for r in 0..n_out {
387 for x in row_f32.iter_mut() {
388 *x = rng();
389 }
390 w_orig[r * n_in..(r + 1) * n_in].copy_from_slice(&row_f32);
391 assert!(crate::llm_kernel_parity::quantize_q8_0_from_f32(
392 &row_f32,
393 &mut raw[r * row_bytes..(r + 1) * row_bytes],
394 ));
395 }
396 let input: Vec<f32> = (0..n_in).map(|_| rng()).collect();
397
398 let info = GgufTensorInfo {
399 dims: [n_in as u64, n_out as u64, 1, 1],
400 n_dims: 2,
401 ggml_type: crate::ggml_quants::GGML_TYPE_Q8_0,
402 byte_offset: 0,
403 };
404
405 let mut out_llm = vec![0f32; n_out];
407 assert!(crate::gguf_bridge::stack_gemm_quant(
408 &raw,
409 &info,
410 &input,
411 &mut out_llm,
412 n_in,
413 n_out
414 ));
415
416 let mut w_deq = vec![0f64; n_in * n_out];
419 let mut deq_row = vec![0f32; n_in];
420 for i in 0..n_out {
421 let got = crate::ggml_quants::dequant_matrix_row_into(&raw, &info, i, &mut deq_row)
422 .unwrap_or(0);
423 assert_eq!(got, n_in, "dequant row {i}");
424 for j in 0..n_in {
425 w_deq[i * n_in + j] = deq_row[j] as f64;
426 }
427 }
428 let x_f64: Vec<f64> = input.iter().map(|&v| v as f64).collect();
429 let mut out_sub_deq = vec![0f64; n_out];
430 matvec(Transpose::No, n_out, n_in, &w_deq, &x_f64, &mut out_sub_deq).unwrap();
431
432 let w_orig_f64: Vec<f64> = w_orig.iter().map(|&v| v as f64).collect();
434 let mut out_sub_orig = vec![0f64; n_out];
435 matvec(
436 Transpose::No,
437 n_out,
438 n_in,
439 &w_orig_f64,
440 &x_f64,
441 &mut out_sub_orig,
442 )
443 .unwrap();
444
445 let exact_err = (0..n_out)
446 .map(|i| (out_llm[i] as f64 - out_sub_deq[i]).abs() as f32)
447 .fold(0.0f32, f32::max);
448 let quant_err = (0..n_out)
449 .map(|i| (out_llm[i] as f64 - out_sub_orig[i]).abs() as f32)
450 .fold(0.0f32, f32::max);
451 (exact_err, quant_err)
452 }
453
454 #[test]
455 fn llm_quant_gemv_is_the_substrate_gemm() {
456 for &(n_in, n_out, seed) in &[
458 (64usize, 32usize, 0xC0FFEEu64),
459 (128, 96, 7),
460 (256, 64, 0xBEEF),
461 ] {
462 let (exact_err, quant_err) = run(n_in, n_out, seed);
463 assert!(
466 exact_err < 1e-4,
467 "LLM GEMV diverges from substrate GEMM on identical weights: exact_err={exact_err} (n_in={n_in}, n_out={n_out})"
468 );
469 assert!(
471 quant_err < 0.5,
472 "Q8 quantization error unexpectedly large: quant_err={quant_err}"
473 );
474 }
475 }
476}