Skip to main content

qualia_core_db/inference/
ternary_gpu.rs

1//! Task #12 / STELLAR §A — **native GPU dispatch of the ternary GEMM** + on-device parity.
2//!
3//! Runs `shaders/ternary_gemm.wgsl` (`ternary::TERNARY_GEMM_WGSL`) on a real wgpu device. This is
4//! the piece the FFN inference loop calls to execute a ternary-packed weight on the GPU; the
5//! `#[test]` below verifies it **on silicon** against the byte-exact CPU oracle
6//! `ternary::ternary_gemm_cpu` (it skips cleanly when no adapter is present, e.g. headless CI).
7//!
8//! Native only — the wasm WebGPU path reuses the same WGSL through `gguf_bridge`'s pipeline set
9//! when the kernel is spliced into the layer loop (the remaining integration step).
10
11use crate::ternary::{rebake_ternary_blob_to_2bit, TERNARY_GEMM_2BIT_WGSL, TERNARY_GEMM_WGSL};
12use std::collections::HashMap;
13
14/// 32-byte `TernaryParams` uniform matching `ternary_gemm.wgsl` (n_in, n_out, n_batch,
15/// in_row_stride, out_row_stride, scale, + 2 pad words).
16fn ternary_params_bytes(n_in: u32, n_out: u32, n_batch: u32, scale: f32) -> [u8; 32] {
17    let mut b = [0u8; 32];
18    b[0..4].copy_from_slice(&n_in.to_le_bytes());
19    b[4..8].copy_from_slice(&n_out.to_le_bytes());
20    b[8..12].copy_from_slice(&n_batch.to_le_bytes());
21    // in_row_stride / out_row_stride = 0 → dense (the shader falls back to n_in / n_out)
22    b[20..24].copy_from_slice(&scale.to_le_bytes());
23    b
24}
25
26/// Execute the **base-3** ternary GEMM on the GPU (`ternary_gemm.wgsl`).
27pub fn ternary_gemm_gpu(
28    device: &etalon::Device,
29    queue: &etalon::Queue,
30    activations: &[f32],
31    packed: &[u8],
32    scale: f32,
33    n_in: usize,
34    n_out: usize,
35    n_batch: usize,
36) -> Vec<f32> {
37    run_gemm(
38        device,
39        queue,
40        TERNARY_GEMM_WGSL,
41        activations,
42        packed,
43        scale,
44        n_in,
45        n_out,
46        n_batch,
47    )
48}
49
50/// Execute the **2-bit branchless** ternary GEMM on the GPU (`ternary_gemm_2bit.wgsl`). `packed`
51/// must be 2-bit packed (`ternary::pack_trits_2bit`).
52pub fn ternary_gemm_gpu_2bit(
53    device: &etalon::Device,
54    queue: &etalon::Queue,
55    activations: &[f32],
56    packed: &[u8],
57    scale: f32,
58    n_in: usize,
59    n_out: usize,
60    n_batch: usize,
61) -> Vec<f32> {
62    run_gemm(
63        device,
64        queue,
65        TERNARY_GEMM_2BIT_WGSL,
66        activations,
67        packed,
68        scale,
69        n_in,
70        n_out,
71        n_batch,
72    )
73}
74
75/// Shared dispatch for both ternary GEMM kernels (identical bindings/params; only the WGSL differs).
76/// Returns the `n_batch × n_out` output. Strides default to dense. Blocking (native readback).
77#[allow(clippy::too_many_arguments)]
78fn run_gemm(
79    device: &etalon::Device,
80    queue: &etalon::Queue,
81    wgsl: &str,
82    activations: &[f32],
83    packed: &[u8],
84    scale: f32,
85    n_in: usize,
86    n_out: usize,
87    n_batch: usize,
88) -> Vec<f32> {
89    let n_batch = n_batch.max(1);
90    let out_elems = n_batch * n_out;
91
92    let shader = device.create_shader_module(etalon::ShaderModuleDescriptor {
93        label: Some("ternary_gemm"),
94        source: etalon::ShaderSource::Wgsl(wgsl.into()),
95    });
96    let pipeline = device.create_compute_pipeline(&etalon::ComputePipelineDescriptor {
97        label: Some("ternary_gemm_pipeline"),
98        layout: None, // auto bind-group layout (native)
99        module: &shader,
100        entry_point: Some("ternary_gemm"),
101        compilation_options: Default::default(),
102        cache: None,
103    });
104
105    let mk_buf = |label: &str, contents: &[u8], usage: etalon::BufferUsages| {
106        let buf = device.create_buffer(&etalon::BufferDescriptor {
107            label: Some(label),
108            size: contents.len().max(4) as u64,
109            usage,
110            mapped_at_creation: false,
111        });
112        if !contents.is_empty() {
113            queue.write_buffer(&buf, 0, contents);
114        }
115        buf
116    };
117
118    let act_buf = mk_buf(
119        "ternary_act",
120        bytemuck::cast_slice(activations),
121        etalon::BufferUsages::STORAGE | etalon::BufferUsages::COPY_DST,
122    );
123    // trit_words is array<u32> → pad the packed bytes to a 4-byte multiple.
124    let mut trits = packed.to_vec();
125    while trits.len() % 4 != 0 || trits.is_empty() {
126        trits.push(0);
127    }
128    let trit_buf = mk_buf(
129        "ternary_trits",
130        &trits,
131        etalon::BufferUsages::STORAGE | etalon::BufferUsages::COPY_DST,
132    );
133    let params = ternary_params_bytes(n_in as u32, n_out as u32, n_batch as u32, scale);
134    let param_buf = mk_buf(
135        "ternary_params",
136        &params,
137        etalon::BufferUsages::UNIFORM | etalon::BufferUsages::COPY_DST,
138    );
139    let out_buf = device.create_buffer(&etalon::BufferDescriptor {
140        label: Some("ternary_out"),
141        size: (out_elems * 4).max(4) as u64,
142        usage: etalon::BufferUsages::STORAGE | etalon::BufferUsages::COPY_SRC,
143        mapped_at_creation: false,
144    });
145    let staging = device.create_buffer(&etalon::BufferDescriptor {
146        label: Some("ternary_staging"),
147        size: (out_elems * 4).max(4) as u64,
148        usage: etalon::BufferUsages::COPY_DST | etalon::BufferUsages::MAP_READ,
149        mapped_at_creation: false,
150    });
151
152    let bind = device.create_bind_group(&etalon::BindGroupDescriptor {
153        label: Some("ternary_bind"),
154        layout: &pipeline.get_bind_group_layout(0),
155        entries: &[
156            etalon::BindGroupEntry {
157                binding: 0,
158                resource: act_buf.as_entire_binding(),
159            },
160            etalon::BindGroupEntry {
161                binding: 1,
162                resource: trit_buf.as_entire_binding(),
163            },
164            etalon::BindGroupEntry {
165                binding: 2,
166                resource: param_buf.as_entire_binding(),
167            },
168            etalon::BindGroupEntry {
169                binding: 3,
170                resource: out_buf.as_entire_binding(),
171            },
172        ],
173    });
174
175    let mut encoder = device.create_command_encoder(&etalon::CommandEncoderDescriptor {
176        label: Some("ternary_enc"),
177    });
178    {
179        let mut pass = encoder.begin_compute_pass(&etalon::ComputePassDescriptor {
180            label: Some("ternary_pass"),
181            timestamp_writes: None,
182        });
183        pass.set_pipeline(&pipeline);
184        pass.set_bind_group(0, &bind, &[]);
185        let wg_x = (n_out as u32).div_ceil(64).max(1);
186        pass.dispatch_workgroups(wg_x, n_batch as u32, 1);
187    }
188    encoder.copy_buffer_to_buffer(&out_buf, 0, &staging, 0, (out_elems * 4).max(4) as u64);
189    queue.submit(Some(encoder.finish()));
190
191    let slice = staging.slice(..);
192    let (tx, rx) = std::sync::mpsc::channel();
193    slice.map_async(etalon::MapMode::Read, move |r| {
194        let _ = tx.send(r);
195    });
196    let _ = device.poll(etalon::PollType::wait_indefinitely());
197    rx.recv()
198        .expect("map channel")
199        .expect("map ternary staging");
200    let data = slice
201        .get_mapped_range()
202        .expect("wgpu buffer map_range failed");
203    let out: Vec<f32> = bytemuck::cast_slice(&data)[..out_elems].to_vec();
204    drop(data);
205    staging.unmap();
206    out
207}
208
209// ── A1b inc 2b: resident 2-bit ternary-FFN GEMM (persistent pipeline + resident weights) ─────────
210
211/// Storage-buffer binding offset alignment. wgpu's `min_storage_buffer_offset_alignment` is 256 on
212/// the engine's baselines (A2000/Vulkan, DX12, WebGPU downlevel), so every resident tensor starts on
213/// this boundary — a sub-range binding at a non-aligned offset is a validation error.
214const TERNARY_RESIDENT_ALIGN: u64 = 256;
215
216#[inline]
217fn align_up_u64(x: u64, align: u64) -> u64 {
218    (x + align - 1) / align * align
219}
220
221/// One resident ternary tensor's location + metadata inside [`TernaryFfnResident::weights`].
222#[derive(Clone, Copy, Debug)]
223struct ResidentTernaryTensor {
224    /// 256-aligned byte offset of this tensor's 2-bit packed weights in the resident buffer.
225    gpu_offset: u64,
226    /// 2-bit packed byte length (padded to a 4-byte multiple = the bound `size`).
227    packed_len: u32,
228    n_in: u32,
229    n_out: u32,
230    /// Per-tensor BitNet absmean scale (applied once per output element).
231    scale: f32,
232}
233
234/// A1b (STELLAR §A): the resident **2-bit branchless** ternary-FFN GEMM dispatcher — the perf core.
235///
236/// **The D7 fix.** The 2-bit kernel pipeline + bind-group layout are built **once**; every FFN
237/// ternary weight is rebaked base-3 → 2-bit (lossless; see [`rebake_ternary_blob_to_2bit`]) and
238/// uploaded **once** into a single resident VRAM buffer. (The prior 1.02× failure rebuilt the
239/// pipeline per call.) At decode each FFN GEMV binds its own weight sub-range — no re-upload —
240/// writes only the activation row + params, dispatches, and reads back `n_out` floats. Tensors are
241/// keyed by their P64 blob offset (`GgufTensorInfo::byte_offset`), the unique per-tensor handle.
242///
243/// On-disk the FFN weights stay base-3 (densest, 1.6 bit); 2-bit is the GPU-resident layout only
244/// (2.0 bit, shift/mask, divergence-free → 1.77× vs F16 on the A2000). Native-only; the wasm WebGPU
245/// ternary path reuses the same WGSL through the MC8 resident arena (a later step).
246pub struct TernaryFfnResident {
247    pipeline: etalon::ComputePipeline,
248    weights: etalon::Buffer,
249    act_buf: etalon::Buffer,
250    out_buf: etalon::Buffer,
251    params_buf: etalon::Buffer,
252    staging: etalon::Buffer,
253    map: HashMap<u64, ResidentTernaryTensor>,
254    max_in: u32,
255    max_out: u32,
256    resident_bytes: u64,
257}
258
259impl TernaryFfnResident {
260    /// Build the resident set from `(key, n_in, n_out, base3_blob)` tuples. Each base-3 blob
261    /// (`[scale f32][5-trits/byte]`) is rebaked to the 2-bit runtime layout here — load-time heap is
262    /// the sanctioned path; the hot loop stays zero-heap. `key` is the tensor's P64 blob offset.
263    /// Returns `None` if `tensors` is empty or any blob is malformed.
264    pub fn build(
265        device: &etalon::Device,
266        queue: &etalon::Queue,
267        tensors: &[(u64, usize, usize, &[u8])],
268    ) -> Option<Self> {
269        if tensors.is_empty() {
270            return None;
271        }
272        // 1. Rebake each base-3 blob → 2-bit; lay out 256-aligned within one resident buffer.
273        let mut map: HashMap<u64, ResidentTernaryTensor> = HashMap::with_capacity(tensors.len());
274        let mut uploads: Vec<(u64, Vec<u8>)> = Vec::with_capacity(tensors.len());
275        let mut cursor: u64 = 0;
276        let (mut max_in, mut max_out) = (0u32, 0u32);
277        for &(key, n_in, n_out, blob) in tensors {
278            let count = n_in.checked_mul(n_out)?;
279            if count == 0 {
280                return None;
281            }
282            let (scale, mut packed) = rebake_ternary_blob_to_2bit(blob, count);
283            if packed.is_empty() {
284                return None;
285            }
286            // trit_words is array<u32> → pad the packed bytes to a 4-byte multiple.
287            while packed.len() % 4 != 0 {
288                packed.push(0);
289            }
290            let gpu_offset = align_up_u64(cursor, TERNARY_RESIDENT_ALIGN);
291            let packed_len = packed.len() as u32;
292            map.insert(
293                key,
294                ResidentTernaryTensor {
295                    gpu_offset,
296                    packed_len,
297                    n_in: n_in as u32,
298                    n_out: n_out as u32,
299                    scale,
300                },
301            );
302            uploads.push((gpu_offset, packed));
303            cursor = gpu_offset + packed_len as u64;
304            max_in = max_in.max(n_in as u32);
305            max_out = max_out.max(n_out as u32);
306        }
307        let resident_bytes = cursor.max(4);
308
309        // 2. Persistent pipeline (auto layout) — built ONCE.
310        let shader = device.create_shader_module(etalon::ShaderModuleDescriptor {
311            label: Some("ternary_ffn_2bit"),
312            source: etalon::ShaderSource::Wgsl(TERNARY_GEMM_2BIT_WGSL.into()),
313        });
314        let pipeline = device.create_compute_pipeline(&etalon::ComputePipelineDescriptor {
315            label: Some("ternary_ffn_resident_pipeline"),
316            layout: None,
317            module: &shader,
318            entry_point: Some("ternary_gemm"),
319            compilation_options: Default::default(),
320            cache: None,
321        });
322
323        // 3. Resident weight buffer + reusable IO buffers.
324        let weights = device.create_buffer(&etalon::BufferDescriptor {
325            label: Some("TernaryFfnResidentWeights"),
326            size: resident_bytes,
327            usage: etalon::BufferUsages::STORAGE | etalon::BufferUsages::COPY_DST,
328            mapped_at_creation: false,
329        });
330        for (off, bytes) in &uploads {
331            queue.write_buffer(&weights, *off, bytes);
332        }
333        let act_buf = device.create_buffer(&etalon::BufferDescriptor {
334            label: Some("TernaryFfnAct"),
335            size: (max_in as u64 * 4).max(4),
336            usage: etalon::BufferUsages::STORAGE | etalon::BufferUsages::COPY_DST,
337            mapped_at_creation: false,
338        });
339        let out_buf = device.create_buffer(&etalon::BufferDescriptor {
340            label: Some("TernaryFfnOut"),
341            size: (max_out as u64 * 4).max(4),
342            usage: etalon::BufferUsages::STORAGE | etalon::BufferUsages::COPY_SRC,
343            mapped_at_creation: false,
344        });
345        let params_buf = device.create_buffer(&etalon::BufferDescriptor {
346            label: Some("TernaryFfnParams"),
347            size: 32,
348            usage: etalon::BufferUsages::UNIFORM | etalon::BufferUsages::COPY_DST,
349            mapped_at_creation: false,
350        });
351        let staging = device.create_buffer(&etalon::BufferDescriptor {
352            label: Some("TernaryFfnStaging"),
353            size: (max_out as u64 * 4).max(4),
354            usage: etalon::BufferUsages::COPY_DST | etalon::BufferUsages::MAP_READ,
355            mapped_at_creation: false,
356        });
357
358        Some(Self {
359            pipeline,
360            weights,
361            act_buf,
362            out_buf,
363            params_buf,
364            staging,
365            map,
366            max_in,
367            max_out,
368            resident_bytes,
369        })
370    }
371
372    /// Whether a tensor with `key` is resident (and its shape, for the dispatch guard).
373    pub fn contains(&self, key: u64) -> bool {
374        self.map.contains_key(&key)
375    }
376
377    /// Total resident weight bytes (the VRAM footprint of the ternary FFN).
378    pub fn resident_bytes(&self) -> u64 {
379        self.resident_bytes
380    }
381
382    /// Number of resident ternary tensors.
383    pub fn len(&self) -> usize {
384        self.map.len()
385    }
386
387    /// `true` when empty.
388    pub fn is_empty(&self) -> bool {
389        self.map.is_empty()
390    }
391
392    /// Run `out[..n_out] = scale · W·act` for the resident tensor `key` (batch 1 = decode GEMV).
393    /// Returns `false` (caller falls back) if the key is absent, the shape mismatches the resident
394    /// metadata, the IO bounds are exceeded, or the GPU readback fails — fail-closed, never garbage.
395    pub fn gemv(
396        &self,
397        device: &etalon::Device,
398        queue: &etalon::Queue,
399        key: u64,
400        input: &[f32],
401        out: &mut [f32],
402        n_in: usize,
403        n_out: usize,
404    ) -> bool {
405        let t = match self.map.get(&key) {
406            Some(t) => *t,
407            None => return false,
408        };
409        if t.n_in as usize != n_in
410            || t.n_out as usize != n_out
411            || n_in > input.len()
412            || n_out > out.len()
413            || n_in as u32 > self.max_in
414            || n_out as u32 > self.max_out
415        {
416            return false;
417        }
418
419        queue.write_buffer(&self.act_buf, 0, bytemuck::cast_slice(&input[..n_in]));
420        let params = ternary_params_bytes(n_in as u32, n_out as u32, 1, t.scale);
421        queue.write_buffer(&self.params_buf, 0, &params);
422
423        let weight_binding = etalon::BindingResource::Buffer(etalon::BufferBinding {
424            buffer: &self.weights,
425            offset: t.gpu_offset,
426            size: std::num::NonZeroU64::new(t.packed_len as u64),
427        });
428        let bind = device.create_bind_group(&etalon::BindGroupDescriptor {
429            label: Some("ternary_ffn_resident_bind"),
430            layout: &self.pipeline.get_bind_group_layout(0),
431            entries: &[
432                etalon::BindGroupEntry {
433                    binding: 0,
434                    resource: self.act_buf.as_entire_binding(),
435                },
436                etalon::BindGroupEntry {
437                    binding: 1,
438                    resource: weight_binding,
439                },
440                etalon::BindGroupEntry {
441                    binding: 2,
442                    resource: self.params_buf.as_entire_binding(),
443                },
444                etalon::BindGroupEntry {
445                    binding: 3,
446                    resource: self.out_buf.as_entire_binding(),
447                },
448            ],
449        });
450
451        let mut encoder = device.create_command_encoder(&etalon::CommandEncoderDescriptor {
452            label: Some("ternary_ffn_enc"),
453        });
454        {
455            let mut pass = encoder.begin_compute_pass(&etalon::ComputePassDescriptor {
456                label: Some("ternary_ffn_pass"),
457                timestamp_writes: None,
458            });
459            pass.set_pipeline(&self.pipeline);
460            pass.set_bind_group(0, &bind, &[]);
461            pass.dispatch_workgroups((n_out as u32).div_ceil(64).max(1), 1, 1);
462        }
463        let out_bytes = (n_out * 4) as etalon::BufferAddress;
464        encoder.copy_buffer_to_buffer(&self.out_buf, 0, &self.staging, 0, out_bytes);
465        queue.submit(Some(encoder.finish()));
466
467        let slice = self.staging.slice(..out_bytes);
468        let (tx, rx) = std::sync::mpsc::channel();
469        slice.map_async(etalon::MapMode::Read, move |r| {
470            let _ = tx.send(r);
471        });
472        let _ = device.poll(etalon::PollType::wait_indefinitely());
473        match rx.recv() {
474            Ok(Ok(())) => {}
475            _ => return false,
476        }
477        let data = slice
478            .get_mapped_range()
479            .expect("wgpu buffer map_range failed");
480        out[..n_out].copy_from_slice(&bytemuck::cast_slice(&data)[..n_out]);
481        drop(data);
482        self.staging.unmap();
483        true
484    }
485}
486
487#[cfg(test)]
488mod tests {
489    use super::*;
490    use crate::ternary::{pack_trits, ternary_gemm_cpu};
491
492    fn try_gpu() -> Option<(etalon::Device, etalon::Queue)> {
493        let instance = etalon::Instance::default();
494        let adapter = pollster::block_on(instance.request_adapter(&etalon::RequestAdapterOptions {
495            power_preference: etalon::PowerPreference::HighPerformance,
496            ..Default::default()
497        }))
498        .ok()?;
499        pollster::block_on(adapter.request_device(&etalon::DeviceDescriptor::default())).ok()
500    }
501
502    /// ON-DEVICE PARITY: `ternary_gemm.wgsl` on a real GPU == `ternary_gemm_cpu`. Skips cleanly with
503    /// no adapter (headless CI); runs for real on a machine with a GPU.
504    #[test]
505    fn ternary_gemm_gpu_matches_cpu_oracle() {
506        let Some((device, queue)) = try_gpu() else {
507            eprintln!("ternary_gemm_gpu: no wgpu adapter — skipping on-device parity");
508            return;
509        };
510
511        // (n_out=5 × n_in=7), 3 batch rows; deterministic trits + activations.
512        let (n_in, n_out, n_batch) = (7usize, 5usize, 3usize);
513        let scale = 0.37_f32;
514        let trits: Vec<i8> = (0..n_in * n_out).map(|k| (k % 3) as i8 - 1).collect();
515        let packed = pack_trits(&trits);
516        let act: Vec<f32> = (0..n_in * n_batch)
517            .map(|j| (j as f32) * 0.25 - 1.5)
518            .collect();
519
520        let gpu = ternary_gemm_gpu(&device, &queue, &act, &packed, scale, n_in, n_out, n_batch);
521
522        let mut cpu = vec![0.0f32; n_batch * n_out];
523        ternary_gemm_cpu(&act, &packed, scale, n_in, n_out, n_batch, 0, 0, &mut cpu);
524
525        assert_eq!(gpu.len(), cpu.len());
526        for i in 0..cpu.len() {
527            assert!(
528                (gpu[i] - cpu[i]).abs() < 1e-4,
529                "elem {i}: gpu {} vs cpu {}",
530                gpu[i],
531                cpu[i]
532            );
533        }
534        eprintln!(
535            "ternary_gemm_gpu: on-device parity OK ({} elems)",
536            cpu.len()
537        );
538    }
539
540    /// ON-DEVICE PARITY (2-bit branchless): `ternary_gemm_2bit.wgsl` == `ternary_gemm_cpu_2bit`.
541    #[test]
542    fn ternary_gemm_gpu_2bit_matches_cpu_oracle() {
543        use crate::ternary::{pack_trits_2bit, ternary_gemm_cpu_2bit};
544        let Some((device, queue)) = try_gpu() else {
545            eprintln!("ternary_gemm_gpu_2bit: no wgpu adapter — skipping");
546            return;
547        };
548        let (n_in, n_out, n_batch) = (7usize, 5usize, 3usize);
549        let scale = 0.37_f32;
550        let trits: Vec<i8> = (0..n_in * n_out).map(|k| (k % 3) as i8 - 1).collect();
551        let packed = pack_trits_2bit(&trits);
552        let act: Vec<f32> = (0..n_in * n_batch)
553            .map(|j| (j as f32) * 0.25 - 1.5)
554            .collect();
555
556        let gpu =
557            ternary_gemm_gpu_2bit(&device, &queue, &act, &packed, scale, n_in, n_out, n_batch);
558        let mut cpu = vec![0.0f32; n_batch * n_out];
559        ternary_gemm_cpu_2bit(&act, &packed, scale, n_in, n_out, n_batch, 0, 0, &mut cpu);
560        for i in 0..cpu.len() {
561            assert!(
562                (gpu[i] - cpu[i]).abs() < 1e-4,
563                "elem {i}: gpu {} vs cpu {}",
564                gpu[i],
565                cpu[i]
566            );
567        }
568        eprintln!(
569            "ternary_gemm_gpu_2bit: on-device parity OK ({} elems)",
570            cpu.len()
571        );
572    }
573
574    /// Build a base-3 ternary blob (`[scale f32][5-trits/byte]`) from explicit trits + scale.
575    fn base3_blob(scale: f32, trits: &[i8]) -> Vec<u8> {
576        let mut b = scale.to_le_bytes().to_vec();
577        b.extend_from_slice(&pack_trits(trits));
578        b
579    }
580
581    /// A1b inc 2b ON-DEVICE GATE: the resident 2-bit dispatcher (persistent pipeline + ONE resident
582    /// weight buffer + 256-aligned sub-range bindings, keyed by P64 blob offset) reproduces the
583    /// base-3 CPU oracle byte-for-byte — for TWO differently-shaped tensors at distinct keys, proving
584    /// the per-key lookup, sub-range binding, rebake, and GEMV are all correct. Skips with no GPU.
585    #[test]
586    fn ternary_ffn_resident_matches_cpu_oracle() {
587        let Some((device, queue)) = try_gpu() else {
588            eprintln!("ternary_ffn_resident: no wgpu adapter — skipping");
589            return;
590        };
591        // Two FFN-shaped tensors with DIFFERENT shapes + scales + distinct keys (blob offsets).
592        let a = (0x1000u64, 64usize, 40usize, 0.30f32);
593        let b = (0x2000u64, 96usize, 24usize, 0.11f32);
594        let trits_a: Vec<i8> = (0..a.1 * a.2).map(|k| (k % 3) as i8 - 1).collect();
595        let trits_b: Vec<i8> = (0..b.1 * b.2)
596            .map(|k| ((k * 7 + 2) % 3) as i8 - 1)
597            .collect();
598        let blob_a = base3_blob(a.3, &trits_a);
599        let blob_b = base3_blob(b.3, &trits_b);
600
601        let resident = TernaryFfnResident::build(
602            &device,
603            &queue,
604            &[(a.0, a.1, a.2, &blob_a), (b.0, b.1, b.2, &blob_b)],
605        )
606        .expect("build resident ternary set");
607        assert_eq!(resident.len(), 2);
608        assert!(resident.contains(a.0) && resident.contains(b.0));
609
610        for (key, n_in, n_out, scale, blob) in
611            [(a.0, a.1, a.2, a.3, &blob_a), (b.0, b.1, b.2, b.3, &blob_b)]
612        {
613            let act: Vec<f32> = (0..n_in).map(|j| (j as f32) * 0.25 - 1.0).collect();
614            let mut gpu = vec![0f32; n_out];
615            assert!(
616                resident.gemv(&device, &queue, key, &act, &mut gpu, n_in, n_out),
617                "resident gemv must succeed for key {key:#x}"
618            );
619            let mut cpu = vec![0f32; n_out];
620            ternary_gemm_cpu(&act, &blob[4..], scale, n_in, n_out, 1, 0, 0, &mut cpu);
621            for i in 0..n_out {
622                assert!(
623                    (gpu[i] - cpu[i]).abs() < 1e-4,
624                    "key {key:#x} row {i}: gpu {} vs cpu {}",
625                    gpu[i],
626                    cpu[i]
627                );
628            }
629        }
630        // fail-closed (false, never garbage): an absent key, and a present key with the WRONG shape.
631        let mut tmp = vec![0f32; 64];
632        assert!(
633            !resident.gemv(&device, &queue, 0xDEAD, &[0.0; 64], &mut tmp, 64, 40),
634            "absent key must fail-closed"
635        );
636        assert!(
637            !resident.gemv(&device, &queue, a.0, &[0.0; 64], &mut tmp, 64, 64),
638            "present key with mismatched n_out must fail-closed (a is 64x40, not 64x64)"
639        );
640        eprintln!(
641            "ternary_ffn_resident: on-device parity OK (2 tensors, {} resident bytes)",
642            resident.resident_bytes()
643        );
644    }
645
646    /// INDICATIVE A/B timing: base-3 branchy vs 2-bit branchless on a large GEMV. Wall-clock incl.
647    /// per-dispatch submit/readback overhead — a *relative* signal, not a rigorous TPS number
648    /// (real measurement needs timestamp queries inside the fused FFN loop). Skips with no GPU.
649    #[test]
650    fn ternary_gemm_2bit_vs_base3_indicative_timing() {
651        use crate::ternary::{pack_trits, pack_trits_2bit};
652        use std::time::Instant;
653        let Some((device, queue)) = try_gpu() else {
654            eprintln!("ternary timing: no wgpu adapter — skipping");
655            return;
656        };
657        let (n_in, n_out, n_batch) = (4096usize, 4096usize, 1usize); // decode-shape GEMV
658        let scale = 0.05_f32;
659        let trits: Vec<i8> = (0..n_in * n_out)
660            .map(|k| ((k * 7 + 1) % 3) as i8 - 1)
661            .collect();
662        let p3 = pack_trits(&trits);
663        let p2 = pack_trits_2bit(&trits);
664        let act: Vec<f32> = (0..n_in).map(|j| (j as f32 % 13.0) * 0.1 - 0.6).collect();
665        let iters = 60;
666
667        // warmup
668        let _ = ternary_gemm_gpu(&device, &queue, &act, &p3, scale, n_in, n_out, n_batch);
669        let _ = ternary_gemm_gpu_2bit(&device, &queue, &act, &p2, scale, n_in, n_out, n_batch);
670
671        let t0 = Instant::now();
672        for _ in 0..iters {
673            let _ = ternary_gemm_gpu(&device, &queue, &act, &p3, scale, n_in, n_out, n_batch);
674        }
675        let base3 = t0.elapsed().as_secs_f64() / iters as f64 * 1e3;
676
677        let t1 = Instant::now();
678        for _ in 0..iters {
679            let _ = ternary_gemm_gpu_2bit(&device, &queue, &act, &p2, scale, n_in, n_out, n_batch);
680        }
681        let bit2 = t1.elapsed().as_secs_f64() / iters as f64 * 1e3;
682
683        eprintln!(
684            "ternary GEMV {}x{} (indicative, incl. overhead): base-3 branchy {:.3} ms/iter | 2-bit branchless {:.3} ms/iter | speedup {:.2}x",
685            n_out, n_in, base3, bit2, base3 / bit2.max(1e-9)
686        );
687    }
688
689    /// F16 GEMV baseline shader (same bindings/params as the ternary kernels).
690    const F16_GEMV_WGSL: &str = include_str!("../shaders/f16_gemv.wgsl");
691
692    /// Persistent-pipeline, batched-dispatch GEMV timing — fixes the per-call rebuild flaw of the
693    /// indicative test. Pipeline + buffers are created ONCE; `K` dispatches are encoded per submit
694    /// and `S` submits are timed, so per-dispatch time is GPU-execution-dominated (submit/alloc
695    /// overhead amortized away). Returns ms per dispatch.
696    fn bench_kernel(
697        device: &etalon::Device,
698        queue: &etalon::Queue,
699        wgsl: &str,
700        entry: &str,
701        weight_bytes: &[u8],
702        n_in: usize,
703        n_out: usize,
704    ) -> f64 {
705        use std::time::Instant;
706        let shader = device.create_shader_module(etalon::ShaderModuleDescriptor {
707            label: Some("bench"),
708            source: etalon::ShaderSource::Wgsl(wgsl.into()),
709        });
710        let pipeline = device.create_compute_pipeline(&etalon::ComputePipelineDescriptor {
711            label: Some("bench_pipeline"),
712            layout: None,
713            module: &shader,
714            entry_point: Some(entry),
715            compilation_options: Default::default(),
716            cache: None,
717        });
718        let mk = |contents: &[u8], usage: etalon::BufferUsages| {
719            let b = device.create_buffer(&etalon::BufferDescriptor {
720                label: None,
721                size: contents.len().max(4) as u64,
722                usage,
723                mapped_at_creation: false,
724            });
725            if !contents.is_empty() {
726                queue.write_buffer(&b, 0, contents);
727            }
728            b
729        };
730        let act = mk(
731            bytemuck::cast_slice(&vec![0.1f32; n_in]),
732            etalon::BufferUsages::STORAGE | etalon::BufferUsages::COPY_DST,
733        );
734        let mut w = weight_bytes.to_vec();
735        while w.len() % 4 != 0 || w.is_empty() {
736            w.push(0);
737        }
738        let wbuf = mk(
739            &w,
740            etalon::BufferUsages::STORAGE | etalon::BufferUsages::COPY_DST,
741        );
742        let params = super::ternary_params_bytes(n_in as u32, n_out as u32, 1, 1.0);
743        let pbuf = mk(
744            &params,
745            etalon::BufferUsages::UNIFORM | etalon::BufferUsages::COPY_DST,
746        );
747        let obuf = device.create_buffer(&etalon::BufferDescriptor {
748            label: None,
749            size: (n_out * 4).max(4) as u64,
750            usage: etalon::BufferUsages::STORAGE,
751            mapped_at_creation: false,
752        });
753        let bind = device.create_bind_group(&etalon::BindGroupDescriptor {
754            label: None,
755            layout: &pipeline.get_bind_group_layout(0),
756            entries: &[
757                etalon::BindGroupEntry {
758                    binding: 0,
759                    resource: act.as_entire_binding(),
760                },
761                etalon::BindGroupEntry {
762                    binding: 1,
763                    resource: wbuf.as_entire_binding(),
764                },
765                etalon::BindGroupEntry {
766                    binding: 2,
767                    resource: pbuf.as_entire_binding(),
768                },
769                etalon::BindGroupEntry {
770                    binding: 3,
771                    resource: obuf.as_entire_binding(),
772                },
773            ],
774        });
775        let wg_x = (n_out as u32).div_ceil(64).max(1);
776        let (k, s) = (32u32, 8u32);
777        let submit_batch = || {
778            let mut enc =
779                device.create_command_encoder(&etalon::CommandEncoderDescriptor { label: None });
780            {
781                let mut pass = enc.begin_compute_pass(&etalon::ComputePassDescriptor {
782                    label: None,
783                    timestamp_writes: None,
784                });
785                pass.set_pipeline(&pipeline);
786                pass.set_bind_group(0, &bind, &[]);
787                for _ in 0..k {
788                    pass.dispatch_workgroups(wg_x, 1, 1);
789                }
790            }
791            queue.submit(Some(enc.finish()));
792            let _ = device.poll(etalon::PollType::wait_indefinitely());
793        };
794        submit_batch(); // warmup
795        let t0 = Instant::now();
796        for _ in 0..s {
797            submit_batch();
798        }
799        t0.elapsed().as_secs_f64() * 1e3 / (k * s) as f64
800    }
801
802    /// A2000 KERNEL BENCHMARK (real numbers): F16 vs base-3 ternary vs 2-bit branchless ternary, at a
803    /// decode-shape GEMV, with persistent pipeline + buffer reuse. Skips with no GPU.
804    #[test]
805    fn ternary_kernel_benchmark() {
806        use crate::ternary::{pack_trits, pack_trits_2bit};
807        let Some((device, queue)) = try_gpu() else {
808            eprintln!("ternary_kernel_benchmark: no wgpu adapter — skipping");
809            return;
810        };
811        let (n_in, n_out) = (4096usize, 4096usize); // decode GEMV (batch 1)
812        let trits: Vec<i8> = (0..n_in * n_out)
813            .map(|k| ((k * 7 + 1) % 3) as i8 - 1)
814            .collect();
815        let p3 = pack_trits(&trits);
816        let p2 = pack_trits_2bit(&trits);
817        let f16 = vec![0u8; n_in * n_out * 2]; // f16 weights (size = bandwidth; values irrelevant to timing)
818
819        let f16ms = bench_kernel(
820            &device,
821            &queue,
822            F16_GEMV_WGSL,
823            "f16_gemv",
824            &f16,
825            n_in,
826            n_out,
827        );
828        let base3 = bench_kernel(
829            &device,
830            &queue,
831            TERNARY_GEMM_WGSL,
832            "ternary_gemm",
833            &p3,
834            n_in,
835            n_out,
836        );
837        let bit2 = bench_kernel(
838            &device,
839            &queue,
840            TERNARY_GEMM_2BIT_WGSL,
841            "ternary_gemm",
842            &p2,
843            n_in,
844            n_out,
845        );
846
847        eprintln!(
848            "── A2000 GEMV {}x{} batch=1 (persistent pipeline, {}MB/{}KB/{}KB weights) ──",
849            n_out,
850            n_in,
851            f16.len() / (1 << 20),
852            p3.len() >> 10,
853            p2.len() >> 10
854        );
855        eprintln!("  F16 baseline        : {:.4} ms/dispatch", f16ms);
856        eprintln!(
857            "  ternary base-3      : {:.4} ms/dispatch  ({:.2}x vs F16)",
858            base3,
859            f16ms / base3.max(1e-12)
860        );
861        eprintln!(
862            "  ternary 2-bit branchless: {:.4} ms/dispatch  ({:.2}x vs F16, {:.2}x vs base-3)",
863            bit2,
864            f16ms / bit2.max(1e-12),
865            base3 / bit2.max(1e-12)
866        );
867    }
868}