Skip to main content

qualia_core_db/inference/
inference_gpu_profiler.rs

1//! Per-kernel GPU timing for the native LLM forward/decode path (W2 / D17).
2//!
3//! Wraps each LLM compute pass with `TIMESTAMP_QUERY` writes, resolves the query set,
4//! and accumulates GPU-internal nanoseconds per [`Phase`]. Gated by a runtime flag
5//! (`QUALIA_LLM_GPU_PROFILE=1` or [`set_enabled`]) so production decode pays nothing:
6//! when disabled, the `pass_writes_*` helpers return `None` and the pass is byte-identical
7//! to before (`timestamp_writes: None`).
8//!
9//! Requires the shared device to have negotiated `TIMESTAMP_QUERY`
10//! (see [`crate::gpu_context::SharedGpuContext::timestamps_supported`]); degrades to a
11//! no-op otherwise, so it is safe to call unconditionally from every dispatch site.
12//!
13//! **Honesty note:** the per-phase nanoseconds are GPU-internal (begin→end of the pass) and
14//! individually accurate. A *profiling* run serialises per-op readback, so the headline
15//! tok/s of a profiled run is perturbed — report the per-phase split, not the profiled tok/s.
16//!
17//! Not re-entrant: it uses one shared 2-slot query set, which the single LLM engine thread's
18//! per-op blocking readback serialises. That matches how decode actually runs.
19
20use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
21
22/// Distinct kernel families on the LLM forward/decode path.
23#[repr(usize)]
24#[derive(Debug, Clone, Copy, PartialEq, Eq)]
25pub enum Phase {
26    /// Quantized token-embedding lookup/dequant.
27    Embedding = 0,
28    /// Projection + FFN matmuls (incl. the ternary-FFN dispatch).
29    Gemm = 1,
30    /// Fused attention (Q·K, softmax, ·V).
31    Attention = 2,
32    /// Single-pass fused transformer-block shader.
33    FusedBlock = 3,
34    /// lm_head logits + top-k / argmax sampling.
35    OutputTopk = 4,
36}
37
38impl Phase {
39    pub const COUNT: usize = 5;
40    pub const ALL: [Phase; Self::COUNT] = [
41        Phase::Embedding,
42        Phase::Gemm,
43        Phase::Attention,
44        Phase::FusedBlock,
45        Phase::OutputTopk,
46    ];
47
48    #[inline]
49    /// Phase label.
50    pub fn label(self) -> &'static str {
51        match self {
52            Phase::Embedding => "embedding",
53            Phase::Gemm => "gemm",
54            Phase::Attention => "attention",
55            Phase::FusedBlock => "fused_block",
56            Phase::OutputTopk => "output_topk",
57        }
58    }
59}
60
61static ENABLED: AtomicBool = AtomicBool::new(false);
62static ACC_NS: [AtomicU64; Phase::COUNT] = [const { AtomicU64::new(0) }; Phase::COUNT];
63static ACC_CALLS: [AtomicU64; Phase::COUNT] = [const { AtomicU64::new(0) }; Phase::COUNT];
64
65/// Force GPU pass profiling on/off at runtime (bench-only; production leaves it off).
66#[inline]
67pub fn set_enabled(on: bool) {
68    ENABLED.store(on, Ordering::Relaxed);
69}
70
71/// One-shot `QUALIA_LLM_GPU_PROFILE` env opt-in (cached; no per-call allocation).
72fn env_opt_in() -> bool {
73    static ENV: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
74    *ENV.get_or_init(|| {
75        std::env::var("QUALIA_LLM_GPU_PROFILE")
76            .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
77            .unwrap_or(false)
78    })
79}
80
81/// True when profiling is requested AND the shared device negotiated `TIMESTAMP_QUERY`.
82#[inline]
83pub fn enabled() -> bool {
84    if !(ENABLED.load(Ordering::Relaxed) || env_opt_in()) {
85        return false;
86    }
87    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
88    {
89        crate::gpu_context::shared_gpu().timestamps_supported
90    }
91    #[cfg(not(all(not(target_arch = "wasm32"), feature = "gpu-runtime")))]
92    {
93        false
94    }
95}
96
97// ── Native timestamp resources + accumulation ─────────────────────────────────
98
99#[cfg(not(target_arch = "wasm32"))]
100fn resources() -> &'static etalon_wgsl_forge::numerical::profiling::TimestampResources {
101    static R: std::sync::OnceLock<etalon_wgsl_forge::numerical::profiling::TimestampResources> =
102        std::sync::OnceLock::new();
103    R.get_or_init(|| {
104        etalon_wgsl_forge::numerical::profiling::TimestampResources::new(
105            &crate::gpu_context::shared_gpu().device,
106        )
107    })
108}
109
110/// `timestamp_writes` for a single-pass kernel (writes both begin and end on this pass).
111/// Returns `None` (zero overhead) when profiling is off — drop straight into the descriptor.
112#[cfg(not(target_arch = "wasm32"))]
113pub fn pass_writes_both() -> Option<etalon::ComputePassTimestampWrites<'static>> {
114    if !enabled() {
115        return None;
116    }
117    Some(resources().pass_writes(true, true))
118}
119
120/// `timestamp_writes` for the FIRST pass of a multi-pass kernel (begin only).
121#[cfg(not(target_arch = "wasm32"))]
122pub fn pass_writes_begin() -> Option<etalon::ComputePassTimestampWrites<'static>> {
123    if !enabled() {
124        return None;
125    }
126    Some(resources().pass_writes(true, false))
127}
128
129/// `timestamp_writes` for the LAST pass of a multi-pass kernel (end only).
130#[cfg(not(target_arch = "wasm32"))]
131pub fn pass_writes_end() -> Option<etalon::ComputePassTimestampWrites<'static>> {
132    if !enabled() {
133        return None;
134    }
135    Some(resources().pass_writes(false, true))
136}
137
138/// Encode the query-set resolve into `encoder` (call after the pass, before `finish()`).
139/// No-op when profiling is off.
140#[cfg(not(target_arch = "wasm32"))]
141pub fn resolve(encoder: &mut etalon::CommandEncoder) {
142    if !enabled() {
143        return;
144    }
145    resources().resolve(encoder);
146}
147
148/// Read the resolved timestamps and add `end - begin` ns to `phase`'s accumulator.
149/// Call after the kernel's submit + device poll. No-op when profiling is off.
150#[cfg(not(target_arch = "wasm32"))]
151pub fn accumulate(phase: Phase) {
152    if !enabled() {
153        return;
154    }
155    let ctx = crate::gpu_context::shared_gpu();
156    if let Ok(ts) = resources().read_ticks(&ctx.device) {
157        if let Some(ns) = etalon_diagnostics::measurement::timestamp_nanoseconds(
158            ts[0],
159            ts[1],
160            ctx.timestamp_period_ns,
161        ) {
162            ACC_NS[phase as usize].fetch_add(ns, Ordering::Relaxed);
163            ACC_CALLS[phase as usize].fetch_add(1, Ordering::Relaxed);
164        }
165    }
166}
167
168#[cfg(target_arch = "wasm32")]
169/// Timestamp writes for a both-phase pass.
170pub fn pass_writes_both() -> Option<etalon::ComputePassTimestampWrites<'static>> {
171    None
172}
173#[cfg(target_arch = "wasm32")]
174/// Timestamp writes for a begin-phase pass.
175pub fn pass_writes_begin() -> Option<etalon::ComputePassTimestampWrites<'static>> {
176    None
177}
178#[cfg(target_arch = "wasm32")]
179/// Timestamp writes for an end-phase pass.
180pub fn pass_writes_end() -> Option<etalon::ComputePassTimestampWrites<'static>> {
181    None
182}
183#[cfg(target_arch = "wasm32")]
184/// Resolves timestamps on `encoder`.
185pub fn resolve(_encoder: &mut etalon::CommandEncoder) {}
186#[cfg(target_arch = "wasm32")]
187/// Accumulates `phase` timings.
188pub fn accumulate(_phase: Phase) {}
189
190// ── Snapshot / reset for the bench ────────────────────────────────────────────
191
192/// Accumulated GPU time for one phase across all dispatches since the last [`reset`].
193#[derive(Debug, Clone, Copy)]
194pub struct PhaseTiming {
195    pub phase: Phase,
196    pub total_ns: u64,
197    pub calls: u64,
198}
199
200impl PhaseTiming {
201    #[inline]
202    /// Timing in microseconds.
203    pub fn micros(&self) -> f64 {
204        self.total_ns as f64 / 1000.0
205    }
206}
207
208/// Snapshot all phase accumulators (for the bench JSON).
209pub fn snapshot() -> [PhaseTiming; Phase::COUNT] {
210    let mut out = [PhaseTiming {
211        phase: Phase::Embedding,
212        total_ns: 0,
213        calls: 0,
214    }; Phase::COUNT];
215    for (i, p) in Phase::ALL.iter().enumerate() {
216        out[i] = PhaseTiming {
217            phase: *p,
218            total_ns: ACC_NS[i].load(Ordering::Relaxed),
219            calls: ACC_CALLS[i].load(Ordering::Relaxed),
220        };
221    }
222    out
223}
224
225/// Zero all phase accumulators (call before a measured run).
226pub fn reset() {
227    for i in 0..Phase::COUNT {
228        ACC_NS[i].store(0, Ordering::Relaxed);
229        ACC_CALLS[i].store(0, Ordering::Relaxed);
230    }
231}
232
233/// True if any phase recorded GPU time since the last reset (test/diagnostic helper).
234pub fn any_recorded() -> bool {
235    (0..Phase::COUNT).any(|i| ACC_CALLS[i].load(Ordering::Relaxed) > 0)
236}
237
238#[cfg(all(test, not(target_arch = "wasm32")))]
239mod tests {
240    use super::*;
241
242    // Trivial compute kernel with enough arithmetic to register measurable GPU time.
243    const SELF_TEST_WGSL: &str = r#"
244@group(0) @binding(0) var<storage, read_write> data: array<u32>;
245@compute @workgroup_size(64)
246fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
247    let i = gid.x;
248    if (i < arrayLength(&data)) {
249        var acc = data[i];
250        for (var k: u32 = 0u; k < 4096u; k = k + 1u) {
251            acc = acc * 1664525u + 1013904223u;
252        }
253        data[i] = acc;
254    }
255}
256"#;
257
258    /// Proves the end-to-end timestamp path on real hardware: enable → wrap a real
259    /// compute pass → resolve → accumulate → assert non-zero GPU time. Named with `gpu`
260    /// so `--skip gpu` excludes it on adapters/CI without a timestamp-capable device.
261    #[test]
262    fn gpu_timestamp_self_test_records_nonzero() {
263        let ctx = crate::gpu_context::shared_gpu();
264        if !ctx.timestamps_supported {
265            eprintln!("SKIP gpu_timestamp_self_test: adapter has no TIMESTAMP_QUERY");
266            return;
267        }
268        let device = &ctx.device;
269        let queue = &ctx.queue;
270
271        set_enabled(true);
272        reset();
273
274        let n: u64 = 65_536;
275        let buf = device.create_buffer(&etalon::BufferDescriptor {
276            label: Some("prof-self-test-buf"),
277            size: n * 4,
278            usage: etalon::BufferUsages::STORAGE,
279            mapped_at_creation: false,
280        });
281        let shader = device.create_shader_module(etalon::ShaderModuleDescriptor {
282            label: Some("prof-self-test-shader"),
283            source: etalon::ShaderSource::Wgsl(SELF_TEST_WGSL.into()),
284        });
285        let pipeline = device.create_compute_pipeline(&etalon::ComputePipelineDescriptor {
286            label: Some("prof-self-test-pipe"),
287            layout: None,
288            module: &shader,
289            entry_point: Some("main"),
290            compilation_options: etalon::PipelineCompilationOptions::default(),
291            cache: None,
292        });
293        let bgl = pipeline.get_bind_group_layout(0);
294        let bind_group = device.create_bind_group(&etalon::BindGroupDescriptor {
295            label: Some("prof-self-test-bg"),
296            layout: &bgl,
297            entries: &[etalon::BindGroupEntry {
298                binding: 0,
299                resource: buf.as_entire_binding(),
300            }],
301        });
302
303        let mut encoder =
304            device.create_command_encoder(&etalon::CommandEncoderDescriptor { label: None });
305        {
306            let mut cpass = encoder.begin_compute_pass(&etalon::ComputePassDescriptor {
307                label: Some("prof-self-test-pass"),
308                timestamp_writes: pass_writes_both(),
309            });
310            cpass.set_pipeline(&pipeline);
311            cpass.set_bind_group(0, &bind_group, &[]);
312            cpass.dispatch_workgroups((n as u32 + 63) / 64, 1, 1);
313        }
314        resolve(&mut encoder);
315        queue.submit(Some(encoder.finish()));
316        accumulate(Phase::Gemm);
317
318        assert!(
319            any_recorded(),
320            "profiler recorded no timestamps on a TIMESTAMP_QUERY device"
321        );
322        let snap = snapshot();
323        let gemm = snap
324            .iter()
325            .find(|t| matches!(t.phase, Phase::Gemm))
326            .copied()
327            .expect("gemm phase present");
328        eprintln!(
329            "gpu_timestamp_self_test: gemm pass = {:.3} µs ({} ns, {} call)",
330            gemm.micros(),
331            gemm.total_ns,
332            gemm.calls
333        );
334        assert_eq!(gemm.calls, 1, "expected exactly one recorded pass");
335        assert!(
336            gemm.total_ns > 0,
337            "expected non-zero GPU time for a 4096-iter kernel"
338        );
339
340        set_enabled(false);
341    }
342
343    #[test]
344    fn disabled_profiler_is_noop() {
345        set_enabled(false);
346        reset();
347        // With profiling off, the pass-writes helpers must return None (zero overhead).
348        assert!(pass_writes_both().is_none());
349        assert!(pass_writes_begin().is_none());
350        assert!(pass_writes_end().is_none());
351        accumulate(Phase::Attention);
352        assert!(!any_recorded());
353    }
354}