1#[cfg(feature = "gpu-runtime")]
17use crate::device_benchmark::CircuitKind;
18#[cfg(feature = "gpu-runtime")]
19use crate::platform::hetero_dispatch::{
20 select_precision, HeterogeneousDispatcher, HostCapabilities, PowerThermalBudget, Precision,
21 ZeroCopyStrategy,
22};
23
24use super::backend::BackendId;
25#[cfg(feature = "gpu-runtime")]
26use super::backend::KernelPanel;
27use super::kernel_class::KernelClass;
28#[cfg(feature = "gpu-runtime")]
29use super::matrix::{probe_class_matrix, ClassMatrix};
30
31#[derive(Debug, Clone, Copy, PartialEq)]
38pub struct Plan {
39 pub backend: BackendId,
41 #[cfg(feature = "gpu-runtime")]
43 pub circuit_kind: CircuitKind,
44 #[cfg(feature = "gpu-runtime")]
46 pub precision: Precision,
47 pub tiles: u32,
50 #[cfg(feature = "gpu-runtime")]
52 pub zero_copy: ZeroCopyStrategy,
53}
54
55impl Plan {
56 pub fn is_cpu(self) -> bool {
58 #[cfg(feature = "gpu-runtime")]
59 {
60 self.backend == BackendId::CPU || self.circuit_kind == CircuitKind::Cpu
61 }
62 #[cfg(not(feature = "gpu-runtime"))]
63 {
64 self.backend == BackendId::CPU
65 }
66 }
67
68 fn cpu() -> Self {
69 Self {
70 backend: BackendId::CPU,
71 #[cfg(feature = "gpu-runtime")]
72 circuit_kind: CircuitKind::Cpu,
73 #[cfg(feature = "gpu-runtime")]
74 precision: Precision::F32,
75 tiles: 1,
76 #[cfg(feature = "gpu-runtime")]
77 zero_copy: ZeroCopyStrategy::MmapDirect,
78 }
79 }
80}
81
82pub struct ComputePolicy {
86 #[cfg(feature = "gpu-runtime")]
87 matrix: ClassMatrix,
88 #[cfg(feature = "gpu-runtime")]
89 budget: PowerThermalBudget,
90 #[cfg(feature = "gpu-runtime")]
91 host: HostCapabilities,
92}
93
94impl ComputePolicy {
95 #[cfg(not(feature = "gpu-runtime"))]
97 pub fn cpu_only() -> Self {
98 Self {}
99 }
100
101 #[cfg(feature = "gpu-runtime")]
104 pub fn from_class_matrix(
105 matrix: ClassMatrix,
106 budget: PowerThermalBudget,
107 host: HostCapabilities,
108 ) -> Self {
109 Self {
110 matrix,
111 budget,
112 host,
113 }
114 }
115
116 #[cfg(feature = "gpu-runtime")]
119 pub fn probe(
120 registry: &super::backend::BackendRegistry,
121 panel: &KernelPanel,
122 budget: PowerThermalBudget,
123 host: HostCapabilities,
124 ) -> Self {
125 Self::from_class_matrix(probe_class_matrix(registry, panel), budget, host)
126 }
127
128 #[cfg(feature = "gpu-runtime")]
130 pub fn matrix(&self) -> &ClassMatrix {
131 &self.matrix
132 }
133
134 pub fn select(&self, class: KernelClass, problem_bytes: u64) -> Plan {
141 #[cfg(not(feature = "gpu-runtime"))]
142 {
143 let _ = (class, problem_bytes);
144 return Plan::cpu();
145 }
146 #[cfg(feature = "gpu-runtime")]
147 {
148 let param_count = (problem_bytes / 4).max(1);
151 let precision = select_precision(param_count, &self.budget);
152
153 let best = self
154 .matrix
155 .rows(class)
156 .iter()
157 .find(|row| row.kind != CircuitKind::Npu);
158 let cpu_ms = self
159 .matrix
160 .rows(class)
161 .iter()
162 .find(|r| r.kind == CircuitKind::Cpu)
163 .map(|r| r.ms_per_gemv);
164
165 let choose_gpu = self.host.gpu_available
166 && match best {
167 None => false, Some(b) if b.kind == CircuitKind::Cpu => false, Some(b) => {
170 if class.is_typically_gpu_amenable() {
174 true
175 } else if let Some(cms) = cpu_ms {
176 b.ms_per_gemv < cms * 0.95
177 } else {
178 true
179 }
180 }
181 };
182
183 if let (true, Some(b)) = (choose_gpu, best) {
184 let dispatcher = HeterogeneousDispatcher::new(self.host);
185 let tiles = dispatcher.gpu_tiles(problem_bytes);
186 let zero_copy = ZeroCopyStrategy::StagingUpload;
187 Plan {
188 backend: BackendId(backend_id_for(&b.backend)),
189 circuit_kind: b.kind,
190 precision,
191 tiles,
192 zero_copy,
193 }
194 } else {
195 Plan::cpu()
196 }
197 }
198 }
199}
200
201#[cfg(feature = "gpu-runtime")]
205fn backend_id_for(backend: &str) -> &'static str {
206 match backend {
207 "native" => "cpu",
208 _ => "wgpu",
209 }
210}
211
212#[cfg(all(test, feature = "gpu-runtime"))]
213mod tests {
214 use super::*;
215 use crate::device_benchmark::CircuitBench;
216
217 const GIB: u64 = 1 << 30;
218
219 fn roomy_host() -> (PowerThermalBudget, HostCapabilities) {
220 (
221 PowerThermalBudget {
222 vram_budget_bytes: 16 * GIB,
223 power_budget_mw: 60_000,
224 thermal_headroom_c: 30.0,
225 },
226 HostCapabilities {
227 gpu_available: true,
228 vram_available: 8 * GIB,
229 npu_available: false,
230 cpu_threads: 16,
231 },
232 )
233 }
234
235 fn row(kind: CircuitKind, backend: &str, ms: f64) -> CircuitBench {
236 CircuitBench {
237 label: format!("{backend} circuit"),
238 kind,
239 backend: backend.to_string(),
240 ms_per_gemv: ms,
241 gflops: 0.0,
242 upload_gbps: if kind == CircuitKind::Cpu {
243 f64::INFINITY
244 } else {
245 5.0
246 },
247 rel_score: 1.0,
248 decode_proxy_tok_s: None,
249 }
250 }
251
252 fn synth_matrix() -> ClassMatrix {
254 let mut per_class = Vec::new();
255 for class in KernelClass::ALL {
256 let rows = match class {
257 KernelClass::DenseLinear => vec![
258 row(CircuitKind::DiscreteGpu, "Vulkan", 0.4),
259 row(CircuitKind::Cpu, "native", 20.0),
260 ],
261 KernelClass::Divergent => vec![row(CircuitKind::Cpu, "native", 3.0)],
262 _ => vec![row(CircuitKind::Cpu, "native", 5.0)],
263 };
264 per_class.push((class, rows));
265 }
266 ClassMatrix::from_per_class(per_class)
267 }
268
269 #[test]
270 fn selects_measured_gpu_winner_for_dense_linear() {
271 let (budget, host) = roomy_host();
272 let policy = ComputePolicy::from_class_matrix(synth_matrix(), budget, host);
273 let plan = policy.select(KernelClass::DenseLinear, 64 * 1024 * 1024);
274 assert_eq!(plan.backend, BackendId::WGPU);
275 assert_eq!(plan.circuit_kind, CircuitKind::DiscreteGpu);
276 assert_eq!(plan.zero_copy, ZeroCopyStrategy::StagingUpload); assert!(!plan.is_cpu());
278 }
279
280 #[test]
281 fn an_integrated_gpu_uses_portable_staging() {
282 let (budget, host) = roomy_host();
283 let matrix = ClassMatrix::from_per_class(vec![(
284 KernelClass::DenseLinear,
285 vec![
286 row(CircuitKind::IntegratedGpu, "Dx12", 0.4),
287 row(CircuitKind::Cpu, "native", 20.0),
288 ],
289 )]);
290 let policy = ComputePolicy::from_class_matrix(matrix, budget, host);
291 let plan = policy.select(KernelClass::DenseLinear, 64 * 1024 * 1024);
292 assert_eq!(plan.circuit_kind, CircuitKind::IntegratedGpu);
293 assert_eq!(plan.zero_copy, ZeroCopyStrategy::StagingUpload);
294 }
295
296 #[test]
297 fn an_npu_measurement_cannot_be_dispatched_as_wgpu() {
298 let (budget, mut host) = roomy_host();
299 host.npu_available = true;
300 let matrix = ClassMatrix::from_per_class(vec![(
301 KernelClass::DenseLinear,
302 vec![
303 row(CircuitKind::Npu, "OpenVINO", 0.1),
304 row(CircuitKind::Cpu, "native", 20.0),
305 ],
306 )]);
307 let policy = ComputePolicy::from_class_matrix(matrix, budget, host);
308 assert!(policy
309 .select(KernelClass::DenseLinear, 64 * 1024 * 1024)
310 .is_cpu());
311 }
312
313 #[test]
314 fn an_unroutable_npu_row_does_not_hide_a_gpu_candidate() {
315 let (budget, host) = roomy_host();
316 let matrix = ClassMatrix::from_per_class(vec![(
317 KernelClass::DenseLinear,
318 vec![
319 row(CircuitKind::Npu, "OpenVINO", 0.1),
320 row(CircuitKind::IntegratedGpu, "Dx12", 0.4),
321 row(CircuitKind::Cpu, "native", 20.0),
322 ],
323 )]);
324 let policy = ComputePolicy::from_class_matrix(matrix, budget, host);
325 assert_eq!(
326 policy
327 .select(KernelClass::DenseLinear, 64 * 1024 * 1024)
328 .circuit_kind,
329 CircuitKind::IntegratedGpu
330 );
331 }
332
333 #[test]
334 fn falls_back_to_cpu_when_only_cpu_probed() {
335 let (budget, host) = roomy_host();
336 let policy = ComputePolicy::from_class_matrix(synth_matrix(), budget, host);
337 let plan = policy.select(KernelClass::Divergent, 1 << 20);
338 assert!(plan.is_cpu());
339 assert_eq!(plan.backend, BackendId::CPU);
340 assert_eq!(plan.tiles, 1);
341 }
342
343 #[test]
344 fn cached_gpu_measurements_do_not_override_current_unavailability() {
345 let (budget, mut host) = roomy_host();
346 host.gpu_available = false;
347 host.npu_available = true;
348 let policy = ComputePolicy::from_class_matrix(synth_matrix(), budget, host);
349 assert!(policy
350 .select(KernelClass::DenseLinear, 64 * 1024 * 1024)
351 .is_cpu());
352 }
353
354 #[test]
355 fn unprobed_class_is_cpu_not_a_panic() {
356 let (budget, host) = roomy_host();
358 let empty = ClassMatrix::from_per_class(vec![(KernelClass::Fft, Vec::new())]);
359 let policy = ComputePolicy::from_class_matrix(empty, budget, host);
360 assert!(policy.select(KernelClass::Fft, 1 << 20).is_cpu());
361 assert!(policy.select(KernelClass::AllPairs, 1 << 20).is_cpu()); }
363
364 #[test]
365 fn vram_pressure_tiles_instead_of_failing() {
366 let budget = PowerThermalBudget {
367 vram_budget_bytes: 16 * GIB,
368 power_budget_mw: 60_000,
369 thermal_headroom_c: 30.0,
370 };
371 let host = HostCapabilities {
372 gpu_available: true,
373 vram_available: 2 * GIB,
374 npu_available: false,
375 cpu_threads: 8,
376 };
377 let policy = ComputePolicy::from_class_matrix(synth_matrix(), budget, host);
378 let plan = policy.select(KernelClass::DenseLinear, 5 * GIB);
380 assert!(!plan.is_cpu());
381 assert_eq!(plan.tiles, 3);
382 }
383}
384
385#[cfg(all(test, not(feature = "gpu-runtime")))]
386mod tests {
387 use super::*;
388
389 #[test]
390 fn cpu_only_policy_always_selects_cpu() {
391 let policy = ComputePolicy::cpu_only();
392 for class in KernelClass::ALL {
393 let plan = policy.select(class, 1 << 30);
394 assert!(plan.is_cpu());
395 assert_eq!(plan.backend, BackendId::CPU);
396 assert_eq!(plan.tiles, 1);
397 }
398 }
399}