qualia_core_db/inference/thermal_telemetry.rs
1//! W7 — real GPU thermal/power telemetry + a detect-and-recommend thermal governor.
2//!
3//! `orchestrator::ThermalGovernor` previously had only a *simulated* implementation
4//! (`CalculusThermalGovernor`, a Newton-cooling ODE). This module adds a REAL one backed by NVIDIA
5//! NVML (via the optional `nvml` feature / `nvml-wrapper`), reading actual GPU temperature and power.
6//!
7//! **Policy: detect + recommend, never silently escalate.** The governor maps live temperature to
8//! `ThermalStatus` and exposes a *recommended* TDP cap; it does NOT change the GPU's power limit.
9//! Enforcement is a separate, explicit, privileged opt-in
10//! (`NvmlThermalGovernor::apply_power_limit_w`) that nothing here calls automatically — a human/admin
11//! policy must invoke it. This is the in-repo form of the human-centric-control norm for the off-grid
12//! / constrained-power target: the machine reports and recommends; the human decides.
13//!
14//! When the `nvml` feature is off, or NVML/the driver is absent (non-NVIDIA host), telemetry degrades
15//! cleanly: `sample_gpu_thermal()` returns `None` and `open_thermal_governor()` returns the
16//! `NullThermalGovernor` (always `Cool`), so callers never need to know whether NVML is present.
17
18#![cfg(not(target_arch = "wasm32"))]
19
20use crate::inference::orchestrator::{NullThermalGovernor, ThermalGovernor, ThermalStatus};
21use std::sync::atomic::{AtomicBool, Ordering};
22
23// ── Auto-cap mode switch (user option; default ON) ────────────────────────────
24// When ON *and* a real NVML governor is active (i.e. there is an NVIDIA card), the governor
25// automatically applies its recommended TDP cap after temperature has been sustained-Critical for
26// `CRITICAL_DWELL` checks, and restores the original limit once the card cools back to Cool. This is a
27// protective default the principal (Timothy) chose for the off-grid / constrained-power target — it
28// only ever REDUCES power under sustained heat (never raises it or acts against the user), it is fully
29// reversible, and it is user-toggleable, so it respects the human-centric-control norm. When OFF the
30// governor is recommend-only (logs an advisory cap, changes nothing). On non-NVIDIA hosts there is no
31// NVML governor, so this flag has no effect. Applying a cap needs admin/root; if that fails the
32// governor degrades to recommend-only and logs it.
33//
34// MODE SWITCH (UI-reachable, like spec-decode): env `QUALIA_LLM_GPU_AUTO_CAP=0` (off) / `=1` (on) at
35// launch; `set_gpu_auto_cap(bool)` at runtime (the UI/host calls this); `gpu_auto_cap_enabled()` to
36// read the effective mode. The env var, when set, overrides the runtime flag in both directions.
37static GPU_AUTO_CAP: AtomicBool = AtomicBool::new(true);
38
39/// Number of consecutive Critical checks before auto-cap engages (hysteresis — ignores transient
40/// spikes). At the decode-loop tick cadence (~every 64 tokens) this is several seconds of sustained
41/// Critical.
42pub const CRITICAL_DWELL: u32 = 3;
43
44/// Enable/disable automatic TDP capping under sustained Critical temperature (`QUALIA_LLM_GPU_AUTO_CAP`).
45/// Runtime mode switch — the desktop UI / host calls this. Default ON (effective only when an NVIDIA
46/// card + NVML are present).
47#[inline]
48pub fn set_gpu_auto_cap(on: bool) {
49 GPU_AUTO_CAP.store(on, Ordering::Relaxed);
50}
51
52/// Whether automatic TDP capping is enabled (env var wins if set, else the runtime flag). Read this to
53/// reflect the current mode in a UI.
54#[inline]
55pub fn gpu_auto_cap_enabled() -> bool {
56 match std::env::var("QUALIA_LLM_GPU_AUTO_CAP").ok().as_deref() {
57 Some("0") | Some("false") => false,
58 Some("1") | Some("true") => true,
59 _ => GPU_AUTO_CAP.load(Ordering::Relaxed),
60 }
61}
62
63/// What the governor should do this check — the pure decision, separated from the NVML side effects so
64/// it is unit-testable. `critical_streak` counts consecutive Critical checks INCLUDING this one;
65/// `capped` is whether we currently hold an applied cap.
66#[derive(Debug, Clone, Copy, PartialEq, Eq)]
67pub enum ThermalAction {
68 /// Nothing to do.
69 Nothing,
70 /// Advisory only — log the recommended cap; do not change hardware.
71 Recommend,
72 /// Auto-cap is on and Critical has been sustained past the dwell — apply the recommended cap now.
73 ApplyCap,
74 /// Cooled back to Cool after we had applied a cap — restore the original limit.
75 Restore,
76}
77
78/// Pure enforcement decision (see [`ThermalAction`]). Hysteresis: only caps on **sustained** Critical,
79/// holds the cap through Warm, and restores only once back to Cool.
80pub fn decide_thermal_action(
81 status: ThermalStatus,
82 critical_streak: u32,
83 auto_cap: bool,
84 capped: bool,
85) -> ThermalAction {
86 match status {
87 ThermalStatus::Critical => {
88 if auto_cap && !capped && critical_streak >= CRITICAL_DWELL {
89 ThermalAction::ApplyCap
90 } else if !capped {
91 ThermalAction::Recommend
92 } else {
93 ThermalAction::Nothing // already capped — hold
94 }
95 }
96 ThermalStatus::Warm => {
97 if capped {
98 ThermalAction::Nothing // hold the cap through Warm (hysteresis)
99 } else {
100 ThermalAction::Recommend
101 }
102 }
103 ThermalStatus::Cool => {
104 if capped {
105 ThermalAction::Restore
106 } else {
107 ThermalAction::Nothing
108 }
109 }
110 }
111}
112
113/// GPU temperature bands (°C) — the same thresholds the simulated `CalculusThermalGovernor` uses, so
114/// the real and simulated governors classify identically.
115pub const WARM_THRESHOLD_C: u32 = 65;
116pub const CRITICAL_THRESHOLD_C: u32 = 85;
117
118/// Map a GPU temperature to the project `ThermalStatus`.
119#[inline]
120pub fn status_for_temp(temp_c: u32) -> ThermalStatus {
121 if temp_c > CRITICAL_THRESHOLD_C {
122 ThermalStatus::Critical
123 } else if temp_c > WARM_THRESHOLD_C {
124 ThermalStatus::Warm
125 } else {
126 ThermalStatus::Cool
127 }
128}
129
130/// A point-in-time GPU thermal/power reading.
131#[derive(Debug, Clone, Copy)]
132pub struct GpuThermalSample {
133 /// GPU core temperature (°C).
134 pub temp_c: u32,
135 /// Instantaneous board power draw (W).
136 pub power_w: f64,
137 /// Currently enforced power limit / TDP (W).
138 pub power_limit_w: f64,
139 /// Min settable power limit (W) from the driver constraints.
140 pub power_min_w: f64,
141 /// Max settable power limit (W) from the driver constraints.
142 pub power_max_w: f64,
143 /// Thermal classification derived from `temp_c`.
144 pub status: ThermalStatus,
145}
146
147impl GpuThermalSample {
148 /// A *recommended* TDP cap (W) when the GPU is running hot — advisory only; nothing here applies
149 /// it. `None` when `Cool` (no action recommended). Clamped to the driver's `[min, max]` limits.
150 pub fn recommended_power_cap_w(&self) -> Option<f64> {
151 let frac = match self.status {
152 ThermalStatus::Cool => return None,
153 ThermalStatus::Warm => 0.90,
154 ThermalStatus::Critical => 0.80,
155 };
156 let lo = self.power_min_w.max(1.0);
157 let hi = self.power_max_w.max(lo);
158 Some((self.power_limit_w * frac).clamp(lo, hi))
159 }
160}
161
162/// Read one GPU thermal/power sample. `None` when the `nvml` feature is off or NVML/the driver is
163/// unavailable (non-NVIDIA host, no driver). Never panics. Suitable for a UI telemetry poll.
164pub fn sample_gpu_thermal() -> Option<GpuThermalSample> {
165 #[cfg(feature = "nvml")]
166 {
167 nvml_impl::one_shot_sample().ok()
168 }
169 #[cfg(not(feature = "nvml"))]
170 {
171 None
172 }
173}
174
175/// Construct the best available thermal governor: the real NVML one when the `nvml` feature is on and
176/// NVML initializes, else the `NullThermalGovernor` (always `Cool`). Mirrors `open_storage` /
177/// `open_platform_filter` — callers don't branch on platform/feature.
178pub fn open_thermal_governor() -> Box<dyn ThermalGovernor> {
179 #[cfg(feature = "nvml")]
180 {
181 match nvml_impl::NvmlThermalGovernor::new() {
182 Ok(g) => {
183 log::info!("W7|thermal|NVML governor active: {}", g.device_label());
184 return Box::new(g);
185 }
186 Err(e) => {
187 log::info!("W7|thermal|NVML unavailable ({e}) — NullThermalGovernor (always Cool)");
188 }
189 }
190 }
191 Box::new(NullThermalGovernor)
192}
193
194/// Periodic thermal check for the decode hot path — samples a RESIDENT NVML governor and, per the
195/// auto-cap user option, enforces (sustained Critical → apply the recommended cap; cooled → restore)
196/// or recommends. Cheap (an NVML read); call every N decode tokens. No-op when the `nvml` feature is
197/// off or NVML/the driver is unavailable. The resident governor keeps the auto-cap hysteresis state
198/// (Critical streak, capped flag, saved limit) across ticks.
199pub fn thermal_tick() {
200 #[cfg(feature = "nvml")]
201 {
202 use std::sync::OnceLock;
203 static RESIDENT: OnceLock<Option<nvml_impl::NvmlThermalGovernor>> = OnceLock::new();
204 if let Some(g) = RESIDENT.get_or_init(|| nvml_impl::NvmlThermalGovernor::new().ok()) {
205 g.tick();
206 }
207 }
208}
209
210/// The real NVML-backed governor. Construct directly (`NvmlThermalGovernor::new()`) when you need the
211/// privileged `apply_power_limit_w` enforcement path or repeated `sample()`s; use
212/// `open_thermal_governor()` for the trait-object detect+recommend role.
213#[cfg(feature = "nvml")]
214pub use nvml_impl::NvmlThermalGovernor;
215
216#[cfg(feature = "nvml")]
217mod nvml_impl {
218 use super::*;
219 use nvml_wrapper::enum_wrappers::device::TemperatureSensor;
220 use nvml_wrapper::Nvml;
221 use std::sync::atomic::AtomicU32;
222
223 /// A real NVML-backed thermal governor over GPU 0. Detect + recommend only; the sole enforcement
224 /// path (`apply_power_limit_w`) is explicit, privileged, and never called automatically.
225 pub struct NvmlThermalGovernor {
226 nvml: Nvml,
227 index: u32,
228 label: String,
229 /// Consecutive Critical checks (hysteresis for auto-cap).
230 critical_streak: AtomicU32,
231 /// Whether we currently hold an applied cap.
232 capped: AtomicBool,
233 /// The enforced power limit (mW) captured before we capped, restored on cool-down.
234 saved_limit_mw: AtomicU32,
235 }
236
237 impl NvmlThermalGovernor {
238 /// Creates `GpuThermalSample` with the given state.
239 pub fn new() -> Result<Self, String> {
240 let nvml = Nvml::init().map_err(|e| format!("NVML init: {e}"))?;
241 let index = 0u32;
242 let label = nvml
243 .device_by_index(index)
244 .and_then(|d| d.name())
245 .unwrap_or_else(|_| "NVIDIA GPU".to_string());
246 Ok(Self {
247 nvml,
248 index,
249 label,
250 critical_streak: AtomicU32::new(0),
251 capped: AtomicBool::new(false),
252 saved_limit_mw: AtomicU32::new(0),
253 })
254 }
255
256 /// Device label.
257 pub fn device_label(&self) -> &str {
258 &self.label
259 }
260
261 /// Read one live sample from the resident NVML handle.
262 pub fn sample(&self) -> Result<GpuThermalSample, String> {
263 let dev = self
264 .nvml
265 .device_by_index(self.index)
266 .map_err(|e| format!("device: {e}"))?;
267 let temp_c = dev
268 .temperature(TemperatureSensor::Gpu)
269 .map_err(|e| format!("temperature: {e}"))?;
270 let power_w = dev
271 .power_usage()
272 .map(|mw| mw as f64 / 1000.0)
273 .unwrap_or(0.0);
274 let power_limit_w = dev
275 .enforced_power_limit()
276 .map(|mw| mw as f64 / 1000.0)
277 .unwrap_or(0.0);
278 let (power_min_w, power_max_w) = dev
279 .power_management_limit_constraints()
280 .map(|c| (c.min_limit as f64 / 1000.0, c.max_limit as f64 / 1000.0))
281 .unwrap_or((0.0, power_limit_w));
282 Ok(GpuThermalSample {
283 temp_c,
284 power_w,
285 power_limit_w,
286 power_min_w,
287 power_max_w,
288 status: status_for_temp(temp_c),
289 })
290 }
291
292 /// EXPLICIT, PRIVILEGED opt-in enforcement — set the GPU power limit (W). This is the ONLY
293 /// method that mutates hardware state; it is NEVER called automatically by the governor and
294 /// requires admin/root (returns `Err` otherwise). A human/admin policy must invoke it (no
295 /// silent escalation).
296 pub fn apply_power_limit_w(&self, watts: f64) -> Result<(), String> {
297 self.apply_power_limit_mw((watts * 1000.0).round().max(0.0) as u32)
298 }
299
300 fn apply_power_limit_mw(&self, mw: u32) -> Result<(), String> {
301 let mut dev = self
302 .nvml
303 .device_by_index(self.index)
304 .map_err(|e| format!("device: {e}"))?;
305 dev.set_power_management_limit(mw)
306 .map_err(|e| format!("set_power_management_limit (needs admin): {e}"))
307 }
308
309 /// One thermal check: sample, decide (with hysteresis), then enforce (auto-cap on + sustained
310 /// Critical → apply the recommended cap; cooled after a cap → restore) or recommend-only.
311 /// Call periodically (e.g. every N decode tokens). Never panics; a failed NVML apply degrades
312 /// to recommend-only + a log line.
313 pub fn tick(&self) {
314 let s = match self.sample() {
315 Ok(s) => s,
316 Err(_) => return,
317 };
318 let streak = if matches!(s.status, ThermalStatus::Critical) {
319 self.critical_streak.fetch_add(1, Ordering::Relaxed) + 1
320 } else {
321 self.critical_streak.store(0, Ordering::Relaxed);
322 0
323 };
324 let auto_cap = gpu_auto_cap_enabled();
325 let capped = self.capped.load(Ordering::Relaxed);
326 match decide_thermal_action(s.status, streak, auto_cap, capped) {
327 ThermalAction::ApplyCap => {
328 if let Some(cap) = s.recommended_power_cap_w() {
329 self.saved_limit_mw
330 .store((s.power_limit_w * 1000.0).round() as u32, Ordering::Relaxed);
331 match self.apply_power_limit_w(cap) {
332 Ok(()) => {
333 self.capped.store(true, Ordering::Relaxed);
334 log::warn!(
335 "W7|thermal|AUTO-CAP: {} sustained {}\u{b0}C — capped {:.0}W \u{2192} {:.0}W (reversible; auto_cap on)",
336 self.label, s.temp_c, s.power_limit_w, cap
337 );
338 }
339 Err(e) => log::warn!(
340 "W7|thermal|auto-cap FAILED (needs admin) at {}\u{b0}C — RECOMMEND {:.0}W only: {e}",
341 s.temp_c, cap
342 ),
343 }
344 }
345 }
346 ThermalAction::Restore => {
347 let saved = self.saved_limit_mw.load(Ordering::Relaxed);
348 if saved > 0 {
349 match self.apply_power_limit_mw(saved) {
350 Ok(()) => log::info!(
351 "W7|thermal|cooled to {}\u{b0}C — restored power limit {:.0}W",
352 s.temp_c,
353 saved as f64 / 1000.0
354 ),
355 Err(e) => log::warn!("W7|thermal|restore power limit FAILED: {e}"),
356 }
357 }
358 self.capped.store(false, Ordering::Relaxed);
359 }
360 ThermalAction::Recommend => {
361 if let Some(cap) = s.recommended_power_cap_w() {
362 log::warn!(
363 "W7|thermal|{:?} {}\u{b0}C {:.0}W/{:.0}W — RECOMMEND cap {:.0}W (advisory)",
364 s.status, s.temp_c, s.power_w, s.power_limit_w, cap
365 );
366 }
367 }
368 ThermalAction::Nothing => {}
369 }
370 }
371 }
372
373 impl ThermalGovernor for NvmlThermalGovernor {
374 fn get_thermal_state(&self) -> ThermalStatus {
375 self.sample()
376 .map(|s| s.status)
377 .unwrap_or(ThermalStatus::Cool)
378 }
379
380 fn adjust_policy(&self, _status: ThermalStatus) {
381 // Full check: recommend, or auto-cap/restore when the user option is on (see `tick`).
382 self.tick();
383 }
384 }
385
386 /// One-shot read (init NVML, sample, drop). For repeated polling prefer a resident
387 /// `NvmlThermalGovernor` to avoid re-initializing NVML each call.
388 pub fn one_shot_sample() -> Result<GpuThermalSample, String> {
389 NvmlThermalGovernor::new()?.sample()
390 }
391}
392
393#[cfg(test)]
394mod tests {
395 use super::*;
396
397 #[test]
398 fn temp_bands_classify_correctly() {
399 assert_eq!(status_for_temp(30), ThermalStatus::Cool);
400 assert_eq!(status_for_temp(WARM_THRESHOLD_C), ThermalStatus::Cool); // boundary is inclusive-Cool
401 assert_eq!(status_for_temp(WARM_THRESHOLD_C + 1), ThermalStatus::Warm);
402 assert_eq!(status_for_temp(80), ThermalStatus::Warm);
403 assert_eq!(status_for_temp(CRITICAL_THRESHOLD_C), ThermalStatus::Warm);
404 assert_eq!(
405 status_for_temp(CRITICAL_THRESHOLD_C + 1),
406 ThermalStatus::Critical
407 );
408 assert_eq!(status_for_temp(95), ThermalStatus::Critical);
409 }
410
411 #[test]
412 fn recommended_cap_is_advisory_and_clamped() {
413 let mk = |temp, limit, min, max| GpuThermalSample {
414 temp_c: temp,
415 power_w: 0.0,
416 power_limit_w: limit,
417 power_min_w: min,
418 power_max_w: max,
419 status: status_for_temp(temp),
420 };
421 // Cool → no recommendation.
422 assert!(mk(40, 70.0, 40.0, 90.0).recommended_power_cap_w().is_none());
423 // Warm → 90% of the enforced limit.
424 let warm = mk(70, 70.0, 40.0, 90.0).recommended_power_cap_w().unwrap();
425 assert!((warm - 63.0).abs() < 1e-6, "warm cap {warm}");
426 // Critical → 80%, clamped up to the driver minimum.
427 let crit = mk(90, 70.0, 60.0, 90.0).recommended_power_cap_w().unwrap();
428 assert!(
429 (crit - 60.0).abs() < 1e-6,
430 "critical cap clamped to min: {crit}"
431 );
432 }
433
434 #[test]
435 fn auto_cap_decision_hysteresis() {
436 use ThermalAction::*;
437 use ThermalStatus::*;
438 // Cool, nothing held → nothing; Cool while capped → restore.
439 assert_eq!(decide_thermal_action(Cool, 0, true, false), Nothing);
440 assert_eq!(decide_thermal_action(Cool, 0, true, true), Restore);
441 // Warm recommends when uncapped; holds the cap when capped (hysteresis).
442 assert_eq!(decide_thermal_action(Warm, 0, true, false), Recommend);
443 assert_eq!(decide_thermal_action(Warm, 0, true, true), Nothing);
444 // Critical below the dwell → recommend only, even with auto-cap on.
445 assert_eq!(
446 decide_thermal_action(Critical, CRITICAL_DWELL - 1, true, false),
447 Recommend
448 );
449 // Critical sustained past the dwell with auto-cap ON → apply the cap.
450 assert_eq!(
451 decide_thermal_action(Critical, CRITICAL_DWELL, true, false),
452 ApplyCap
453 );
454 // Same, but auto-cap OFF → recommend only (the user option gates enforcement).
455 assert_eq!(
456 decide_thermal_action(Critical, CRITICAL_DWELL, false, false),
457 Recommend
458 );
459 // Already capped and still Critical → hold (don't re-apply).
460 assert_eq!(
461 decide_thermal_action(Critical, CRITICAL_DWELL + 5, true, true),
462 Nothing
463 );
464 }
465
466 #[test]
467 fn auto_cap_toggle_roundtrips() {
468 set_gpu_auto_cap(false);
469 assert!(!gpu_auto_cap_enabled() || std::env::var("QUALIA_LLM_GPU_AUTO_CAP").is_ok());
470 set_gpu_auto_cap(true);
471 assert!(
472 gpu_auto_cap_enabled()
473 || std::env::var("QUALIA_LLM_GPU_AUTO_CAP").as_deref() == Ok("0")
474 );
475 }
476
477 /// Hardware smoke test — only compiled with `--features nvml` and only meaningful on an NVIDIA
478 /// host. Reads one live sample and sanity-checks it; skips gracefully if NVML is absent.
479 #[cfg(feature = "nvml")]
480 #[test]
481 fn nvml_live_sample_smoke() {
482 match sample_gpu_thermal() {
483 Some(s) => {
484 println!(
485 "[w7] {:?} temp={}\u{b0}C power={:.1}W limit={:.1}W [{:.1}..{:.1}]W rec={:?}",
486 s.status,
487 s.temp_c,
488 s.power_w,
489 s.power_limit_w,
490 s.power_min_w,
491 s.power_max_w,
492 s.recommended_power_cap_w()
493 );
494 assert!(
495 s.temp_c > 0 && s.temp_c < 130,
496 "implausible GPU temp {}",
497 s.temp_c
498 );
499 assert_eq!(s.status, status_for_temp(s.temp_c));
500 }
501 None => eprintln!("[w7] NVML unavailable on this host — smoke test skipped"),
502 }
503 }
504}