Skip to main content

qualia_core_db/inference/inference_bench/
prepared_settings.rs

1//! Immutable numerical settings for a single native execution scope.
2//! Cold capture; scalar getters borrow TLS without allocation. Worker threads
3//! must explicitly enter a clone before numerical work. Provider identity and
4//! live semantic inputs are separate requirements, never inferred from settings.
5use sha2::{Digest, Sha256};
6use std::{cell::RefCell, marker::PhantomData, rc::Rc};
7use std::{env::VarError, ffi::OsString};
8
9const ENV_KEYS: &[&str] = &[
10    "QUALIA_HYBRID_ROUTE",
11    "QUALIA_GRAPH_FORCE",
12    "QUALIA_LLM_CUDA_TC_REDUCED",
13    "QUALIA_LLM_PROFILE_DECODE",
14    "QUALIA_CUDA_CONTEXT",
15    "QUALIA_DIRECTML",
16    "QUALIA_LLM_CUDA_DECODE",
17    "QUALIA_LLM_CUDA_TC_DECODE",
18    "QUALIA_LLM_CUDA_TC_PREWARM",
19    "QUALIA_LLM_DEBUG_DECODE",
20    "QUALIA_LLM_DIRECTML",
21    "QUALIA_LLM_DUAL_MR",
22    "QUALIA_LLM_FFN_MR",
23    "QUALIA_LLM_FFN_WARP",
24    "QUALIA_LLM_MULTIROW",
25    "QUALIA_LLM_TRIPLE_QKV",
26    "QUALIA_LLM_WARP_GEMV",
27];
28const MAX_ENV_VALUE_BYTES: usize = 32768;
29const MAX_SETTINGS_BYTES: usize = 256 * 1024;
30
31#[derive(Debug, Clone, Copy)]
32pub struct SettingsCapacity;
33
34#[derive(Clone, Debug)]
35pub struct PreparedInferenceSettings {
36    #[cfg(feature = "cuda")]
37    cached_cuda_tuning: [u8; 32],
38    pub(crate) inference_mode: crate::inference_modes::InferenceMode,
39    environment: Vec<(&'static str, Option<OsString>)>,
40    pub(crate) decode_budget_override: u32,
41    pub(crate) inference_timeout_ms: u64,
42    pub(crate) gpu_topk_enabled: bool,
43    pub(crate) ternary_ffn_enabled: bool,
44    pub(crate) attention_preproject_enabled: bool,
45    pub(crate) attention_o_fuse_enabled: bool,
46    pub(crate) forge_attention_enabled: bool,
47    pub(crate) resident_weights_enabled: bool,
48    pub(crate) resident_decode_enabled: bool,
49    pub(crate) resident_prefill_enabled: bool,
50    pub(crate) spec_decode_enabled: bool,
51    pub(crate) kv_int8_enabled: bool,
52    pub(crate) kv_dict_enabled: bool,
53    pub(crate) sampler_config: Option<crate::sampler::SamplerConfig>,
54    pub(crate) ffn_fusion_enabled: bool,
55    pub(crate) ffn_f16_enabled: bool,
56    pub(crate) coop_gemv_enabled: bool,
57    pub(crate) coopmat_gemm_enabled: bool,
58    pub(crate) cpu_attention_enabled: bool,
59    pub(crate) prefix_cache_enabled: bool,
60    pub(crate) kv_pool_budget_mb: u32,
61    pub(crate) prefill_chunk_size_override: u32,
62    pub(crate) prompt_precision_mode: String,
63    pub(crate) stage_telemetry_enabled: bool,
64}
65
66thread_local! {
67    static PREPARED: RefCell<Option<PreparedInferenceSettings>> = const { RefCell::new(None) };
68}
69
70pub(crate) fn read<T>(get: impl FnOnce(&PreparedInferenceSettings) -> T) -> Option<T> {
71    PREPARED.with(|slot| slot.borrow().as_ref().map(get))
72}
73
74impl PreparedInferenceSettings {
75    /// Capture actual effective getter values. Existing scopes stay immutable.
76    pub fn capture() -> Result<Self, SettingsCapacity> {
77        let mut environment = Vec::with_capacity(ENV_KEYS.len());
78        let mut charged = 0usize;
79        for &key in ENV_KEYS {
80            let value = env_var_os(key);
81            let bytes = value
82                .as_ref()
83                .map_or(0, |value| value.as_encoded_bytes().len());
84            charged = charged.checked_add(bytes).ok_or(SettingsCapacity)?;
85            if bytes > MAX_ENV_VALUE_BYTES || charged > MAX_SETTINGS_BYTES {
86                return Err(SettingsCapacity);
87            }
88            environment.push((key, value));
89        }
90        let prompt_precision_mode = super::toggles::prompt_precision_mode();
91        if prompt_precision_mode.len() > MAX_ENV_VALUE_BYTES {
92            return Err(SettingsCapacity);
93        }
94        Ok(Self {
95            #[cfg(feature = "cuda")]
96            cached_cuda_tuning: crate::inference::cuda_lane::cached_tuning_digest(),
97            inference_mode: crate::inference_modes::atomic_inference_mode(),
98            environment,
99            decode_budget_override: super::toggles::decode_budget_override(),
100            inference_timeout_ms: super::toggles::inference_timeout_ms(),
101            gpu_topk_enabled: super::toggles::gpu_topk_enabled(),
102            ternary_ffn_enabled: super::toggles::ternary_ffn_enabled(),
103            attention_preproject_enabled: super::toggles::attention_preproject_enabled(),
104            attention_o_fuse_enabled: super::toggles::attention_o_fuse_enabled(),
105            forge_attention_enabled: super::toggles::forge_attention_enabled(),
106            resident_weights_enabled: super::toggles::resident_weights_enabled(),
107            resident_decode_enabled: super::toggles::resident_decode_enabled(),
108            resident_prefill_enabled: super::toggles::resident_prefill_enabled(),
109            spec_decode_enabled: super::toggles::spec_decode_enabled(),
110            kv_int8_enabled: super::toggles::kv_int8_enabled(),
111            kv_dict_enabled: super::toggles::kv_dict_enabled(),
112            sampler_config: super::toggles::sampler_config(),
113            ffn_fusion_enabled: super::toggles::ffn_fusion_enabled(),
114            ffn_f16_enabled: super::toggles::ffn_f16_enabled(),
115            coop_gemv_enabled: super::toggles::coop_gemv_enabled(),
116            coopmat_gemm_enabled: super::toggles::coopmat_gemm_enabled(),
117            cpu_attention_enabled: super::toggles::cpu_attention_enabled(),
118            prefix_cache_enabled: super::toggles::prefix_cache_enabled(),
119            kv_pool_budget_mb: super::toggles::kv_pool_budget_mb(),
120            prefill_chunk_size_override: super::toggles::prefill_chunk_size_override(),
121            prompt_precision_mode,
122            stage_telemetry_enabled: super::toggles::stage_telemetry_enabled(),
123        })
124    }
125
126    /// Canonical numeric-bit digest, including the consumed sampling seed.
127    pub fn digest(&self) -> [u8; 32] {
128        let mut hash = Sha256::new();
129        hash.update(b"qualia:prepared-inference-settings:v1\0");
130        #[cfg(feature = "cuda")]
131        hash.update(self.cached_cuda_tuning);
132        hash.update([self.inference_mode as u8]);
133        for (key, value) in &self.environment {
134            hash.update((key.len() as u64).to_le_bytes());
135            hash.update(key.as_bytes());
136            if let Some(value) = value {
137                hash.update([1]);
138                hash.update((value.as_encoded_bytes().len() as u64).to_le_bytes());
139                hash.update(value.as_encoded_bytes());
140            } else {
141                hash.update([0]);
142            }
143        }
144        hash.update(b"decode_budget_override\0");
145        hash.update(self.decode_budget_override.to_le_bytes());
146        hash.update(b"inference_timeout_ms\0");
147        hash.update(self.inference_timeout_ms.to_le_bytes());
148        hash.update(b"gpu_topk_enabled\0");
149        hash.update([u8::from(self.gpu_topk_enabled)]);
150        hash.update(b"ternary_ffn_enabled\0");
151        hash.update([u8::from(self.ternary_ffn_enabled)]);
152        hash.update(b"attention_preproject_enabled\0");
153        hash.update([u8::from(self.attention_preproject_enabled)]);
154        hash.update(b"attention_o_fuse_enabled\0");
155        hash.update([u8::from(self.attention_o_fuse_enabled)]);
156        hash.update(b"forge_attention_enabled\0");
157        hash.update([u8::from(self.forge_attention_enabled)]);
158        hash.update(b"resident_weights_enabled\0");
159        hash.update([u8::from(self.resident_weights_enabled)]);
160        hash.update(b"resident_decode_enabled\0");
161        hash.update([u8::from(self.resident_decode_enabled)]);
162        hash.update(b"resident_prefill_enabled\0");
163        hash.update([u8::from(self.resident_prefill_enabled)]);
164        hash.update(b"spec_decode_enabled\0");
165        hash.update([u8::from(self.spec_decode_enabled)]);
166        hash.update(b"kv_int8_enabled\0");
167        hash.update([u8::from(self.kv_int8_enabled)]);
168        hash.update(b"kv_dict_enabled\0");
169        hash.update([u8::from(self.kv_dict_enabled)]);
170        hash.update(b"sampler_config\0");
171        if let Some(config) = self.sampler_config {
172            hash.update([1]);
173            for value in [
174                config.temperature,
175                config.top_p,
176                config.repeat_penalty,
177                config.freq_penalty,
178                config.presence_penalty,
179            ] {
180                hash.update(value.to_bits().to_le_bytes());
181            }
182            hash.update(config.top_k.to_le_bytes());
183            hash.update(config.penalty_window.to_le_bytes());
184            hash.update(config.seed.to_le_bytes());
185        } else {
186            hash.update([0]);
187        }
188        hash.update(b"ffn_fusion_enabled\0");
189        hash.update([u8::from(self.ffn_fusion_enabled)]);
190        hash.update(b"ffn_f16_enabled\0");
191        hash.update([u8::from(self.ffn_f16_enabled)]);
192        hash.update(b"coop_gemv_enabled\0");
193        hash.update([u8::from(self.coop_gemv_enabled)]);
194        hash.update(b"coopmat_gemm_enabled\0");
195        hash.update([u8::from(self.coopmat_gemm_enabled)]);
196        hash.update(b"cpu_attention_enabled\0");
197        hash.update([u8::from(self.cpu_attention_enabled)]);
198        hash.update(b"prefix_cache_enabled\0");
199        hash.update([u8::from(self.prefix_cache_enabled)]);
200        hash.update(b"kv_pool_budget_mb\0");
201        hash.update(self.kv_pool_budget_mb.to_le_bytes());
202        hash.update(b"prefill_chunk_size_override\0");
203        hash.update(self.prefill_chunk_size_override.to_le_bytes());
204        hash.update(b"prompt_precision_mode\0");
205        hash.update((self.prompt_precision_mode.len() as u64).to_le_bytes());
206        hash.update(self.prompt_precision_mode.as_bytes());
207        hash.update(b"stage_telemetry_enabled\0");
208        hash.update([u8::from(self.stage_telemetry_enabled)]);
209        hash.finalize().into()
210    }
211
212    /// Runs only on the preparing thread. Nested scopes restore exact prior state.
213    pub fn enter(self) -> PreparedSettingsGuard {
214        let previous = PREPARED.with(|slot| slot.replace(Some(self)));
215        PreparedSettingsGuard {
216            previous,
217            _thread: PhantomData,
218        }
219    }
220}
221
222/// Thread-affine RAII scope; unwind restores the previous settings.
223pub struct PreparedSettingsGuard {
224    previous: Option<PreparedInferenceSettings>,
225    _thread: PhantomData<Rc<()>>,
226}
227impl Drop for PreparedSettingsGuard {
228    fn drop(&mut self) {
229        PREPARED.with(|slot| {
230            slot.replace(self.previous.take());
231        });
232    }
233}
234
235/// Frozen environment for enrolled numerical settings; unrelated keys preserve
236/// normal process behavior. Values retain exact native OS encoding.
237pub fn env_var_os(key: &str) -> Option<OsString> {
238    if let Some(value) = read(|settings| {
239        settings
240            .environment
241            .iter()
242            .find(|(name, _)| *name == key)
243            .map(|(_, value)| value.clone())
244    }) {
245        if let Some(value) = value {
246            return value;
247        }
248    }
249    std::env::var_os(key)
250}
251pub fn env_var(key: &str) -> Result<String, VarError> {
252    env_var_os(key)
253        .ok_or(VarError::NotPresent)?
254        .into_string()
255        .map_err(VarError::NotUnicode)
256}