qualia_core_db/inference/inference_bench/
prepared_settings.rs1use sha2::{Digest, Sha256};
6use std::{cell::RefCell, marker::PhantomData, rc::Rc};
7use std::{env::VarError, ffi::OsString};
8
9const ENV_KEYS: &[&str] = &[
10 "QUALIA_HYBRID_ROUTE",
11 "QUALIA_GRAPH_FORCE",
12 "QUALIA_LLM_CUDA_TC_REDUCED",
13 "QUALIA_LLM_PROFILE_DECODE",
14 "QUALIA_CUDA_CONTEXT",
15 "QUALIA_DIRECTML",
16 "QUALIA_LLM_CUDA_DECODE",
17 "QUALIA_LLM_CUDA_TC_DECODE",
18 "QUALIA_LLM_CUDA_TC_PREWARM",
19 "QUALIA_LLM_DEBUG_DECODE",
20 "QUALIA_LLM_DIRECTML",
21 "QUALIA_LLM_DUAL_MR",
22 "QUALIA_LLM_FFN_MR",
23 "QUALIA_LLM_FFN_WARP",
24 "QUALIA_LLM_MULTIROW",
25 "QUALIA_LLM_TRIPLE_QKV",
26 "QUALIA_LLM_WARP_GEMV",
27];
28const MAX_ENV_VALUE_BYTES: usize = 32768;
29const MAX_SETTINGS_BYTES: usize = 256 * 1024;
30
31#[derive(Debug, Clone, Copy)]
32pub struct SettingsCapacity;
33
34#[derive(Clone, Debug)]
35pub struct PreparedInferenceSettings {
36 #[cfg(feature = "cuda")]
37 cached_cuda_tuning: [u8; 32],
38 pub(crate) inference_mode: crate::inference_modes::InferenceMode,
39 environment: Vec<(&'static str, Option<OsString>)>,
40 pub(crate) decode_budget_override: u32,
41 pub(crate) inference_timeout_ms: u64,
42 pub(crate) gpu_topk_enabled: bool,
43 pub(crate) ternary_ffn_enabled: bool,
44 pub(crate) attention_preproject_enabled: bool,
45 pub(crate) attention_o_fuse_enabled: bool,
46 pub(crate) forge_attention_enabled: bool,
47 pub(crate) resident_weights_enabled: bool,
48 pub(crate) resident_decode_enabled: bool,
49 pub(crate) resident_prefill_enabled: bool,
50 pub(crate) spec_decode_enabled: bool,
51 pub(crate) kv_int8_enabled: bool,
52 pub(crate) kv_dict_enabled: bool,
53 pub(crate) sampler_config: Option<crate::sampler::SamplerConfig>,
54 pub(crate) ffn_fusion_enabled: bool,
55 pub(crate) ffn_f16_enabled: bool,
56 pub(crate) coop_gemv_enabled: bool,
57 pub(crate) coopmat_gemm_enabled: bool,
58 pub(crate) cpu_attention_enabled: bool,
59 pub(crate) prefix_cache_enabled: bool,
60 pub(crate) kv_pool_budget_mb: u32,
61 pub(crate) prefill_chunk_size_override: u32,
62 pub(crate) prompt_precision_mode: String,
63 pub(crate) stage_telemetry_enabled: bool,
64}
65
66thread_local! {
67 static PREPARED: RefCell<Option<PreparedInferenceSettings>> = const { RefCell::new(None) };
68}
69
70pub(crate) fn read<T>(get: impl FnOnce(&PreparedInferenceSettings) -> T) -> Option<T> {
71 PREPARED.with(|slot| slot.borrow().as_ref().map(get))
72}
73
74impl PreparedInferenceSettings {
75 pub fn capture() -> Result<Self, SettingsCapacity> {
77 let mut environment = Vec::with_capacity(ENV_KEYS.len());
78 let mut charged = 0usize;
79 for &key in ENV_KEYS {
80 let value = env_var_os(key);
81 let bytes = value
82 .as_ref()
83 .map_or(0, |value| value.as_encoded_bytes().len());
84 charged = charged.checked_add(bytes).ok_or(SettingsCapacity)?;
85 if bytes > MAX_ENV_VALUE_BYTES || charged > MAX_SETTINGS_BYTES {
86 return Err(SettingsCapacity);
87 }
88 environment.push((key, value));
89 }
90 let prompt_precision_mode = super::toggles::prompt_precision_mode();
91 if prompt_precision_mode.len() > MAX_ENV_VALUE_BYTES {
92 return Err(SettingsCapacity);
93 }
94 Ok(Self {
95 #[cfg(feature = "cuda")]
96 cached_cuda_tuning: crate::inference::cuda_lane::cached_tuning_digest(),
97 inference_mode: crate::inference_modes::atomic_inference_mode(),
98 environment,
99 decode_budget_override: super::toggles::decode_budget_override(),
100 inference_timeout_ms: super::toggles::inference_timeout_ms(),
101 gpu_topk_enabled: super::toggles::gpu_topk_enabled(),
102 ternary_ffn_enabled: super::toggles::ternary_ffn_enabled(),
103 attention_preproject_enabled: super::toggles::attention_preproject_enabled(),
104 attention_o_fuse_enabled: super::toggles::attention_o_fuse_enabled(),
105 forge_attention_enabled: super::toggles::forge_attention_enabled(),
106 resident_weights_enabled: super::toggles::resident_weights_enabled(),
107 resident_decode_enabled: super::toggles::resident_decode_enabled(),
108 resident_prefill_enabled: super::toggles::resident_prefill_enabled(),
109 spec_decode_enabled: super::toggles::spec_decode_enabled(),
110 kv_int8_enabled: super::toggles::kv_int8_enabled(),
111 kv_dict_enabled: super::toggles::kv_dict_enabled(),
112 sampler_config: super::toggles::sampler_config(),
113 ffn_fusion_enabled: super::toggles::ffn_fusion_enabled(),
114 ffn_f16_enabled: super::toggles::ffn_f16_enabled(),
115 coop_gemv_enabled: super::toggles::coop_gemv_enabled(),
116 coopmat_gemm_enabled: super::toggles::coopmat_gemm_enabled(),
117 cpu_attention_enabled: super::toggles::cpu_attention_enabled(),
118 prefix_cache_enabled: super::toggles::prefix_cache_enabled(),
119 kv_pool_budget_mb: super::toggles::kv_pool_budget_mb(),
120 prefill_chunk_size_override: super::toggles::prefill_chunk_size_override(),
121 prompt_precision_mode,
122 stage_telemetry_enabled: super::toggles::stage_telemetry_enabled(),
123 })
124 }
125
126 pub fn digest(&self) -> [u8; 32] {
128 let mut hash = Sha256::new();
129 hash.update(b"qualia:prepared-inference-settings:v1\0");
130 #[cfg(feature = "cuda")]
131 hash.update(self.cached_cuda_tuning);
132 hash.update([self.inference_mode as u8]);
133 for (key, value) in &self.environment {
134 hash.update((key.len() as u64).to_le_bytes());
135 hash.update(key.as_bytes());
136 if let Some(value) = value {
137 hash.update([1]);
138 hash.update((value.as_encoded_bytes().len() as u64).to_le_bytes());
139 hash.update(value.as_encoded_bytes());
140 } else {
141 hash.update([0]);
142 }
143 }
144 hash.update(b"decode_budget_override\0");
145 hash.update(self.decode_budget_override.to_le_bytes());
146 hash.update(b"inference_timeout_ms\0");
147 hash.update(self.inference_timeout_ms.to_le_bytes());
148 hash.update(b"gpu_topk_enabled\0");
149 hash.update([u8::from(self.gpu_topk_enabled)]);
150 hash.update(b"ternary_ffn_enabled\0");
151 hash.update([u8::from(self.ternary_ffn_enabled)]);
152 hash.update(b"attention_preproject_enabled\0");
153 hash.update([u8::from(self.attention_preproject_enabled)]);
154 hash.update(b"attention_o_fuse_enabled\0");
155 hash.update([u8::from(self.attention_o_fuse_enabled)]);
156 hash.update(b"forge_attention_enabled\0");
157 hash.update([u8::from(self.forge_attention_enabled)]);
158 hash.update(b"resident_weights_enabled\0");
159 hash.update([u8::from(self.resident_weights_enabled)]);
160 hash.update(b"resident_decode_enabled\0");
161 hash.update([u8::from(self.resident_decode_enabled)]);
162 hash.update(b"resident_prefill_enabled\0");
163 hash.update([u8::from(self.resident_prefill_enabled)]);
164 hash.update(b"spec_decode_enabled\0");
165 hash.update([u8::from(self.spec_decode_enabled)]);
166 hash.update(b"kv_int8_enabled\0");
167 hash.update([u8::from(self.kv_int8_enabled)]);
168 hash.update(b"kv_dict_enabled\0");
169 hash.update([u8::from(self.kv_dict_enabled)]);
170 hash.update(b"sampler_config\0");
171 if let Some(config) = self.sampler_config {
172 hash.update([1]);
173 for value in [
174 config.temperature,
175 config.top_p,
176 config.repeat_penalty,
177 config.freq_penalty,
178 config.presence_penalty,
179 ] {
180 hash.update(value.to_bits().to_le_bytes());
181 }
182 hash.update(config.top_k.to_le_bytes());
183 hash.update(config.penalty_window.to_le_bytes());
184 hash.update(config.seed.to_le_bytes());
185 } else {
186 hash.update([0]);
187 }
188 hash.update(b"ffn_fusion_enabled\0");
189 hash.update([u8::from(self.ffn_fusion_enabled)]);
190 hash.update(b"ffn_f16_enabled\0");
191 hash.update([u8::from(self.ffn_f16_enabled)]);
192 hash.update(b"coop_gemv_enabled\0");
193 hash.update([u8::from(self.coop_gemv_enabled)]);
194 hash.update(b"coopmat_gemm_enabled\0");
195 hash.update([u8::from(self.coopmat_gemm_enabled)]);
196 hash.update(b"cpu_attention_enabled\0");
197 hash.update([u8::from(self.cpu_attention_enabled)]);
198 hash.update(b"prefix_cache_enabled\0");
199 hash.update([u8::from(self.prefix_cache_enabled)]);
200 hash.update(b"kv_pool_budget_mb\0");
201 hash.update(self.kv_pool_budget_mb.to_le_bytes());
202 hash.update(b"prefill_chunk_size_override\0");
203 hash.update(self.prefill_chunk_size_override.to_le_bytes());
204 hash.update(b"prompt_precision_mode\0");
205 hash.update((self.prompt_precision_mode.len() as u64).to_le_bytes());
206 hash.update(self.prompt_precision_mode.as_bytes());
207 hash.update(b"stage_telemetry_enabled\0");
208 hash.update([u8::from(self.stage_telemetry_enabled)]);
209 hash.finalize().into()
210 }
211
212 pub fn enter(self) -> PreparedSettingsGuard {
214 let previous = PREPARED.with(|slot| slot.replace(Some(self)));
215 PreparedSettingsGuard {
216 previous,
217 _thread: PhantomData,
218 }
219 }
220}
221
222pub struct PreparedSettingsGuard {
224 previous: Option<PreparedInferenceSettings>,
225 _thread: PhantomData<Rc<()>>,
226}
227impl Drop for PreparedSettingsGuard {
228 fn drop(&mut self) {
229 PREPARED.with(|slot| {
230 slot.replace(self.previous.take());
231 });
232 }
233}
234
235pub fn env_var_os(key: &str) -> Option<OsString> {
238 if let Some(value) = read(|settings| {
239 settings
240 .environment
241 .iter()
242 .find(|(name, _)| *name == key)
243 .map(|(_, value)| value.clone())
244 }) {
245 if let Some(value) = value {
246 return value;
247 }
248 }
249 std::env::var_os(key)
250}
251pub fn env_var(key: &str) -> Result<String, VarError> {
252 env_var_os(key)
253 .ok_or(VarError::NotPresent)?
254 .into_string()
255 .map_err(VarError::NotUnicode)
256}