qualia_core_db/inference/inference_agent/
sticky_infer.rs1use crate::gguf_bridge::QTensorEngine;
7use crate::inference::runtime::prepared::generation::ResidencyGenerationTracker;
8use crate::inference::runtime::scheduler::queue::{
9 BoundedIntakeQueue, IntakeError, IntakeRequest, IntakeState,
10};
11use std::cell::RefCell;
12use std::sync::{Mutex, OnceLock};
13
14pub static RESIDENCY_TRACKER: ResidencyGenerationTracker = ResidencyGenerationTracker::new(1);
16
17pub struct StickyEngine {
18 pub path: String,
19 pub generation: u64,
20 pub settings_digest: Option<[u8; 32]>,
22 pub engine: QTensorEngine,
23}
24
25thread_local! {
26 static ENGINE: RefCell<Option<StickyEngine>> = const { RefCell::new(None) };
27}
28
29pub fn pool() -> &'static rayon::ThreadPool {
31 static POOL: OnceLock<rayon::ThreadPool> = OnceLock::new();
32 POOL.get_or_init(|| {
33 rayon::ThreadPoolBuilder::new()
34 .num_threads(1)
35 .stack_size(64 * 1024 * 1024)
45 .thread_name(|i| format!("qualia-infer-{i}"))
46 .build()
47 .expect("qualia sticky infer pool")
48 })
49}
50
51pub fn invalidate_sticky_residency() -> u64 {
54 RESIDENCY_TRACKER.advance_generation()
55}
56
57pub fn current_residency_generation() -> u64 {
59 RESIDENCY_TRACKER.generation()
60}
61
62pub fn with_engine<R>(
65 path: &str,
66 load: impl FnMut(&mut QTensorEngine),
67 f: impl FnOnce(&mut QTensorEngine) -> R,
68) -> R {
69 with_engine_disposition(path, load, f, || false)
70}
71
72pub fn with_engine_disposition<R>(
76 path: &str,
77 load: impl FnMut(&mut QTensorEngine),
78 f: impl FnOnce(&mut QTensorEngine) -> R,
79 discard: impl FnOnce() -> bool,
80) -> R {
81 let mut load = load;
82 let mut current_gen = RESIDENCY_TRACKER.generation();
83 let settings_digest = crate::llm_bench::prepared_settings::read(|settings| settings.digest());
84 ENGINE.with(|cell| {
85 let mut slot = cell.borrow_mut();
86 let reload = match slot.as_ref() {
87 Some(s) => {
88 s.path != path
89 || s.generation != current_gen
90 || s.settings_digest != settings_digest
91 }
92 None => true,
93 };
94 if reload {
95 if slot.is_some() {
96 current_gen = invalidate_sticky_residency();
97 #[cfg(feature = "cuda")]
98 crate::inference::cuda_lane::discard_numerical_owner();
99 if let Ok(mut cache) = super::decode_helpers::get_prefix_cache().lock() {
100 cache.clear();
101 }
102 *slot = None;
104 }
105 let mut engine = QTensorEngine::new();
106 load(&mut engine);
107 *slot = Some(StickyEngine {
108 path: path.to_string(),
109 generation: current_gen,
110 settings_digest,
111 engine,
112 });
113 }
114 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
115 f(&mut slot.as_mut().expect("sticky engine just loaded").engine)
116 }));
117 if result.is_err() || discard() {
118 invalidate_sticky_residency();
119 #[cfg(feature = "cuda")]
121 crate::inference::cuda_lane::discard_numerical_owner();
122 *slot = None;
124 if let Ok(mut cache) = super::decode_helpers::get_prefix_cache().lock() {
125 cache.clear();
126 }
127 crate::llm_bench::set_domino_masker(None);
128 } else if let Some(sticky) = slot.as_mut() {
129 if let Some(generation) = sticky.engine.prepared_model_residency_generation() {
130 sticky.generation = generation;
131 }
132 }
133 match result {
134 Ok(value) => value,
135 Err(payload) => std::panic::resume_unwind(payload),
136 }
137 })
138}
139
140pub struct StickyAdmissionDriver {
142 queue: Mutex<BoundedIntakeQueue<128>>,
143}
144
145impl Default for StickyAdmissionDriver {
146 fn default() -> Self {
147 Self::new()
148 }
149}
150
151impl StickyAdmissionDriver {
152 pub fn new() -> Self {
154 Self {
155 queue: Mutex::new(BoundedIntakeQueue::new()),
156 }
157 }
158
159 pub fn global() -> &'static Self {
161 static DRIVER: OnceLock<StickyAdmissionDriver> = OnceLock::new();
162 DRIVER.get_or_init(Self::new)
163 }
164
165 pub fn enqueue(&self, request: IntakeRequest) -> Result<(), IntakeError> {
167 let mut q = self.queue.lock().unwrap();
168 q.enqueue(request)
169 }
170
171 pub fn pop_next(&self) -> Option<IntakeRequest> {
173 let mut q = self.queue.lock().unwrap();
174 q.pop_next_runnable()
175 }
176
177 pub fn cancel(&self, request_id: u64) -> Result<IntakeState, IntakeError> {
179 let mut q = self.queue.lock().unwrap();
180 q.cancel(request_id)
181 }
182}
183
184#[cfg(test)]
185mod tests {
186 use super::*;
187
188 #[test]
189 fn test_residency_invalidation_lifecycle() {
190 let gen1 = current_residency_generation();
191 let gen2 = invalidate_sticky_residency();
192 assert_eq!(gen2, gen1 + 1);
193 assert_eq!(current_residency_generation(), gen2);
194 }
195
196 #[test]
197 fn test_sticky_admission_driver_intake_and_cancel() {
198 let driver = StickyAdmissionDriver::new();
199 let req1 = IntakeRequest::new(101, 50, 20);
200 let req2 = IntakeRequest::new(102, 100, 40);
201
202 driver.enqueue(req1).unwrap();
203 driver.enqueue(req2).unwrap();
204
205 driver.cancel(101).unwrap();
206
207 let next = driver.pop_next().unwrap();
209 assert_eq!(next.request_id, 102);
210
211 assert!(driver.pop_next().is_none());
213
214 assert!(StickyAdmissionDriver::global().pop_next().is_none());
216 }
217}