Skip to main content

qualia_core_db/inference/inference_agent/
sticky_infer.rs

1// ─── Sticky 1-thread infer pool (native) ────────────────────────────────────
2// Every `infer` previously did `thread::spawn` + `QTensorEngine::new()` which
3// rebuilds wgpu pipelines (~seconds) even when the mmap is already resident.
4// A size-1 rayon pool keeps a dedicated OS thread whose `thread_local` engine
5// survives across jobs; same-path multi-turn / multi-prompt reuses the engine.
6use crate::gguf_bridge::QTensorEngine;
7use crate::inference::runtime::prepared::generation::ResidencyGenerationTracker;
8use crate::inference::runtime::scheduler::queue::{
9    BoundedIntakeQueue, IntakeError, IntakeRequest, IntakeState,
10};
11use std::cell::RefCell;
12use std::sync::{Mutex, OnceLock};
13
14/// Global residency generation tracker for the sticky infer engine.
15pub static RESIDENCY_TRACKER: ResidencyGenerationTracker = ResidencyGenerationTracker::new(1);
16
17pub struct StickyEngine {
18    pub path: String,
19    pub generation: u64,
20    /// Exact effective preparation options used when this engine was loaded.
21    pub settings_digest: Option<[u8; 32]>,
22    pub engine: QTensorEngine,
23}
24
25thread_local! {
26    static ENGINE: RefCell<Option<StickyEngine>> = const { RefCell::new(None) };
27}
28
29/// Single-threaded rayon pool for sticky decode.
30pub fn pool() -> &'static rayon::ThreadPool {
31    static POOL: OnceLock<rayon::ThreadPool> = OnceLock::new();
32    POOL.get_or_init(|| {
33        rayon::ThreadPoolBuilder::new()
34            .num_threads(1)
35            // The decode path is deliberately zero-heap (§6): it holds large
36            // buffers on the stack — e.g. `[0f32; PREFILL_CHUNK_STACK_FLOATS]`
37            // (2560·64 floats ≈ 640 KB) — and runs a full transformer forward
38            // pass whose (debug-unoptimised) call chain adds several MB more.
39            // rayon's default worker stack (~2 MB) overflows on the real decode
40            // path (the `qualia-infer-0` STACK_OVERFLOW), on device as much as in
41            // tests. Reserve a generous stack for this single dedicated worker;
42            // on 64-bit the reservation is address space only (committed on
43            // demand), so it costs nothing until touched.
44            .stack_size(64 * 1024 * 1024)
45            .thread_name(|i| format!("qualia-infer-{i}"))
46            .build()
47            .expect("qualia sticky infer pool")
48    })
49}
50
51/// Advance the residency generation, invalidating cached sticky engine handles
52/// across all worker threads.
53pub fn invalidate_sticky_residency() -> u64 {
54    RESIDENCY_TRACKER.advance_generation()
55}
56
57/// Read the current residency generation number.
58pub fn current_residency_generation() -> u64 {
59    RESIDENCY_TRACKER.generation()
60}
61
62/// Borrow-or-reload the sticky engine for `path`, then run `f`.
63/// Validates both pathname equality and residency generation freshness.
64pub fn with_engine<R>(
65    path: &str,
66    load: impl FnMut(&mut QTensorEngine),
67    f: impl FnOnce(&mut QTensorEngine) -> R,
68) -> R {
69    with_engine_disposition(path, load, f, || false)
70}
71
72/// Failed/uncertain turns discard the numerical owner on completion or unwind.
73/// No checkpoint digest is represented as a rollback image. The epoch advances
74/// before destruction, so queued/prepared handles cannot reuse this state.
75pub fn with_engine_disposition<R>(
76    path: &str,
77    load: impl FnMut(&mut QTensorEngine),
78    f: impl FnOnce(&mut QTensorEngine) -> R,
79    discard: impl FnOnce() -> bool,
80) -> R {
81    let mut load = load;
82    let mut current_gen = RESIDENCY_TRACKER.generation();
83    let settings_digest = crate::llm_bench::prepared_settings::read(|settings| settings.digest());
84    ENGINE.with(|cell| {
85        let mut slot = cell.borrow_mut();
86        let reload = match slot.as_ref() {
87            Some(s) => {
88                s.path != path
89                    || s.generation != current_gen
90                    || s.settings_digest != settings_digest
91            }
92            None => true,
93        };
94        if reload {
95            if slot.is_some() {
96                current_gen = invalidate_sticky_residency();
97                #[cfg(feature = "cuda")]
98                crate::inference::cuda_lane::discard_numerical_owner();
99                if let Ok(mut cache) = super::decode_helpers::get_prefix_cache().lock() {
100                    cache.clear();
101                }
102                // Drop the old owner before loading a differently prepared model.
103                *slot = None;
104            }
105            let mut engine = QTensorEngine::new();
106            load(&mut engine);
107            *slot = Some(StickyEngine {
108                path: path.to_string(),
109                generation: current_gen,
110                settings_digest,
111                engine,
112            });
113        }
114        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
115            f(&mut slot.as_mut().expect("sticky engine just loaded").engine)
116        }));
117        if result.is_err() || discard() {
118            invalidate_sticky_residency();
119            // The legacy CUDA lane has a separate global KV/graph/slab owner.
120            #[cfg(feature = "cuda")]
121            crate::inference::cuda_lane::discard_numerical_owner();
122            // Destruction owns engine KV/recurrent/device buffers.
123            *slot = None;
124            if let Ok(mut cache) = super::decode_helpers::get_prefix_cache().lock() {
125                cache.clear();
126            }
127            crate::llm_bench::set_domino_masker(None);
128        } else if let Some(sticky) = slot.as_mut() {
129            if let Some(generation) = sticky.engine.prepared_model_residency_generation() {
130                sticky.generation = generation;
131            }
132        }
133        match result {
134            Ok(value) => value,
135            Err(payload) => std::panic::resume_unwind(payload),
136        }
137    })
138}
139
140/// Shared bounded intake manager for sticky infer execution (Work Package F4).
141pub struct StickyAdmissionDriver {
142    queue: Mutex<BoundedIntakeQueue<128>>,
143}
144
145impl Default for StickyAdmissionDriver {
146    fn default() -> Self {
147        Self::new()
148    }
149}
150
151impl StickyAdmissionDriver {
152    /// Creates a default `StickyAdmissionDriver`.
153    pub fn new() -> Self {
154        Self {
155            queue: Mutex::new(BoundedIntakeQueue::new()),
156        }
157    }
158
159    /// Global singleton instance of the sticky intake driver.
160    pub fn global() -> &'static Self {
161        static DRIVER: OnceLock<StickyAdmissionDriver> = OnceLock::new();
162        DRIVER.get_or_init(Self::new)
163    }
164
165    /// Enqueue a request descriptor into the bounded intake queue.
166    pub fn enqueue(&self, request: IntakeRequest) -> Result<(), IntakeError> {
167        let mut q = self.queue.lock().unwrap();
168        q.enqueue(request)
169    }
170
171    /// Pop the next runnable request descriptor.
172    pub fn pop_next(&self) -> Option<IntakeRequest> {
173        let mut q = self.queue.lock().unwrap();
174        q.pop_next_runnable()
175    }
176
177    /// Cancel a request in the queue.
178    pub fn cancel(&self, request_id: u64) -> Result<IntakeState, IntakeError> {
179        let mut q = self.queue.lock().unwrap();
180        q.cancel(request_id)
181    }
182}
183
184#[cfg(test)]
185mod tests {
186    use super::*;
187
188    #[test]
189    fn test_residency_invalidation_lifecycle() {
190        let gen1 = current_residency_generation();
191        let gen2 = invalidate_sticky_residency();
192        assert_eq!(gen2, gen1 + 1);
193        assert_eq!(current_residency_generation(), gen2);
194    }
195
196    #[test]
197    fn test_sticky_admission_driver_intake_and_cancel() {
198        let driver = StickyAdmissionDriver::new();
199        let req1 = IntakeRequest::new(101, 50, 20);
200        let req2 = IntakeRequest::new(102, 100, 40);
201
202        driver.enqueue(req1).unwrap();
203        driver.enqueue(req2).unwrap();
204
205        driver.cancel(101).unwrap();
206
207        // req1 was cancelled, so pop_next returns req2
208        let next = driver.pop_next().unwrap();
209        assert_eq!(next.request_id, 102);
210
211        // No more runnable requests
212        assert!(driver.pop_next().is_none());
213
214        // Global singleton is accessible
215        assert!(StickyAdmissionDriver::global().pop_next().is_none());
216    }
217}