Skip to main content

qualia_core_db/inference/
resident_model.rs

1//! Process-wide resident GGUF mmap — released explicitly on model eviction.
2
3#[cfg(not(target_arch = "wasm32"))]
4use crate::gguf_bridge::{GgufLoadReport, QTensorEngine};
5#[cfg(not(target_arch = "wasm32"))]
6use std::path::Path;
7#[cfg(not(target_arch = "wasm32"))]
8use std::sync::{Arc, Mutex, OnceLock};
9
10#[cfg(all(unix, not(target_arch = "wasm32")))]
11fn apply_mlock(mmap: &memmap2::Mmap, mlock: bool) {
12    if mlock {
13        unsafe {
14            libc::mlock(mmap.as_ptr() as *const libc::c_void, mmap.len());
15        }
16    }
17}
18
19#[cfg(all(not(target_arch = "wasm32"), not(unix)))]
20fn apply_mlock<T>(_mmap: &T, _mlock: bool) {}
21
22#[cfg(not(target_arch = "wasm32"))]
23#[derive(Debug)]
24pub struct ResidentModelSlot {
25    pub model_id: u64,
26    pub gguf_path: String,
27    pub mmap: Arc<memmap2::Mmap>,
28    pub report: GgufLoadReport,
29}
30
31#[cfg(not(target_arch = "wasm32"))]
32fn slot() -> &'static Arc<Mutex<Option<ResidentModelSlot>>> {
33    static SLOT: OnceLock<Arc<Mutex<Option<ResidentModelSlot>>>> = OnceLock::new();
34    SLOT.get_or_init(|| Arc::new(Mutex::new(None)))
35}
36
37/// Memory-map `path` and retain until [`clear_resident_model`].
38///
39/// Accepts **GGUF or P64** (sniffed by magic). Prefer calling
40/// [`mount_resident_model`]; this name remains for callers that still say "gguf".
41#[cfg(not(target_arch = "wasm32"))]
42pub fn mount_resident_gguf(
43    model_id: u64,
44    path: &str,
45    mlock: bool,
46) -> Result<GgufLoadReport, String> {
47    mount_resident_model(model_id, path, mlock)
48}
49
50#[cfg(target_arch = "wasm32")]
51/// WASM gate: fails closed since raw filesystem memory-mapping is unsupported on browser wasm.
52pub fn mount_resident_gguf(_model_id: u64, _path: &str) -> Result<(), String> {
53    Err("WASM target does not support filesystem mmap; use adopt_resident_mmap with preloaded bytes".to_string())
54}
55
56/// Memory-map a P64 weight container and retain it as the resident model.
57///
58/// The function name is retained for source compatibility. New format-neutral
59/// callers should use [`mount_resident_model`].
60#[cfg(not(target_arch = "wasm32"))]
61pub fn mount_resident_q42(
62    model_id: u64,
63    path: &str,
64    mlock: bool,
65) -> Result<GgufLoadReport, String> {
66    let file = std::fs::File::open(path).map_err(|e| format!("open {path}: {e}"))?;
67    let mmap_raw =
68        unsafe { memmap2::MmapOptions::new().populate().map(&file) }.map_err(|e| e.to_string())?;
69    apply_mlock(&mmap_raw, mlock);
70    let mmap = Arc::new(mmap_raw);
71    let mut engine = tokio::task::block_in_place(|| {
72        tokio::runtime::Handle::current().block_on(QTensorEngine::try_new())
73    })?;
74    let report = engine.adopt_resident_p64_mmap(Arc::clone(&mmap))?;
75    let normalized = Path::new(path)
76        .canonicalize()
77        .map(|p| p.to_string_lossy().into_owned())
78        .unwrap_or_else(|_| path.to_string());
79    *slot().lock().map_err(|e| e.to_string())? = Some(ResidentModelSlot {
80        model_id,
81        gguf_path: normalized,
82        mmap,
83        report,
84    });
85    Ok(report)
86}
87
88/// Memory-map a local model and select P64 or GGUF by canonical magic.
89///
90/// This is the preferred format-neutral entry point. The historical
91/// `mount_resident_q42` function remains as a compatibility alias for callers
92/// that already know they have a P64 container.
93#[cfg(not(target_arch = "wasm32"))]
94pub fn mount_resident_model(
95    model_id: u64,
96    path: &str,
97    mlock: bool,
98) -> Result<GgufLoadReport, String> {
99    let mut engine = tokio::task::block_in_place(|| {
100        tokio::runtime::Handle::current().block_on(QTensorEngine::try_new())
101    })?;
102    let report = engine.load_model_checked(path)?;
103    let mmap = engine
104        .gguf_mmap
105        .take()
106        .ok_or_else(|| "Internal error: model mmap missing after load".to_string())?;
107    apply_mlock(&mmap, mlock);
108    let normalized = Path::new(path)
109        .canonicalize()
110        .map(|p| p.to_string_lossy().into_owned())
111        .unwrap_or_else(|_| path.to_string());
112    *slot().lock().map_err(|e| e.to_string())? = Some(ResidentModelSlot {
113        model_id,
114        gguf_path: normalized,
115        mmap,
116        report,
117    });
118    Ok(report)
119}
120
121/// Drop resident mmap (called from orchestrator eviction scrub).
122#[cfg(not(target_arch = "wasm32"))]
123pub fn clear_resident_model() {
124    if let Ok(mut guard) = slot().lock() {
125        if guard.take().is_some() {
126            log::info!("LLM_LOAD|evict-mmap|1.00|Released resident GGUF mmap");
127        }
128    }
129}
130
131#[cfg(target_arch = "wasm32")]
132/// Drops the resident mmap.
133pub fn clear_resident_model() {}
134
135#[cfg(not(target_arch = "wasm32"))]
136/// Resident mmap for `path` when it matches the slot.
137pub fn resident_mmap_for_path(path: &str) -> Option<Arc<memmap2::Mmap>> {
138    let guard = slot().lock().ok()?;
139    let slot = guard.as_ref()?;
140    let requested = Path::new(path);
141    let slot_path = Path::new(&slot.gguf_path);
142    if requested == slot_path {
143        return Some(Arc::clone(&slot.mmap));
144    }
145    let req_canon = requested.canonicalize().ok();
146    let slot_canon = slot_path.canonicalize().ok();
147    if req_canon.is_some() && req_canon == slot_canon {
148        return Some(Arc::clone(&slot.mmap));
149    }
150    if requested.file_name().is_some() && requested.file_name() == slot_path.file_name() {
151        return Some(Arc::clone(&slot.mmap));
152    }
153    None
154}
155
156#[cfg(not(target_arch = "wasm32"))]
157/// Resident model id.
158pub fn resident_model_id() -> Option<u64> {
159    slot()
160        .lock()
161        .ok()
162        .and_then(|g| g.as_ref().map(|s| s.model_id))
163}
164
165#[cfg(not(target_arch = "wasm32"))]
166/// Resident model path.
167pub fn resident_gguf_path() -> Option<String> {
168    slot()
169        .lock()
170        .ok()
171        .and_then(|g| g.as_ref().map(|s| s.gguf_path.clone()))
172}
173
174#[cfg(target_arch = "wasm32")]
175/// Resident model path.
176pub fn resident_gguf_path() -> Option<String> {
177    None
178}
179
180#[cfg(target_arch = "wasm32")]
181/// Resident mmap for `path` when it matches the slot.
182pub fn resident_mmap_for_path(_path: &str) -> Option<()> {
183    None
184}
185
186#[cfg(not(target_arch = "wasm32"))]
187/// Current resident mmap if one is actively mounted in the resident slot.
188pub fn current_resident_mmap() -> Option<Arc<memmap2::Mmap>> {
189    let guard = slot().lock().ok()?;
190    let slot = guard.as_ref()?;
191    Some(Arc::clone(&slot.mmap))
192}
193
194#[cfg(target_arch = "wasm32")]
195/// Current resident mmap (always None on browser wasm).
196pub fn current_resident_mmap() -> Option<()> {
197    None
198}