qualia_core_db/inference/
resident_model.rs

1//! Process-wide resident GGUF mmap — released explicitly on model eviction.
2
3#[cfg(not(target_arch = "wasm32"))]
4use crate::gguf_bridge::{GgufLoadReport, QTensorEngine};
5#[cfg(not(target_arch = "wasm32"))]
6use std::path::Path;
7#[cfg(not(target_arch = "wasm32"))]
8use std::sync::{Arc, Mutex, OnceLock};
9
10#[cfg(all(unix, not(target_arch = "wasm32")))]
11fn apply_mlock(mmap: &memmap2::Mmap, mlock: bool) {
12    if mlock {
13        unsafe {
14            libc::mlock(mmap.as_ptr() as *const libc::c_void, mmap.len());
15        }
16    }
17}
18
19#[cfg(all(not(target_arch = "wasm32"), not(unix)))]
20fn apply_mlock<T>(_mmap: &T, _mlock: bool) {}
21
22#[cfg(not(target_arch = "wasm32"))]
23#[derive(Debug)]
24pub struct ResidentModelSlot {
25    pub model_id: u64,
26    pub gguf_path: String,
27    pub mmap: Arc<memmap2::Mmap>,
28    pub report: GgufLoadReport,
29}
30
31#[cfg(not(target_arch = "wasm32"))]
32fn slot() -> &'static Arc<Mutex<Option<ResidentModelSlot>>> {
33    static SLOT: OnceLock<Arc<Mutex<Option<ResidentModelSlot>>>> = OnceLock::new();
34    SLOT.get_or_init(|| Arc::new(Mutex::new(None)))
35}
36
37/// Memory-map `path` and retain until [`clear_resident_model`].
38///
39/// Accepts **GGUF or P64** (sniffed by magic). Prefer calling
40/// [`mount_resident_model`]; this name remains for callers that still say "gguf".
41#[cfg(not(target_arch = "wasm32"))]
42pub fn mount_resident_gguf(
43    model_id: u64,
44    path: &str,
45    mlock: bool,
46) -> Result<GgufLoadReport, String> {
47    mount_resident_model(model_id, path, mlock)
48}
49
50#[cfg(target_arch = "wasm32")]
51pub fn mount_resident_gguf(_model_id: u64, _path: &str) -> Result<(), String> {
52    Ok(())
53}
54
55/// Memory-map a P64 weight container and retain it as the resident model.
56///
57/// The function name is retained for source compatibility. New format-neutral
58/// callers should use [`mount_resident_model`].
59#[cfg(not(target_arch = "wasm32"))]
60pub fn mount_resident_q42(
61    model_id: u64,
62    path: &str,
63    mlock: bool,
64) -> Result<GgufLoadReport, String> {
65    clear_resident_model();
66    let file = std::fs::File::open(path).map_err(|e| format!("open {path}: {e}"))?;
67    let mmap_raw =
68        unsafe { memmap2::MmapOptions::new().populate().map(&file) }.map_err(|e| e.to_string())?;
69    apply_mlock(&mmap_raw, mlock);
70    let mmap = Arc::new(mmap_raw);
71    let mut engine = tokio::task::block_in_place(|| {
72        tokio::runtime::Handle::current().block_on(QTensorEngine::try_new())
73    })?;
74    let report = engine.adopt_resident_p64_mmap(Arc::clone(&mmap))?;
75    let normalized = Path::new(path)
76        .canonicalize()
77        .map(|p| p.to_string_lossy().into_owned())
78        .unwrap_or_else(|_| path.to_string());
79    *slot().lock().map_err(|e| e.to_string())? = Some(ResidentModelSlot {
80        model_id,
81        gguf_path: normalized,
82        mmap,
83        report,
84    });
85    Ok(report)
86}
87
88/// Memory-map a local model and select P64 or GGUF by canonical magic.
89///
90/// This is the preferred format-neutral entry point. The historical
91/// `mount_resident_q42` function remains as a compatibility alias for callers
92/// that already know they have a P64 container.
93#[cfg(not(target_arch = "wasm32"))]
94pub fn mount_resident_model(
95    model_id: u64,
96    path: &str,
97    mlock: bool,
98) -> Result<GgufLoadReport, String> {
99    clear_resident_model();
100    let mut engine = tokio::task::block_in_place(|| {
101        tokio::runtime::Handle::current().block_on(QTensorEngine::try_new())
102    })?;
103    let report = engine.load_model_checked(path)?;
104    let mmap = engine
105        .gguf_mmap
106        .take()
107        .ok_or_else(|| "Internal error: model mmap missing after load".to_string())?;
108    apply_mlock(&mmap, mlock);
109    let normalized = Path::new(path)
110        .canonicalize()
111        .map(|p| p.to_string_lossy().into_owned())
112        .unwrap_or_else(|_| path.to_string());
113    *slot().lock().map_err(|e| e.to_string())? = Some(ResidentModelSlot {
114        model_id,
115        gguf_path: normalized,
116        mmap,
117        report,
118    });
119    Ok(report)
120}
121
122/// Drop resident mmap (called from orchestrator eviction scrub).
123#[cfg(not(target_arch = "wasm32"))]
124pub fn clear_resident_model() {
125    if let Ok(mut guard) = slot().lock() {
126        if guard.take().is_some() {
127            log::info!("LLM_LOAD|evict-mmap|1.00|Released resident GGUF mmap");
128        }
129    }
130}
131
132#[cfg(target_arch = "wasm32")]
133pub fn clear_resident_model() {}
134
135#[cfg(not(target_arch = "wasm32"))]
136pub fn resident_mmap_for_path(path: &str) -> Option<Arc<memmap2::Mmap>> {
137    let guard = slot().lock().ok()?;
138    let slot = guard.as_ref()?;
139    let requested = Path::new(path);
140    let slot_path = Path::new(&slot.gguf_path);
141    if requested == slot_path {
142        return Some(Arc::clone(&slot.mmap));
143    }
144    let req_canon = requested.canonicalize().ok();
145    let slot_canon = slot_path.canonicalize().ok();
146    if req_canon.is_some() && req_canon == slot_canon {
147        return Some(Arc::clone(&slot.mmap));
148    }
149    if requested.file_name().is_some() && requested.file_name() == slot_path.file_name() {
150        return Some(Arc::clone(&slot.mmap));
151    }
152    None
153}
154
155#[cfg(not(target_arch = "wasm32"))]
156pub fn resident_model_id() -> Option<u64> {
157    slot()
158        .lock()
159        .ok()
160        .and_then(|g| g.as_ref().map(|s| s.model_id))
161}
162
163#[cfg(not(target_arch = "wasm32"))]
164pub fn resident_gguf_path() -> Option<String> {
165    slot()
166        .lock()
167        .ok()
168        .and_then(|g| g.as_ref().map(|s| s.gguf_path.clone()))
169}
170
171#[cfg(target_arch = "wasm32")]
172pub fn resident_gguf_path() -> Option<String> {
173    None
174}
175
176#[cfg(target_arch = "wasm32")]
177pub fn resident_mmap_for_path(_path: &str) -> Option<()> {
178    None
179}
qualia_core_db/inference/resident_model.rs

qualia_core_db/inference/
resident_model.rs
