Skip to main content

qualia_core_db/gguf_bridge/
load.rs

1//! Model load + residency: GGUF load, ternary-FFN resident build, resident mmap adoption,
2//! resident logits upload, KV-cache sizing, decode-profiler hooks. Split from mod.rs (structural).
3use super::*;
4
5impl QTensorEngine {
6    /// KV-cache layout when the engine allocates one.
7    pub fn kv_layout(&self) -> Option<KvCacheLayout> {
8        self.kv_layout
9    }
10
11    /// KV-cache size in bytes.
12    pub fn kv_cache_bytes(&self) -> u64 {
13        #[cfg(not(target_arch = "wasm32"))]
14        if let Some(session) = &self.qwen35_ftw {
15            return session.kv_cache_bytes() as u64;
16        }
17        #[cfg(not(target_arch = "wasm32"))]
18        if let Some(session) = &self.gemma4 {
19            return session.kv_cache_bytes() as u64;
20        }
21        self.kv_layout
22            .as_ref()
23            .map(|layout| (layout.total_f32_elems * std::mem::size_of::<f32>()) as u64)
24            .unwrap_or(0)
25    }
26
27    #[cfg(not(target_arch = "wasm32"))]
28    /// Loads `gguf-checked`.
29    pub fn load_gguf_checked(&mut self, path: &str) -> Result<GgufLoadReport, String> {
30        use std::fs::File;
31
32        log::info!("LLM_LOAD|gguf-open|0.52|Opening GGUF file {}", path);
33        let file = File::open(path).map_err(|e| {
34            log::error!("GGUF mmap open failed for {}: {}", path, e);
35            log::error!("LLM_LOAD|failed|1.00|Could not open GGUF: {}", e);
36            e.to_string()
37        })?;
38        log::info!("LLM_LOAD|mmap-start|0.64|Memory-mapping GGUF into virtual memory");
39        let mmap = unsafe { MmapOptions::new().map(&file) }.map_err(|e| {
40            log::error!("GGUF mmap failed for {}: {}", path, e);
41            log::error!("LLM_LOAD|failed|1.00|Memory map failed: {}", e);
42            e.to_string()
43        })?;
44        let file_size = mmap.len();
45        log::info!(
46            "LLM_LOAD|ram-map|0.70|Mapped {:.2} GiB GGUF into system memory",
47            bytes_to_gib(file_size as u64)
48        );
49        // H2 CONSUMPTION wiring (default-OFF via QUALIA_LLM_ROUTE): compute + record + store
50        // the residency EmploymentPlan for this model, using the honest mapped weight-byte count
51        // (`file_size`) and the already-probed hardware passport. This makes the residency planner
52        // OBSERVABLE and CONSUMED without changing weight placement or the decode path.
53        // TODO(H3-exec): actually running overflow layers on the auxiliary circuit is the remaining
54        // (out-of-scope) H3 step; the recorded plan is advisory until an execution stage consults it.
55        let _ = crate::residency_planner::route_employment_from_passport(file_size as u64);
56        let mut index = crate::gguf_sharder::GgufTensorIndex::from_gguf(&mmap);
57        if index.tensor_data_start == 0
58            && index.max_tensor_bytes == 0
59            && index.hyperparams.n_layer == 0
60        {
61            let msg = "GGUF header parse failed or yielded no tensor metadata".to_string();
62            log::error!("LLM_LOAD|failed|1.00|{}", msg);
63            return Err(msg);
64        }
65        if let Some(profile) = qwen35::gguf::Profile::parse(&mmap, &mut index)? {
66            return self.load_dense_qwen_gguf(mmap, index, profile);
67        }
68        if let Some(profile) = gemma4::Profile::parse(&mmap)? {
69            return self.load_gemma4_gguf(
70                mmap,
71                index,
72                profile,
73                MAX_CONTEXT_WINDOW as usize,
74                self.ssm_state_byte_budget,
75            );
76        }
77        if let Err(msg) = index.hyperparams.decode_supported() {
78            log::error!("LLM_LOAD|failed|1.00|{}", msg);
79            return Err(msg);
80        }
81
82        self.init_ssm_state_with_budget(&index, self.ssm_state_byte_budget)
83            .map_err(|e| e.to_string())?;
84        self.tensor_data_offset = index.tensor_data_start;
85        self.hyperparams = index.hyperparams;
86        #[cfg(feature = "gpu-runtime")]
87        {
88            let vocab_chunk_bytes = index
89                .logits_projection_info()
90                .and_then(|info| {
91                    let (_, vocab) = Self::matmul_dims(info);
92                    crate::ggml_quants::tensor_byte_len(&info)
93                        .and_then(|tb| tb.checked_div(vocab.max(1)))
94                })
95                .map(|rb| rb.saturating_mul(VOCAB_CHUNK_ROWS))
96                .unwrap_or(0);
97            let staging = index
98                .max_layer_tensor_bytes
99                .max(vocab_chunk_bytes)
100                .max(4096)
101                .min(MAX_WGPU_WEIGHT_STAGING);
102            self.ensure_gemm_buffers(staging, MAX_STACK_GEMM_OUT as u32);
103        }
104        self.ensure_kv_cache(&index.hyperparams);
105        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
106        self.invalidate_resident_projection_weights();
107        #[cfg(not(target_arch = "wasm32"))]
108        {
109            self.prepared_native_bytes = None;
110            self.prepared_native_sources = None;
111            self.qwen35_ftw = None;
112            self.qwen35_gguf = None;
113            self.qwen35_gguf_required = false;
114            self.gemma4 = None;
115            self.gemma4_required = false;
116            self.ftw_package = None;
117        }
118        self.gguf_mmap = Some(Arc::new(mmap));
119        self.p64_index = None;
120        self.tensor_index_cache = Some(index.clone());
121
122        let kv_cache_bytes = self.kv_cache_bytes();
123        log::info!(
124            "LLM_LOAD|gguf-index|0.78|Parsed {} layers, {} attention heads",
125            self.hyperparams.n_layer,
126            self.hyperparams.n_head
127        );
128        log::info!(
129            "LLM_LOAD|gguf-ready|0.92|GGUF indexed and cache arena reserved ({} MiB)",
130            kv_cache_bytes / (1024 * 1024)
131        );
132
133        Ok(GgufLoadReport {
134            mapped_bytes: file_size as u64,
135            tensor_data_offset: self.tensor_data_offset,
136            n_layer: self.hyperparams.n_layer,
137            n_head: self.hyperparams.n_head,
138            n_kv_head: self.hyperparams.effective_n_kv_head(),
139            max_tensor_bytes: index.max_tensor_bytes,
140            kv_cache_bytes,
141            directml_enabled: {
142                #[cfg(target_os = "windows")]
143                {
144                    self.dml.is_some()
145                }
146                #[cfg(not(target_os = "windows"))]
147                {
148                    false
149                }
150            },
151        })
152    }
153
154    /// Memory-map a GGUF file so tensor bytes are accessible without heap allocation.
155    /// Call this once after `new()`, before the first `dispatch_fused_transformer_block`.
156    #[cfg(not(target_arch = "wasm32"))]
157    pub fn load_gguf(&mut self, path: &str) {
158        if let Err(e) = self.load_gguf_checked(path) {
159            eprintln!("[gguf_bridge] Could not load {path}: {e}");
160        }
161    }
162
163    /// Memory-map and auto-detect a supported local model container.
164    ///
165    /// Canonical P64 is detected by its exact `p64\0` magic and adopted through
166    /// the P64 validation path. All other inputs are passed to the GGUF parser,
167    /// which rejects malformed or unsupported data.
168    #[cfg(not(target_arch = "wasm32"))]
169    pub fn load_model_checked(&mut self, path: &str) -> Result<GgufLoadReport, String> {
170        let p = std::path::Path::new(path);
171        if p.is_dir() {
172            if p.join("freetoken_weight.json").exists() {
173                return self.adopt_ftw_package(p);
174            }
175        } else if path.ends_with("freetoken_weight.json") || path.ends_with(".ftw") {
176            if let Some(parent) = p.parent() {
177                if parent.join("freetoken_weight.json").exists() {
178                    return self.adopt_ftw_package(parent);
179                }
180            }
181        }
182
183        let file = std::fs::File::open(path).map_err(|e| format!("open {path}: {e}"))?;
184        let mmap = std::sync::Arc::new(
185            unsafe { memmap2::MmapOptions::new().map(&file) }.map_err(|e| e.to_string())?,
186        );
187        if crate::p64_weight::has_p64_magic(&mmap[..]) {
188            self.adopt_resident_p64_mmap(mmap)
189        } else if crate::safetensor::detect_format(&mmap[..])
190            == crate::safetensor::SourceFormat::Safetensor
191        {
192            let config_json = std::path::Path::new(path)
193                .parent()
194                .map(|p| p.join("config.json"))
195                .and_then(|p| std::fs::read_to_string(p).ok());
196            self.adopt_resident_safetensor_mmap(mmap, config_json.as_deref())
197        } else {
198            self.adopt_resident_mmap(mmap)
199        }
200    }
201
202    /// Boot directly from an already-mapped Safetensors model (including BF16).
203    #[cfg(not(target_arch = "wasm32"))]
204    pub fn adopt_resident_safetensor_mmap(
205        &mut self,
206        mmap: Arc<memmap2::Mmap>,
207        config_json: Option<&str>,
208    ) -> Result<GgufLoadReport, String> {
209        let file_size = mmap.len();
210        if file_size == 0 {
211            return Err("Empty Safetensors mmap".to_string());
212        }
213        log::info!(
214            "LLM_LOAD|safetensor|0.75|Reusing resident Safetensors mapping ({:.2} GiB)",
215            bytes_to_gib(file_size as u64)
216        );
217        let index =
218            crate::inference::safetensor_loader::parse_safetensor_to_index(&mmap[..], config_json)?;
219        if index.hyperparams.n_layer == 0 || index.hyperparams.n_embd == 0 {
220            return Err("Safetensor: missing hyperparameters".to_string());
221        }
222        self.init_ssm_state_with_budget(&index, self.ssm_state_byte_budget)
223            .map_err(|e| e.to_string())?;
224        self.tensor_data_offset = index.tensor_data_start;
225        self.hyperparams = index.hyperparams;
226        #[cfg(feature = "gpu-runtime")]
227        {
228            let vocab_chunk_bytes = index
229                .logits_projection_info()
230                .and_then(|info| {
231                    let (_, vocab) = Self::matmul_dims(info);
232                    crate::ggml_quants::tensor_byte_len(&info)
233                        .and_then(|tb| tb.checked_div(vocab.max(1)))
234                })
235                .map(|rb| rb.saturating_mul(VOCAB_CHUNK_ROWS))
236                .unwrap_or(0);
237            let staging = index
238                .max_layer_tensor_bytes
239                .max(vocab_chunk_bytes)
240                .max(4096)
241                .min(MAX_WGPU_WEIGHT_STAGING);
242            self.ensure_gemm_buffers(staging, MAX_STACK_GEMM_OUT as u32);
243        }
244        self.ensure_kv_cache(&index.hyperparams);
245        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
246        if self.prepared_native_bytes.is_some()
247            || !self
248                .gguf_mmap
249                .as_ref()
250                .is_some_and(|old| Arc::ptr_eq(old, &mmap))
251        {
252            self.invalidate_resident_projection_weights();
253        }
254        #[cfg(not(target_arch = "wasm32"))]
255        {
256            self.prepared_native_bytes = None;
257            self.prepared_native_sources = None;
258            self.qwen35_ftw = None;
259            self.qwen35_gguf = None;
260            self.qwen35_gguf_required = false;
261            self.gemma4 = None;
262            self.gemma4_required = false;
263            self.ftw_package = None;
264        }
265        self.gguf_mmap = Some(mmap);
266        self.p64_index = None;
267        self.tensor_index_cache = Some(index.clone());
268        #[cfg(feature = "gpu-runtime")]
269        if !self.mc8_upload_resident_logits(&index) {
270            log::info!("LLM_LOAD|safetensor-logits|0.70|skipped — per-token upload fallback");
271        }
272        let kv_cache_bytes = self.kv_cache_bytes();
273        Ok(GgufLoadReport {
274            mapped_bytes: file_size as u64,
275            tensor_data_offset: self.tensor_data_offset,
276            n_layer: self.hyperparams.n_layer,
277            n_head: self.hyperparams.n_head,
278            n_kv_head: self.hyperparams.effective_n_kv_head(),
279            max_tensor_bytes: index.max_tensor_bytes,
280            kv_cache_bytes,
281            directml_enabled: {
282                #[cfg(target_os = "windows")]
283                {
284                    self.dml.is_some()
285                }
286                #[cfg(not(target_os = "windows"))]
287                {
288                    false
289                }
290            },
291        })
292    }
293
294    /// Fail-soft wrapper retained for the agent decode path.
295    #[cfg(not(target_arch = "wasm32"))]
296    pub fn load_model(&mut self, path: &str) {
297        if let Err(e) = self.load_model_checked(path) {
298            eprintln!("[gguf_bridge] Could not load model {path}: {e}");
299        }
300    }
301
302    /// Build the resident 2-bit ternary-FFN dispatcher from a P64 container's base-3 FFN
303    /// blobs (rebaked to 2-bit + uploaded once). Returns false if there are no ternary FFN tensors
304    /// or the GPU build fails — the FFN then runs the CPU oracle (`dispatch_ternary_ffn` fallback).
305    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
306    pub(crate) fn build_ternary_ffn_resident(
307        &mut self,
308        q: &crate::p64_weight::P64TensorIndex,
309    ) -> bool {
310        let mmap_arc = match self.clone_model_bytes() {
311            Some(a) => a,
312            None => return false,
313        };
314        let data: &[u8] = &mmap_arc;
315        let mut tensors: Vec<(u64, usize, usize, &[u8])> = Vec::new();
316        for e in &q.entries {
317            if e.dtype as u32 != crate::ternary::GGML_TYPE_TERNARY_158 {
318                continue;
319            }
320            let (n_in, n_out) = (e.dimensions[0] as usize, e.dimensions[1] as usize);
321            let (off, len) = (e.blob_offset as usize, e.blob_size as usize);
322            if n_in == 0 || n_out == 0 || off + len > data.len() {
323                continue;
324            }
325            // key = the P64 blob offset == the synthetic index's GgufTensorInfo::byte_offset.
326            tensors.push((e.blob_offset as u64, n_in, n_out, &data[off..off + len]));
327        }
328        if tensors.is_empty() {
329            return false;
330        }
331        match crate::ternary_gpu::TernaryFfnResident::build(
332            self.gpu_device(),
333            self.gpu_queue(),
334            &tensors,
335        ) {
336            Some(r) => {
337                log::info!(
338                    "LLM_LOAD|ternary-ffn|0.71|resident 2-bit FFN: {} tensors, {:.1} MB",
339                    r.len(),
340                    r.resident_bytes() as f64 / (1024.0 * 1024.0)
341                );
342                self.ternary_ffn = Some(r);
343                true
344            }
345            None => false,
346        }
347    }
348
349    /// Boot from an already-mapped P64 weight container (native). Mirrors the GGUF
350    /// `adopt_resident_mmap` but for the `P64` format: validates + builds a synthetic GGUF index
351    /// from the manifest, points the byte source at the P64 bytes (`tensor_data_start = 0`,
352    /// absolute blob offsets), reserves the GEMM/KV arenas, makes the (verbatim) output projection
353    /// resident, and builds the resident 2-bit ternary-FFN dispatcher from the FFN blobs. The
354    /// attention/norm/embed tensors stay at source precision and run the standard GGUF hot path.
355    #[cfg(not(target_arch = "wasm32"))]
356    pub fn adopt_resident_p64_mmap(
357        &mut self,
358        mmap: Arc<memmap2::Mmap>,
359    ) -> Result<GgufLoadReport, String> {
360        let file_size = mmap.len();
361        if file_size == 0 {
362            return Err("Empty P64 mmap".to_string());
363        }
364        let q = crate::p64_weight::P64TensorIndex::from_p64(&mmap[..])?;
365        let index = q.to_gguf_index();
366        let hp = index.hyperparams;
367        if hp.n_layer == 0 || hp.n_embd == 0 {
368            return Err("P64: missing hyperparameters in header".to_string());
369        }
370        if let Err(msg) = hp.decode_supported() {
371            log::error!("LLM_LOAD|failed|1.00|{}", msg);
372            return Err(msg);
373        }
374        self.init_ssm_state_with_budget(&index, self.ssm_state_byte_budget)
375            .map_err(|e| e.to_string())?;
376        self.hyperparams = hp;
377        self.tensor_data_offset = 0; // P64 blob offsets are absolute
378        #[cfg(feature = "gpu-runtime")]
379        {
380            let vocab_chunk_bytes = index
381                .logits_projection_info()
382                .and_then(|info| {
383                    let (_, vocab) = Self::matmul_dims(info);
384                    crate::ggml_quants::tensor_byte_len(&info)
385                        .and_then(|tb| tb.checked_div(vocab.max(1)))
386                })
387                .map(|rb| rb.saturating_mul(VOCAB_CHUNK_ROWS))
388                .unwrap_or(0);
389            let staging = index
390                .max_layer_tensor_bytes
391                .max(vocab_chunk_bytes)
392                .max(4096)
393                .min(MAX_WGPU_WEIGHT_STAGING);
394            self.ensure_gemm_buffers(staging, MAX_STACK_GEMM_OUT as u32);
395        }
396        self.ensure_kv_cache(&hp);
397        if self.kv_layout.is_none() || self.kv_cache_cpu.is_none() {
398            return Err("P64: KV cache allocation failed".to_string());
399        }
400        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
401        if self.prepared_native_bytes.is_some()
402            || !self
403                .gguf_mmap
404                .as_ref()
405                .is_some_and(|old| Arc::ptr_eq(old, &mmap))
406        {
407            self.invalidate_resident_projection_weights();
408        }
409        #[cfg(not(target_arch = "wasm32"))]
410        {
411            self.prepared_native_bytes = None;
412            self.prepared_native_sources = None;
413            self.qwen35_ftw = None;
414            self.qwen35_gguf = None;
415            self.qwen35_gguf_required = false;
416            self.gemma4 = None;
417            self.gemma4_required = false;
418            self.ftw_package = None;
419        }
420        self.gguf_mmap = Some(mmap);
421        // Cache index so decode never re-validates hundreds of MB of tensor CRCs.
422        self.p64_index = Some(q.clone());
423        self.tensor_index_cache = Some(index.clone());
424        #[cfg(feature = "gpu-runtime")]
425        if !self.mc8_upload_resident_logits(&index) {
426            log::info!("LLM_LOAD|p64-logits|0.70|skipped — per-token upload fallback");
427        }
428        #[cfg(feature = "gpu-runtime")]
429        if !self.build_ternary_ffn_resident(&q) {
430            log::info!(
431                "LLM_LOAD|ternary-ffn|0.71|no resident set (no ternary FFN or build failed) — CPU oracle path"
432            );
433        }
434        let kv_cache_bytes = self.kv_cache_bytes();
435        Ok(GgufLoadReport {
436            mapped_bytes: file_size as u64,
437            tensor_data_offset: 0,
438            n_layer: hp.n_layer,
439            n_head: hp.n_head,
440            n_kv_head: hp.effective_n_kv_head(),
441            max_tensor_bytes: index.max_tensor_bytes,
442            kv_cache_bytes,
443            directml_enabled: {
444                #[cfg(target_os = "windows")]
445                {
446                    self.dml.is_some()
447                }
448                #[cfg(not(target_os = "windows"))]
449                {
450                    false
451                }
452            },
453        })
454    }
455
456    /// Compatibility alias for the historical pre-P64 API name.
457    #[deprecated(note = "use adopt_resident_p64_mmap")]
458    #[cfg(not(target_arch = "wasm32"))]
459    pub fn adopt_resident_q42_mmap(
460        &mut self,
461        mmap: Arc<memmap2::Mmap>,
462    ) -> Result<GgufLoadReport, String> {
463        self.adopt_resident_p64_mmap(mmap)
464    }
465
466    /// A1b: number of resident ternary FFN tensors (0 unless a ternary P64 was adopted). Lets a
467    /// test confirm the GPU resident path is actually populated (not a silent CPU-only fallback).
468    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
469    pub fn ternary_ffn_resident_len(&self) -> usize {
470        self.ternary_ffn.as_ref().map_or(0, |r| r.len())
471    }
472
473    /// Attach an already-mapped resident GGUF (shared with orchestrator slot).
474    #[cfg(not(target_arch = "wasm32"))]
475    pub fn adopt_resident_mmap(
476        &mut self,
477        mmap: Arc<memmap2::Mmap>,
478    ) -> Result<GgufLoadReport, String> {
479        let file_size = mmap.len();
480        if file_size == 0 {
481            return Err("Empty GGUF mmap".to_string());
482        }
483        log::info!(
484            "LLM_LOAD|resident-mmap|0.68|Reusing resident GGUF mapping ({:.2} GiB)",
485            bytes_to_gib(file_size as u64)
486        );
487        let index = crate::gguf_sharder::GgufTensorIndex::from_gguf(mmap.as_ref());
488        if index.tensor_data_start == 0
489            && index.max_tensor_bytes == 0
490            && index.hyperparams.n_layer == 0
491        {
492            return Err("GGUF header parse failed or yielded no tensor metadata".to_string());
493        }
494        self.init_ssm_state_with_budget(&index, self.ssm_state_byte_budget)
495            .map_err(|e| e.to_string())?;
496        self.tensor_data_offset = index.tensor_data_start;
497        self.hyperparams = index.hyperparams;
498        #[cfg(feature = "gpu-runtime")]
499        {
500            let vocab_chunk_bytes = index
501                .logits_projection_info()
502                .and_then(|info| {
503                    let (_, vocab) = Self::matmul_dims(info);
504                    crate::ggml_quants::tensor_byte_len(&info)
505                        .and_then(|tb| tb.checked_div(vocab.max(1)))
506                })
507                .map(|rb| rb.saturating_mul(VOCAB_CHUNK_ROWS))
508                .unwrap_or(0);
509            let staging = index
510                .max_layer_tensor_bytes
511                .max(vocab_chunk_bytes)
512                .max(4096)
513                .min(MAX_WGPU_WEIGHT_STAGING);
514            self.ensure_gemm_buffers(staging, MAX_STACK_GEMM_OUT as u32);
515        }
516        self.ensure_kv_cache(&index.hyperparams);
517        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
518        if self.prepared_native_bytes.is_some()
519            || !self
520                .gguf_mmap
521                .as_ref()
522                .is_some_and(|old| Arc::ptr_eq(old, &mmap))
523        {
524            self.invalidate_resident_projection_weights();
525        }
526        #[cfg(not(target_arch = "wasm32"))]
527        {
528            self.prepared_native_bytes = None;
529            self.prepared_native_sources = None;
530            self.qwen35_ftw = None;
531            self.qwen35_gguf = None;
532            self.qwen35_gguf_required = false;
533            self.gemma4 = None;
534            self.gemma4_required = false;
535            self.ftw_package = None;
536        }
537        self.gguf_mmap = Some(mmap);
538        self.p64_index = None;
539        self.tensor_index_cache = Some(index.clone());
540        // A1a step-2: make the output/logits projection resident (upload once) so the per-token
541        // top-k decode binds per-chunk 256-aligned sub-ranges instead of re-uploading the whole
542        // ~47 MB matrix every token (the documented decode throughput killer). Fail-soft: a false
543        // return leaves `mc8_logits_resident_buf=None` and the decode keeps its per-token upload.
544        #[cfg(feature = "gpu-runtime")]
545        if !self.mc8_upload_resident_logits(&index) {
546            log::info!("LLM_LOAD|resident-logits|0.70|skipped — per-token upload fallback");
547        }
548        let kv_cache_bytes = self.kv_cache_bytes();
549        Ok(GgufLoadReport {
550            mapped_bytes: file_size as u64,
551            tensor_data_offset: self.tensor_data_offset,
552            n_layer: self.hyperparams.n_layer,
553            n_head: self.hyperparams.n_head,
554            n_kv_head: self.hyperparams.effective_n_kv_head(),
555            max_tensor_bytes: index.max_tensor_bytes,
556            kv_cache_bytes,
557            directml_enabled: {
558                #[cfg(target_os = "windows")]
559                {
560                    self.dml.is_some()
561                }
562                #[cfg(not(target_os = "windows"))]
563                {
564                    false
565                }
566            },
567        })
568    }
569
570    /// A1a step-2 (native port of Phase 5.3): upload the output/logits projection (tied
571    /// `token_embd`) to a resident `STORAGE` buffer **once**, so the per-token top-k decode binds
572    /// per-chunk 256-aligned sub-ranges instead of re-uploading the whole ~47 MB matrix every
573    /// token (the decode throughput killer). Idempotent. Returns false (→ per-token upload
574    /// fallback) if the projection is missing or its bytes don't divide evenly into rows.
575    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
576    pub(crate) fn mc8_upload_resident_logits(
577        &mut self,
578        index: &crate::gguf_sharder::GgufTensorIndex,
579    ) -> bool {
580        if self.mc8_logits_resident_buf.is_some() {
581            return true;
582        }
583        let info = match index.logits_projection_info() {
584            Some(i) => i,
585            None => return false,
586        };
587        let (_, vocab) = Self::matmul_dims(info);
588        if vocab == 0 {
589            return false;
590        }
591        // Clone the Arc so the mmap borrow does not block mutating `self` below.
592        let mmap_arc = match self.clone_model_bytes() {
593            Some(a) => a,
594            None => return false,
595        };
596        let mmap: &[u8] = &mmap_arc;
597        let raw = match crate::ggml_quants::fetch_tensor_bytes(mmap, index.tensor_data_start, info)
598        {
599            Ok(s) => s,
600            Err(_) => return false,
601        };
602        let total = raw.len();
603        if total == 0 || total % vocab != 0 {
604            return false;
605        }
606        let row_bytes = total / vocab;
607        // VOCAB_CHUNK_ROWS is a multiple of 256, so every chunk's byte offset
608        // (chunk_idx * VOCAB_CHUNK_ROWS * row_bytes) is 256-aligned for the storage binding.
609        let buf = self.gpu_device().create_buffer(&etalon::BufferDescriptor {
610            label: Some("ResidentLogits"),
611            size: total as u64,
612            usage: etalon::BufferUsages::STORAGE | etalon::BufferUsages::COPY_DST,
613            mapped_at_creation: false,
614        });
615        self.gpu_queue().write_buffer(&buf, 0, raw);
616        self.mc8_logits_resident_buf = Some(buf);
617        self.mc8_logits_row_bytes = row_bytes as u32;
618        log::info!(
619            "LLM_LOAD|resident-logits|0.70|output projection resident once: {:.1} MB ({} rows x {} B)",
620            total as f64 / (1024.0 * 1024.0),
621            vocab,
622            row_bytes
623        );
624        true
625    }
626
627    /// Decode-profiler: blocking GPU fence wait + round-trip counter. Every native sync point routes
628    /// through this (via the `self.gpu_device().poll(Maintain::Wait)` → `self.poll_wait()` rewrite),
629    /// so the bench can count submit→wait round-trips per token and separate synchronization stall
630    /// from real kernel time. Behaviourally identical to a bare blocking poll.
631    #[inline]
632    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
633    pub(crate) fn poll_wait(&self) {
634        let _ = self
635            .gpu_device()
636            .poll(etalon::PollType::wait_indefinitely());
637        GPU_WAIT_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
638    }
639
640    /// Decode-profiler: wall-clock for `n` EMPTY `submit → poll(Maintain::Wait)` round-trips (no
641    /// compute dispatched). Isolates the fixed CPU↔GPU fence latency: if a token's forward time ≈
642    /// (its round-trip count × this per-round-trip cost), the bottleneck is synchronization, not
643    /// math; if forward ≫ that, the kernels themselves are slow. Does NOT touch `GPU_WAIT_COUNT`.
644    #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
645    pub fn bench_empty_submit_roundtrip(&self, n: u32) -> u64 {
646        let t = std::time::Instant::now();
647        for _ in 0..n {
648            let enc = self
649                .device()
650                .create_command_encoder(&etalon::CommandEncoderDescriptor {
651                    label: Some("EmptyRT"),
652                });
653            self.submit_commands(Some(enc.finish()));
654            let _ = self
655                .gpu_device()
656                .poll(etalon::PollType::wait_indefinitely());
657        }
658        t.elapsed().as_nanos() as u64
659    }
660    #[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
661    /// `QTensorEngine::adopt_resident_mmap`.
662    pub fn adopt_resident_mmap(&mut self, mmap: Arc<[u8]>) -> Result<GgufLoadReport, String> {
663        let file_size = mmap.len();
664        if file_size == 0 {
665            return Err("Empty GGUF mmap".to_string());
666        }
667        log::info!(
668            "LLM_LOAD|resident-mmap|0.68|Reusing resident GGUF mapping ({:.2} GiB)",
669            bytes_to_gib(file_size as u64)
670        );
671        let index = crate::gguf_sharder::GgufTensorIndex::from_gguf(mmap.as_ref());
672        if index.tensor_data_start == 0
673            && index.max_tensor_bytes == 0
674            && index.hyperparams.n_layer == 0
675        {
676            return Err("GGUF header parse failed or yielded no tensor metadata".to_string());
677        }
678        self.init_ssm_state_with_budget(&index, self.ssm_state_byte_budget)
679            .map_err(|e| e.to_string())?;
680        self.tensor_data_offset = index.tensor_data_start;
681        self.hyperparams = index.hyperparams;
682        let staging = index
683            .max_layer_tensor_bytes
684            .max(4096)
685            .min(MAX_WGPU_WEIGHT_STAGING);
686        self.ensure_gemm_buffers(staging, MAX_STACK_GEMM_OUT as u32);
687        self.ensure_kv_cache(&index.hyperparams);
688        if self.kv_layout.is_none() || self.kv_cache_cpu.is_none() {
689            return Err("KV cache allocation failed (layout or CPU mirror missing)".to_string());
690        }
691        #[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
692        if self.prepared_native_bytes.is_some()
693            || !self
694                .gguf_mmap
695                .as_ref()
696                .is_some_and(|old| Arc::ptr_eq(old, &mmap))
697        {
698            self.invalidate_resident_projection_weights();
699        }
700        #[cfg(not(target_arch = "wasm32"))]
701        {
702            self.prepared_native_bytes = None;
703            self.prepared_native_sources = None;
704            self.qwen35_ftw = None;
705            self.qwen35_gguf = None;
706            self.qwen35_gguf_required = false;
707            self.gemma4 = None;
708            self.gemma4_required = false;
709            self.ftw_package = None;
710        }
711        self.gguf_mmap = Some(mmap);
712        // Full eager upload — required for coherent decode (do not defer).
713        // REVIEW(wasm-mobile-2026-08-02 F6): accelerated/benchmark mode needs a
714        // fail-closed residency contract and receipt. Continuing here can silently
715        // demote the engine into the historical per-forward upload performance floor.
716        if !self.mc8_upload_all_resident_weights(&index) {
717            wlog("[MC8] eager resident weight upload skipped at init — will retry lazily");
718        }
719        if !self.mc8_upload_resident_logits(&index) {
720            wlog("[MC8] resident logits projection skipped at init — per-token upload fallback");
721        }
722        if !self.mc8_upload_resident_norms(&index) {
723            wlog("[MC8] resident norm weights skipped at init — per-layer upload fallback");
724        }
725        let kv_cache_bytes = self.kv_cache_bytes();
726        Ok(GgufLoadReport {
727            mapped_bytes: file_size as u64,
728            tensor_data_offset: self.tensor_data_offset,
729            n_layer: self.hyperparams.n_layer,
730            n_head: self.hyperparams.n_head,
731            n_kv_head: self.hyperparams.effective_n_kv_head(),
732            max_tensor_bytes: index.max_tensor_bytes,
733            kv_cache_bytes,
734            directml_enabled: false,
735        })
736    }
737}