1use crate::llm_agent::{AgentBackend, LocalLlmAgent};
7
8use super::metrics::tok_per_s;
9use super::*;
10
11pub fn compare_topk_decode(
15 model_path: &str,
16 prompt: &str,
17 decode_tokens: u32,
18) -> Result<(String, String), String> {
19 if !std::path::Path::new(model_path).exists() {
20 return Err(format!("model not found: {model_path}"));
21 }
22 let agent = LocalLlmAgent::with_local_backend(
23 "did:qualia:bench",
24 AgentBackend::Local {
25 model_path: model_path.to_string(),
26 context_window: 4096,
27 quantization: "auto".into(),
28 vision_projector_path: None,
29 modality: "text".into(),
30 architecture: None,
31 },
32 );
33 let _benchmark_budget = benchmark_decode_budget_scope(decode_tokens);
34 let model_id = crate::q_hash(model_path);
35 let _ = crate::resident_model::mount_resident_gguf(model_id, model_path, false);
36
37 set_gpu_topk(false);
38 let (off_text, _, _, _) = agent.infer_local_model_with_execution::<fn(String)>(
39 prompt,
40 "",
41 crate::llm_agent::NativeDecodeRequest::Developer,
42 None,
43 None,
44 );
45 set_gpu_topk(true);
46 let (on_text, _, _, _) = agent.infer_local_model_with_execution::<fn(String)>(
47 prompt,
48 "",
49 crate::llm_agent::NativeDecodeRequest::Developer,
50 None,
51 None,
52 );
53
54 set_gpu_topk(false);
55 crate::resident_model::clear_resident_model();
56 Ok((off_text, on_text))
57}
58
59pub fn compare_topk_decode_blocking(
61 model_path: &str,
62 prompt: &str,
63 decode_tokens: u32,
64) -> Result<(String, String), String> {
65 let rt = tokio::runtime::Builder::new_multi_thread()
66 .worker_threads(2)
67 .enable_all()
68 .build()
69 .map_err(|e| e.to_string())?;
70 rt.block_on(async { compare_topk_decode(model_path, prompt, decode_tokens) })
71}
72
73#[cfg(not(target_arch = "wasm32"))]
78pub fn decode_with_metrics(
79 model_path: &str,
80 prompt: &str,
81 decode_tokens: u32,
82) -> Result<(String, f64), String> {
83 if !std::path::Path::new(model_path).exists() {
84 return Err(format!("model not found: {model_path}"));
85 }
86 let is_q42 = {
87 use std::io::Read;
88 let mut buf = [0u8; 4];
89 std::fs::File::open(model_path)
90 .and_then(|mut f| f.read_exact(&mut buf))
91 .map(|_| &buf == b"p64\0")
92 .unwrap_or(false)
93 };
94 let agent = LocalLlmAgent::with_local_backend(
95 "did:qualia:bench",
96 AgentBackend::Local {
97 model_path: model_path.to_string(),
98 context_window: 4096,
99 quantization: "auto".into(),
100 vision_projector_path: None,
101 modality: "text".into(),
102 architecture: None,
103 },
104 );
105 let _benchmark_budget = benchmark_decode_budget_scope(decode_tokens);
106 let model_id = crate::q_hash(model_path);
107 if is_q42 {
108 crate::resident_model::mount_resident_q42(model_id, model_path, false)?;
109 } else {
110 let _ = crate::resident_model::mount_resident_gguf(model_id, model_path, false);
111 }
112 reset_phase_metrics();
113 let (text, _, _, _) = agent.infer_local_model_with_execution::<fn(String)>(
114 prompt,
115 "",
116 crate::llm_agent::NativeDecodeRequest::Developer,
117 None,
118 None,
119 );
120 let snap = phase_snapshot();
121 let decode_tok_s = tok_per_s(snap.decode_tokens, snap.decode_ns);
122 crate::resident_model::clear_resident_model();
123 Ok((text, decode_tok_s))
124}
125
126#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
133pub fn gemm_parity_probe_blocking(
134 n_in: usize,
135 n_out: usize,
136 seed: u64,
137) -> Result<(f32, f64, u64, u64), String> {
138 use crate::gguf_sharder::GgufTensorInfo;
139 if n_in == 0 || n_out == 0 || n_in % 32 != 0 {
140 return Err("n_in must be a non-zero multiple of 32 (Q8_0 block size); n_out > 0".into());
141 }
142 let rt = tokio::runtime::Builder::new_multi_thread()
143 .worker_threads(2)
144 .enable_all()
145 .build()
146 .map_err(|e| e.to_string())?;
147 let mut engine = rt
148 .block_on(crate::gguf_bridge::QTensorEngine::try_new())
149 .map_err(|e| format!("engine init: {e}"))?;
150 let _guard = rt.enter(); let mut s = seed | 1;
154 let mut rng = move || -> f32 {
155 s = s
156 .wrapping_mul(6364136223846793005)
157 .wrapping_add(1442695040888963407);
158 ((s >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0
159 };
160
161 let row_bytes = crate::llm_kernel_parity::q8_0_bytes(n_in);
162 let mut raw = vec![0u8; row_bytes * n_out];
163 let mut row_f32 = vec![0f32; n_in];
164 for r in 0..n_out {
165 for x in row_f32.iter_mut() {
166 *x = rng();
167 }
168 if !crate::llm_kernel_parity::quantize_q8_0_from_f32(
169 &row_f32,
170 &mut raw[r * row_bytes..(r + 1) * row_bytes],
171 ) {
172 return Err("q8_0 quantize failed".into());
173 }
174 }
175 let input: Vec<f32> = (0..n_in).map(|_| rng()).collect();
176
177 let info = GgufTensorInfo {
178 dims: [n_in as u64, n_out as u64, 1, 1],
179 n_dims: 2,
180 ggml_type: crate::ggml_quants::GGML_TYPE_Q8_0,
181 byte_offset: 0,
182 };
183
184 crate::llm_gpu_profiler::set_enabled(true);
185 crate::llm_gpu_profiler::reset();
186 let mut gpu_out = vec![0f32; n_out];
187 let mut cpu_out = vec![0f32; n_out];
188 let ok = engine.gemm_parity_probe(&info, &raw, &input, &mut gpu_out, &mut cpu_out, n_in, n_out);
189 let calls = crate::llm_gpu_profiler::snapshot()
190 .iter()
191 .find(|t| matches!(t.phase, crate::llm_gpu_profiler::Phase::Gemm))
192 .map(|t| t.calls)
193 .unwrap_or(0);
194 crate::llm_gpu_profiler::set_enabled(false);
195 if !ok {
196 return Err("gemm_parity_probe: GPU or CPU path returned false".into());
197 }
198 Ok((
199 crate::llm_kernel_parity::max_abs_err(&gpu_out, &cpu_out),
200 crate::llm_kernel_parity::mean_abs_err(&gpu_out, &cpu_out),
201 crate::llm_kernel_parity::max_ulp_diff(&gpu_out, &cpu_out),
202 calls,
203 ))
204}
205
206#[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
207pub fn gemm_parity_probe_blocking(
209 n_in: usize,
210 n_out: usize,
211 seed: u64,
212) -> Result<(f32, f64, u64, u64), String> {
213 let _ = (n_in, n_out, seed);
214 Err("GEMM parity probe requires the `gpu-runtime` feature; refusing to stub GPU success".into())
215}
216
217#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
222pub fn gemm_parity_probe_f16_blocking(
223 n_in: usize,
224 n_out: usize,
225 seed: u64,
226) -> Result<(f32, f64, u64, u64), String> {
227 use crate::gguf_sharder::GgufTensorInfo;
228 if n_in == 0 || n_out == 0 {
229 return Err("n_in and n_out must be > 0".into());
230 }
231 let rt = tokio::runtime::Builder::new_multi_thread()
232 .worker_threads(2)
233 .enable_all()
234 .build()
235 .map_err(|e| e.to_string())?;
236 let mut engine = rt
237 .block_on(crate::gguf_bridge::QTensorEngine::try_new())
238 .map_err(|e| format!("engine init: {e}"))?;
239 let _guard = rt.enter();
240
241 let mut s = seed | 1;
242 let mut rng = move || -> f32 {
243 s = s
244 .wrapping_mul(6364136223846793005)
245 .wrapping_add(1442695040888963407);
246 ((s >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0
247 };
248
249 let row_bytes = crate::llm_kernel_parity::f16_bytes(n_in);
250 let mut raw = vec![0u8; row_bytes * n_out];
251 let mut row_f32 = vec![0f32; n_in];
252 for r in 0..n_out {
253 for x in row_f32.iter_mut() {
254 *x = rng();
255 }
256 if !crate::llm_kernel_parity::quantize_f16_from_f32(
257 &row_f32,
258 &mut raw[r * row_bytes..(r + 1) * row_bytes],
259 ) {
260 return Err("f16 quantize failed".into());
261 }
262 }
263 let input: Vec<f32> = (0..n_in).map(|_| rng()).collect();
264
265 let info = GgufTensorInfo {
266 dims: [n_in as u64, n_out as u64, 1, 1],
267 n_dims: 2,
268 ggml_type: crate::ggml_quants::GGML_TYPE_F16,
269 byte_offset: 0,
270 };
271
272 crate::llm_gpu_profiler::set_enabled(true);
273 crate::llm_gpu_profiler::reset();
274 let mut gpu_out = vec![0f32; n_out];
275 let mut cpu_out = vec![0f32; n_out];
276 let ok = engine.gemm_parity_probe(&info, &raw, &input, &mut gpu_out, &mut cpu_out, n_in, n_out);
277 let calls = crate::llm_gpu_profiler::snapshot()
278 .iter()
279 .find(|t| matches!(t.phase, crate::llm_gpu_profiler::Phase::Gemm))
280 .map(|t| t.calls)
281 .unwrap_or(0);
282 crate::llm_gpu_profiler::set_enabled(false);
283 if !ok {
284 return Err("gemm_parity_probe (f16): GPU or CPU path returned false".into());
285 }
286 Ok((
287 crate::llm_kernel_parity::max_abs_err(&gpu_out, &cpu_out),
288 crate::llm_kernel_parity::mean_abs_err(&gpu_out, &cpu_out),
289 crate::llm_kernel_parity::max_ulp_diff(&gpu_out, &cpu_out),
290 calls,
291 ))
292}
293
294#[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
295pub fn gemm_parity_probe_f16_blocking(
297 n_in: usize,
298 n_out: usize,
299 seed: u64,
300) -> Result<(f32, f64, u64, u64), String> {
301 let _ = (n_in, n_out, seed);
302 Err(
303 "F16 GEMM parity probe requires the `gpu-runtime` feature; refusing to stub GPU success"
304 .into(),
305 )
306}
307
308#[cfg(not(target_arch = "wasm32"))]
316pub fn perplexity_eval_blocking(model_path: &str, max_tok: usize) -> Result<(f64, usize), String> {
317 use crate::gguf_bridge::QTensorEngine;
318 use crate::gguf_sharder::{GgufTensorIndex, GgufTokenizer};
319
320 let corpus = crate::llm_eval::load_corpus().map_err(|e| format!("corpus load: {e}"))?;
321 if corpus.is_empty() {
322 return Err("eval corpus is empty".into());
323 }
324 let model_path = model_path.to_string();
325
326 std::thread::spawn(move || -> Result<(f64, usize), String> {
327 let rt = tokio::runtime::Builder::new_current_thread()
328 .enable_all()
329 .build()
330 .map_err(|e| e.to_string())?;
331 let _g = rt.enter();
332
333 let mut engine = QTensorEngine::new();
334 let mut magic = [0u8; 4];
335 let is_q42 = {
336 use std::io::Read;
337 std::fs::File::open(&model_path)
338 .and_then(|mut f| f.read_exact(&mut magic))
339 .map(|_| &magic == b"p64\0")
340 .unwrap_or(false)
341 };
342 if is_q42 {
343 let f = std::fs::File::open(&model_path).map_err(|e| e.to_string())?;
344 let mmap = unsafe { memmap2::Mmap::map(&f) }.map_err(|e| e.to_string())?;
345 engine
346 .adopt_resident_p64_mmap(std::sync::Arc::new(mmap))
347 .map_err(|e| format!("q42 adopt: {e}"))?;
348 } else {
349 engine.load_gguf(&model_path);
350 }
351
352 let mmap = engine
353 .clone_model_bytes()
354 .ok_or_else(|| "model did not memory-map (load failed)".to_string())?;
355 let is_q42_mmap = mmap.len() >= 4 && mmap[0..4] == *b"p64\0";
356 let tok = if is_q42_mmap {
357 crate::p64_weight::P64TensorIndex::from_p64(&mmap)
358 .ok()
359 .and_then(|qi| GgufTokenizer::from_p64_section(qi.tokenizer_bytes(&mmap)))
360 .unwrap_or_default()
361 } else {
362 GgufTokenizer::from_gguf(&mmap)
363 };
364 let tensor_idx = if is_q42_mmap {
365 crate::p64_weight::P64TensorIndex::from_p64(&mmap)
366 .map(|qi| qi.to_gguf_index())
367 .map_err(|e| format!("q42 index: {e}"))?
368 } else {
369 GgufTensorIndex::from_gguf(&mmap)
370 };
371
372 let emb_dim = tensor_idx.emb_dim();
373 if emb_dim == 0 {
374 return Err("embedding dimension is 0 (tensor index parse failed)".into());
375 }
376 let vocab = tok.vocab_len().max(1) as usize;
377
378 let mut emb_buf = vec![0f32; emb_dim.max(8192)];
379 let mut scratch_a = vec![0f32; 16384];
380 let mut scratch_b = vec![0f32; 16384];
381 let mut logits = vec![0f32; vocab];
382 let mmap_bytes: &[u8] = &mmap;
383
384 let mut total_nll = 0.0f64;
385 let mut total_tok = 0usize;
386 for passage in &corpus {
387 let toks = tok.encode(passage);
388 if toks.len() < 2 {
389 continue;
390 }
391 let limit = if max_tok > 0 {
392 (max_tok + 1).min(toks.len())
393 } else {
394 toks.len()
395 };
396 engine.reset_kv_cache();
397 for i in 0..limit - 1 {
398 let n_emb = tensor_idx.dequantize_token_embedding_into(
399 mmap_bytes,
400 toks[i],
401 &mut emb_buf[..emb_dim],
402 );
403 if n_emb == 0 {
404 return Err(format!("embedding lookup failed for token {}", toks[i]));
405 }
406 crate::llm_awq::begin_forward();
409 let _ = engine.dispatch_transformer_forward(
410 &tensor_idx,
411 &mut emb_buf[..emb_dim],
412 emb_dim,
413 &mut scratch_a,
414 &mut scratch_b,
415 i as u32,
416 0, );
418 let _ =
419 engine.apply_output_norm_inplace(&tensor_idx, &mut emb_buf[..emb_dim], emb_dim);
420 let n = engine.dispatch_output_logits_into(
421 &tensor_idx,
422 &emb_buf[..emb_dim],
423 emb_dim,
424 &mut logits,
425 );
426 if n == 0 {
427 return Err("output projection produced no logits".into());
428 }
429 let nll = crate::llm_eval::token_nll(&logits[..n], toks[i + 1] as usize);
430 if nll.is_finite() {
431 total_nll += nll;
432 total_tok += 1;
433 }
434 }
435 }
436 if total_tok == 0 {
437 return Err("no tokens scored".into());
438 }
439 Ok((crate::llm_eval::perplexity(total_nll, total_tok), total_tok))
440 })
441 .join()
442 .map_err(|_| "perplexity eval thread panicked".to_string())?
443}
444
445#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
453pub fn capture_kv_gpu_readback(
454 model_path: &str,
455 max_tok: usize,
456 max_per_layer: usize,
457) -> Result<crate::kv_capture::KvCapture, String> {
458 use crate::gguf_bridge::QTensorEngine;
459 use crate::gguf_sharder::{GgufTensorIndex, GgufTokenizer};
460
461 let corpus = crate::llm_eval::load_corpus().map_err(|e| format!("corpus load: {e}"))?;
462 if corpus.is_empty() {
463 return Err("eval corpus is empty".into());
464 }
465 let model_path = model_path.to_string();
466
467 std::thread::spawn(move || -> Result<crate::kv_capture::KvCapture, String> {
468 let rt = tokio::runtime::Builder::new_current_thread()
469 .enable_all()
470 .build()
471 .map_err(|e| e.to_string())?;
472 let _g = rt.enter();
473
474 let prev_int8 = kv_int8_enabled();
476 set_kv_int8(false);
477
478 let mut engine = QTensorEngine::new();
479 let mut magic = [0u8; 4];
480 let is_q42 = {
481 use std::io::Read;
482 std::fs::File::open(&model_path)
483 .and_then(|mut f| f.read_exact(&mut magic))
484 .map(|_| &magic == b"p64\0")
485 .unwrap_or(false)
486 };
487 if is_q42 {
488 let f = std::fs::File::open(&model_path).map_err(|e| e.to_string())?;
489 let mmap = unsafe { memmap2::Mmap::map(&f) }.map_err(|e| e.to_string())?;
490 engine
491 .adopt_resident_p64_mmap(std::sync::Arc::new(mmap))
492 .map_err(|e| format!("q42 adopt: {e}"))?;
493 } else {
494 engine.load_gguf(&model_path);
495 }
496
497 let mmap = engine
498 .clone_model_bytes()
499 .ok_or_else(|| "model did not memory-map (load failed)".to_string())?;
500 let is_q42_mmap = mmap.len() >= 4 && mmap[0..4] == *b"p64\0";
501 let tok = if is_q42_mmap {
502 crate::p64_weight::P64TensorIndex::from_p64(&mmap)
503 .ok()
504 .and_then(|qi| GgufTokenizer::from_p64_section(qi.tokenizer_bytes(&mmap)))
505 .unwrap_or_default()
506 } else {
507 GgufTokenizer::from_gguf(&mmap)
508 };
509 let tensor_idx = if is_q42_mmap {
510 crate::p64_weight::P64TensorIndex::from_p64(&mmap)
511 .map(|qi| qi.to_gguf_index())
512 .map_err(|e| format!("q42 index: {e}"))?
513 } else {
514 GgufTensorIndex::from_gguf(&mmap)
515 };
516
517 let emb_dim = tensor_idx.emb_dim();
518 if emb_dim == 0 {
519 return Err("embedding dimension is 0".into());
520 }
521 let mut emb_buf = vec![0f32; emb_dim.max(8192)];
522 let mut scratch_a = vec![0f32; 16384];
523 let mut scratch_b = vec![0f32; 16384];
524 let mmap_bytes: &[u8] = &mmap;
525
526 let mut acc_k: Vec<Vec<Vec<f32>>> = Vec::new();
527 let mut acc_v: Vec<Vec<Vec<f32>>> = Vec::new();
528 let mut head_dim = 0usize;
529
530 'corpus: for passage in &corpus {
531 let toks = tok.encode(passage);
532 if toks.len() < 2 {
533 continue;
534 }
535 let limit = if max_tok > 0 {
536 (max_tok + 1).min(toks.len())
537 } else {
538 toks.len()
539 };
540 engine.reset_kv_cache();
541 for i in 0..limit - 1 {
542 let n_emb = tensor_idx.dequantize_token_embedding_into(
543 mmap_bytes,
544 toks[i],
545 &mut emb_buf[..emb_dim],
546 );
547 if n_emb == 0 {
548 return Err(format!("embedding lookup failed for token {}", toks[i]));
549 }
550 let _ = engine.dispatch_transformer_forward(
551 &tensor_idx,
552 &mut emb_buf[..emb_dim],
553 emb_dim,
554 &mut scratch_a,
555 &mut scratch_b,
556 i as u32,
557 0,
558 );
559 }
560 if let Some(cap) = engine.capture_kv_f32((limit - 1) as u32, max_per_layer) {
562 head_dim = cap.head_dim;
563 if acc_k.is_empty() {
564 acc_k = vec![Vec::new(); cap.k.len()];
565 acc_v = vec![Vec::new(); cap.v.len()];
566 }
567 let mut all_full = true;
568 for l in 0..cap.k.len().min(acc_k.len()) {
569 for vk in &cap.k[l] {
570 if acc_k[l].len() < max_per_layer {
571 acc_k[l].push(vk.clone());
572 }
573 }
574 for vv in &cap.v[l] {
575 if acc_v[l].len() < max_per_layer {
576 acc_v[l].push(vv.clone());
577 }
578 }
579 if acc_k[l].len() < max_per_layer {
580 all_full = false;
581 }
582 }
583 if all_full {
584 break 'corpus; }
586 }
587 }
588
589 set_kv_int8(prev_int8);
590 if head_dim == 0 {
591 return Err("no KV captured via GPU readback (int8 layout, or empty forward)".into());
592 }
593 Ok(crate::kv_capture::KvCapture {
594 head_dim,
595 k: acc_k,
596 v: acc_v,
597 })
598 })
599 .join()
600 .map_err(|_| "kv capture thread panicked".to_string())?
601}
602
603#[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
604pub fn capture_kv_gpu_readback(
606 model_path: &str,
607 max_tok: usize,
608 max_per_layer: usize,
609) -> Result<crate::kv_capture::KvCapture, String> {
610 let _ = (model_path, max_tok, max_per_layer);
611 Err("GPU KV readback requires the `gpu-runtime` feature; refusing to stub GPU success".into())
612}
613
614#[cfg(not(target_arch = "wasm32"))]
621pub fn awq_sweep_blocking(
622 gguf_path: &str,
623 alphas: &[f32],
624 max_tok: usize,
625 quant: crate::p64_weight::FfnQuant,
626) -> Result<(f64, Vec<(f32, f64, f64)>), String> {
627 use crate::p64_weight::compile_gguf_to_q42_ffn_quant_awq;
628
629 let bytes = std::fs::read(gguf_path).map_err(|e| format!("read gguf: {e}"))?;
630 let idx = crate::gguf_sharder::GgufTensorIndex::from_gguf(&bytes);
631 let n_layer = idx.hyperparams.n_layer;
632 let n_embd = idx.hyperparams.n_embd;
633 if n_layer == 0 || n_embd == 0 {
634 return Err("gguf parse failed (n_layer/n_embd = 0)".into());
635 }
636
637 set_ternary_ffn(false);
639 crate::llm_awq::enable(n_layer, n_embd)?;
640 let (ref_ppl, _) = perplexity_eval_blocking(gguf_path, max_tok)?;
641 let stats = crate::llm_awq::snapshot();
642 crate::llm_awq::disable();
643 if stats.is_empty() {
644 return Err("AWQ: no activation stats captured".into());
645 }
646
647 set_ternary_ffn(matches!(quant, crate::p64_weight::FfnQuant::Ternary));
650 let tmp = std::env::temp_dir();
651 let mut results = Vec::with_capacity(alphas.len());
652 for &alpha in alphas {
653 let scales = if alpha == 0.0 {
654 None
655 } else {
656 Some(stats.as_slice())
657 };
658 let q42 = compile_gguf_to_q42_ffn_quant_awq(&bytes, 14, scales, alpha, quant)
659 .map_err(|e| format!("AWQ compile (alpha={alpha}): {e}"))?;
660 let path = tmp.join(format!("awq_sweep_a{:.2}.q42", alpha));
661 std::fs::write(&path, &q42).map_err(|e| format!("write q42: {e}"))?;
662 let ps = path.to_string_lossy().to_string();
663 let (ppl, _) = perplexity_eval_blocking(&ps, max_tok)?;
664 let (text, _) = decode_with_metrics_blocking(&ps, "Once upon a time, there was a", 24)?;
665 let uniq = crate::llm_eval::unique_word_ratio(&text);
666 let _ = std::fs::remove_file(&path);
667 results.push((alpha, ppl, uniq));
668 }
669 set_ternary_ffn(false);
670 Ok((ref_ppl, results))
671}
672
673#[cfg(not(target_arch = "wasm32"))]
675pub fn decode_with_metrics_blocking(
676 model_path: &str,
677 prompt: &str,
678 decode_tokens: u32,
679) -> Result<(String, f64), String> {
680 let rt = tokio::runtime::Builder::new_multi_thread()
681 .worker_threads(2)
682 .enable_all()
683 .build()
684 .map_err(|e| e.to_string())?;
685 rt.block_on(async { decode_with_metrics(model_path, prompt, decode_tokens) })
686}
687
688#[cfg(not(target_arch = "wasm32"))]
691pub fn decode_sampled_blocking(
692 model_path: &str,
693 prompt: &str,
694 decode_tokens: u32,
695 cfg: crate::sampler::SamplerConfig,
696) -> Result<(String, f64), String> {
697 set_sampler_config(Some(cfg));
698 let out = decode_with_metrics_blocking(model_path, prompt, decode_tokens);
699 set_sampler_config(None);
700 out
701}
702
703#[cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
713pub fn spec_verify_probe_blocking(
714 model_path: &str,
715 prompt: &str,
716 b: usize,
717) -> Result<(Vec<u32>, Vec<u32>, Vec<u32>), String> {
718 use crate::gguf_bridge::QTensorEngine;
719 use crate::gguf_sharder::{GgufTensorIndex, GgufTokenizer};
720 if !std::path::Path::new(model_path).exists() {
721 return Err(format!("model not found: {model_path}"));
722 }
723 let model_path = model_path.to_string();
724 let prompt = prompt.to_string();
725
726 std::thread::spawn(move || -> Result<(Vec<u32>, Vec<u32>, Vec<u32>), String> {
727 let rt = tokio::runtime::Builder::new_current_thread()
728 .enable_all()
729 .build()
730 .map_err(|e| e.to_string())?;
731 let _g = rt.enter();
732
733 let mut engine = QTensorEngine::new();
734 engine.load_gguf(&model_path);
735 let mmap = engine
736 .clone_model_bytes()
737 .ok_or_else(|| "model did not memory-map".to_string())?;
738 let tok = GgufTokenizer::from_gguf(&mmap);
739 let idx = GgufTensorIndex::from_gguf(&mmap);
740 if !engine.mc8_upload_resident_logits(&idx) {
743 return Err("resident logits upload failed (verify tail needs it)".into());
744 }
745 let emb_dim = idx.emb_dim();
746 if emb_dim == 0 {
747 return Err("embedding dimension is 0".into());
748 }
749 let vocab = tok.vocab_len().max(1) as usize;
750 let toks = tok.encode(&prompt);
751 if toks.len() < b + 2 {
752 return Err(format!(
753 "prompt too short: {} tokens, need >= {}",
754 toks.len(),
755 b + 2
756 ));
757 }
758
759 let mut emb = vec![0f32; emb_dim.max(8192)];
760 let mut sa = vec![0f32; 16384];
761 let mut sb = vec![0f32; 16384];
762 let mut logits = vec![0f32; vocab];
763 let mmap_b: &[u8] = &mmap;
764
765 let argmax = |v: &[f32]| -> u32 {
766 let mut best_i = 0u32;
767 let mut best_v = f32::NEG_INFINITY;
768 for (i, &x) in v.iter().enumerate() {
769 if x > best_v {
770 best_v = x;
771 best_i = i as u32;
772 }
773 }
774 best_i
775 };
776
777 let p = toks.len() - 1;
779 engine.reset_kv_cache();
780 for i in 0..p {
781 let n = idx.dequantize_token_embedding_into(mmap_b, toks[i], &mut emb[..emb_dim]);
782 if n == 0 {
783 return Err(format!("embedding lookup failed for token {}", toks[i]));
784 }
785 let _ = engine.dispatch_transformer_forward(
786 &idx,
787 &mut emb[..emb_dim],
788 emb_dim,
789 &mut sa,
790 &mut sb,
791 i as u32,
792 0,
793 );
794 }
795 let mut inputs: Vec<u32> = Vec::with_capacity(b);
802 let mut reference: Vec<u32> = Vec::with_capacity(b);
803 let mut input = toks[p];
804 let mut pos = p as u32;
805 for _ in 0..b {
806 inputs.push(input);
807 let n = idx.dequantize_token_embedding_into(mmap_b, input, &mut emb[..emb_dim]);
808 if n == 0 {
809 return Err("reference embedding lookup failed".into());
810 }
811 let _ = engine.dispatch_transformer_forward(
812 &idx,
813 &mut emb[..emb_dim],
814 emb_dim,
815 &mut sa,
816 &mut sb,
817 pos,
818 0,
819 );
820 let _ = engine.apply_output_norm_inplace(&idx, &mut emb[..emb_dim], emb_dim);
821 let nl =
822 engine.dispatch_output_logits_into(&idx, &emb[..emb_dim], emb_dim, &mut logits);
823 if nl == 0 {
824 return Err("reference output projection produced no logits".into());
825 }
826 let out = argmax(&logits[..nl]);
827 reference.push(out);
828 input = out;
829 pos += 1;
830 }
831
832 let mut verify_out: Vec<u32> = Vec::new();
834 let mut verify_logit: Vec<f32> = Vec::new();
835 engine
836 .verify_draft_batch(&idx, &inputs, p as u32, &mut verify_out, &mut verify_logit)
837 .ok_or_else(|| "verify_draft_batch ineligible/failed".to_string())?;
838
839 engine.reset_kv_cache();
844 for i in 0..p {
845 let n = idx.dequantize_token_embedding_into(mmap_b, toks[i], &mut emb[..emb_dim]);
846 if n == 0 {
847 return Err("resident-ref prefill embedding failed".into());
848 }
849 let _ = engine.dispatch_transformer_forward(
850 &idx,
851 &mut emb[..emb_dim],
852 emb_dim,
853 &mut sa,
854 &mut sb,
855 i as u32,
856 0,
857 );
858 }
859 let mut resident_out: Vec<u32> = Vec::with_capacity(b);
860 for (j, &t) in inputs.iter().enumerate() {
861 let n = idx.dequantize_token_embedding_into(mmap_b, t, &mut emb[..emb_dim]);
862 if n == 0 {
863 return Err("resident-ref embedding failed".into());
864 }
865 match engine.dispatch_token_forward_resident(&idx, &emb[..emb_dim], (p + j) as u32) {
866 Some(r) => resident_out.push(r.best_token_id),
867 None => resident_out.push(u32::MAX),
868 }
869 }
870
871 Ok((reference, verify_out, resident_out))
872 })
873 .join()
874 .map_err(|_| "spec-verify probe thread panicked".to_string())?
875}
876
877#[cfg(all(not(target_arch = "wasm32"), not(feature = "gpu-runtime")))]
878pub fn spec_verify_probe_blocking(
880 model_path: &str,
881 prompt: &str,
882 b: usize,
883) -> Result<(Vec<u32>, Vec<u32>, Vec<u32>), String> {
884 let _ = (model_path, prompt, b);
885 Err(
886 "speculative-verify probe requires the `gpu-runtime` feature; refusing to stub GPU success"
887 .into(),
888 )
889}