qualia_cli/llm_raw_bench/
command.rs1use std::path::Path;
2use std::time::{Duration, SystemTime};
3
4use qualia_core_db::inference::inference_bench::raw_decode::{
5 run_raw_decode_blocking, RawDecodeConfig,
6};
7use qualia_core_db::inference::runtime::{
8 cleanup_stale_runs, ArtifactCleanupCounters, ArtifactRetention, BackendKind, RunArtifactDir,
9};
10
11pub struct CommandConfig<'a> {
12 pub model: &'a Path,
13 pub steps: u32,
14 pub warmups: u16,
15 pub runs: u16,
16 pub quantization: &'a str,
17 pub prompt: &'a str,
18 pub target_prompt_tokens: Option<u32>,
19 pub retain_artifacts: Option<&'a Path>,
20}
21
22pub fn run(command: CommandConfig<'_>) -> Result<(), String> {
24 let mut config = RawDecodeConfig::new(
25 command
26 .model
27 .file_name()
28 .and_then(|name| name.to_str())
29 .unwrap_or("model"),
30 command.model.to_string_lossy(),
31 command.quantization,
32 command.prompt,
33 );
34 config.decode_steps = command.steps;
35 config.warmup_runs = command.warmups;
36 config.measured_runs = command.runs;
37 config.target_prompt_tokens = command.target_prompt_tokens;
38 config.requested_backend = BackendKind::Unknown;
39
40 let mut result = run_raw_decode_blocking(&config)?;
41 let retained = if let Some(target) = command.retain_artifacts {
42 let scratch_parent = target
43 .parent()
44 .filter(|parent| !parent.as_os_str().is_empty())
45 .unwrap_or_else(|| Path::new("."));
46 if scratch_parent.is_dir() {
47 let cutoff = SystemTime::now()
48 .checked_sub(Duration::from_secs(24 * 60 * 60))
49 .unwrap_or(SystemTime::UNIX_EPOCH);
50 match cleanup_stale_runs(scratch_parent, cutoff) {
51 Ok(cleanup) => {
52 result.manifest.receipt.artifacts.temp_removed_bytes = result
53 .manifest
54 .receipt
55 .artifacts
56 .temp_removed_bytes
57 .saturating_add(cleanup.removed_bytes);
58 result.manifest.receipt.artifacts.temp_cleanup_failures = result
59 .manifest
60 .receipt
61 .artifacts
62 .temp_cleanup_failures
63 .saturating_add(cleanup.failures);
64 }
65 Err(_) => {
66 result.manifest.receipt.artifacts.temp_cleanup_failures = result
67 .manifest
68 .receipt
69 .artifacts
70 .temp_cleanup_failures
71 .saturating_add(1);
72 }
73 }
74 }
75 let mut artifacts = RunArtifactDir::new_in(
76 scratch_parent,
77 "raw-decode",
78 8 * 1024 * 1024,
79 ArtifactRetention::RetainTo(target.to_path_buf()),
80 )
81 .map_err(|e| e.to_string())?;
82
83 let manifest_json = manifest_json_with_predicted_stats(&mut result)?;
84 artifacts
85 .write_bounded("manifest.json", manifest_json.as_bytes())
86 .map_err(|e| e.to_string())?;
87 let tokens_json =
88 serde_json::to_vec_pretty(&result.generated_token_ids).map_err(|e| e.to_string())?;
89 artifacts
90 .write_bounded("generated-token-ids.json", &tokens_json)
91 .map_err(|e| e.to_string())?;
92 let token_bytes_json =
93 serde_json::to_vec_pretty(&result.generated_token_bytes).map_err(|e| e.to_string())?;
94 artifacts
95 .write_bounded("generated-token-bytes.json", &token_bytes_json)
96 .map_err(|e| e.to_string())?;
97 artifacts
98 .write_bounded("generated-text.txt", result.generated_text.as_bytes())
99 .map_err(|e| e.to_string())?;
100 let finish = artifacts.finish().map_err(|e| e.to_string())?;
101 finish.retained_path
102 } else {
103 None
104 };
105
106 println!(
107 "RAW_DECODE median_tok_s={:.4} p95_ms_per_token={:.4} backend={:?} steps={} warmups={} runs={} dispatches={} fences={} d2h_bytes={} fallback_count={}",
108 result.manifest.median_tok_s,
109 result.manifest.p95_ms_per_token,
110 result.manifest.receipt.executed_backend,
111 result.manifest.decode_steps_executed,
112 result.manifest.warmup_runs,
113 result.manifest.measured_runs,
114 result.manifest.receipt.counters.compute_dispatches,
115 result.manifest.receipt.counters.device_fences,
116 result.manifest.receipt.counters.device_to_host_bytes,
117 result.manifest.receipt.counters.fallback_count,
118 );
119 println!(
120 "RAW_DECODE_TEXT {}",
121 serde_json::to_string(&result.generated_text).map_err(|e| e.to_string())?
122 );
123 println!(
124 "{}",
125 serde_json::to_string_pretty(&result.manifest).map_err(|e| e.to_string())?
126 );
127 if let Some(path) = retained {
128 eprintln!("RAW_DECODE_ARTIFACTS {}", path.display());
129 }
130 Ok(())
131}
132
133fn manifest_json_with_predicted_stats(
134 result: &mut qualia_core_db::inference::inference_bench::raw_decode::RawDecodeResult,
135) -> Result<String, String> {
136 let token_bytes = serde_json::to_vec_pretty(&result.generated_token_ids)
137 .map_err(|e| e.to_string())?
138 .len() as u64;
139 let token_piece_bytes = serde_json::to_vec_pretty(&result.generated_token_bytes)
140 .map_err(|e| e.to_string())?
141 .len() as u64;
142 let text_bytes = result.generated_text.len() as u64;
143 let mut previous_len = 0u64;
144 for _ in 0..4 {
145 let json = serde_json::to_string_pretty(&result.manifest).map_err(|e| e.to_string())?;
146 let total = json.len() as u64 + token_bytes + token_piece_bytes + text_bytes;
147 result.manifest.receipt.artifacts = ArtifactCleanupCounters {
148 temp_created_bytes: total,
149 temp_removed_bytes: 0,
150 temp_retained_bytes: total,
151 temp_cleanup_failures: 0,
152 };
153 if total == previous_len {
154 return serde_json::to_string_pretty(&result.manifest).map_err(|e| e.to_string());
155 }
156 previous_len = total;
157 }
158 serde_json::to_string_pretty(&result.manifest).map_err(|e| e.to_string())
159}