Skip to main content

qualia_cli/llm_raw_bench/
command.rs

1use std::path::Path;
2use std::time::{Duration, SystemTime};
3
4use qualia_core_db::inference::inference_bench::raw_decode::{
5    run_raw_decode_blocking, RawDecodeConfig,
6};
7use qualia_core_db::inference::runtime::{
8    cleanup_stale_runs, ArtifactCleanupCounters, ArtifactRetention, BackendKind, RunArtifactDir,
9};
10
11pub struct CommandConfig<'a> {
12    pub model: &'a Path,
13    pub steps: u32,
14    pub warmups: u16,
15    pub runs: u16,
16    pub quantization: &'a str,
17    pub prompt: &'a str,
18    pub target_prompt_tokens: Option<u32>,
19    pub retain_artifacts: Option<&'a Path>,
20}
21
22/// Runs the command.
23pub fn run(command: CommandConfig<'_>) -> Result<(), String> {
24    let mut config = RawDecodeConfig::new(
25        command
26            .model
27            .file_name()
28            .and_then(|name| name.to_str())
29            .unwrap_or("model"),
30        command.model.to_string_lossy(),
31        command.quantization,
32        command.prompt,
33    );
34    config.decode_steps = command.steps;
35    config.warmup_runs = command.warmups;
36    config.measured_runs = command.runs;
37    config.target_prompt_tokens = command.target_prompt_tokens;
38    config.requested_backend = BackendKind::Unknown;
39
40    let mut result = run_raw_decode_blocking(&config)?;
41    let retained = if let Some(target) = command.retain_artifacts {
42        let scratch_parent = target
43            .parent()
44            .filter(|parent| !parent.as_os_str().is_empty())
45            .unwrap_or_else(|| Path::new("."));
46        if scratch_parent.is_dir() {
47            let cutoff = SystemTime::now()
48                .checked_sub(Duration::from_secs(24 * 60 * 60))
49                .unwrap_or(SystemTime::UNIX_EPOCH);
50            match cleanup_stale_runs(scratch_parent, cutoff) {
51                Ok(cleanup) => {
52                    result.manifest.receipt.artifacts.temp_removed_bytes = result
53                        .manifest
54                        .receipt
55                        .artifacts
56                        .temp_removed_bytes
57                        .saturating_add(cleanup.removed_bytes);
58                    result.manifest.receipt.artifacts.temp_cleanup_failures = result
59                        .manifest
60                        .receipt
61                        .artifacts
62                        .temp_cleanup_failures
63                        .saturating_add(cleanup.failures);
64                }
65                Err(_) => {
66                    result.manifest.receipt.artifacts.temp_cleanup_failures = result
67                        .manifest
68                        .receipt
69                        .artifacts
70                        .temp_cleanup_failures
71                        .saturating_add(1);
72                }
73            }
74        }
75        let mut artifacts = RunArtifactDir::new_in(
76            scratch_parent,
77            "raw-decode",
78            8 * 1024 * 1024,
79            ArtifactRetention::RetainTo(target.to_path_buf()),
80        )
81        .map_err(|e| e.to_string())?;
82
83        let manifest_json = manifest_json_with_predicted_stats(&mut result)?;
84        artifacts
85            .write_bounded("manifest.json", manifest_json.as_bytes())
86            .map_err(|e| e.to_string())?;
87        let tokens_json =
88            serde_json::to_vec_pretty(&result.generated_token_ids).map_err(|e| e.to_string())?;
89        artifacts
90            .write_bounded("generated-token-ids.json", &tokens_json)
91            .map_err(|e| e.to_string())?;
92        let token_bytes_json =
93            serde_json::to_vec_pretty(&result.generated_token_bytes).map_err(|e| e.to_string())?;
94        artifacts
95            .write_bounded("generated-token-bytes.json", &token_bytes_json)
96            .map_err(|e| e.to_string())?;
97        artifacts
98            .write_bounded("generated-text.txt", result.generated_text.as_bytes())
99            .map_err(|e| e.to_string())?;
100        let finish = artifacts.finish().map_err(|e| e.to_string())?;
101        finish.retained_path
102    } else {
103        None
104    };
105
106    println!(
107        "RAW_DECODE median_tok_s={:.4} p95_ms_per_token={:.4} backend={:?} steps={} warmups={} runs={} dispatches={} fences={} d2h_bytes={} fallback_count={}",
108        result.manifest.median_tok_s,
109        result.manifest.p95_ms_per_token,
110        result.manifest.receipt.executed_backend,
111        result.manifest.decode_steps_executed,
112        result.manifest.warmup_runs,
113        result.manifest.measured_runs,
114        result.manifest.receipt.counters.compute_dispatches,
115        result.manifest.receipt.counters.device_fences,
116        result.manifest.receipt.counters.device_to_host_bytes,
117        result.manifest.receipt.counters.fallback_count,
118    );
119    println!(
120        "RAW_DECODE_TEXT {}",
121        serde_json::to_string(&result.generated_text).map_err(|e| e.to_string())?
122    );
123    println!(
124        "{}",
125        serde_json::to_string_pretty(&result.manifest).map_err(|e| e.to_string())?
126    );
127    if let Some(path) = retained {
128        eprintln!("RAW_DECODE_ARTIFACTS {}", path.display());
129    }
130    Ok(())
131}
132
133fn manifest_json_with_predicted_stats(
134    result: &mut qualia_core_db::inference::inference_bench::raw_decode::RawDecodeResult,
135) -> Result<String, String> {
136    let token_bytes = serde_json::to_vec_pretty(&result.generated_token_ids)
137        .map_err(|e| e.to_string())?
138        .len() as u64;
139    let token_piece_bytes = serde_json::to_vec_pretty(&result.generated_token_bytes)
140        .map_err(|e| e.to_string())?
141        .len() as u64;
142    let text_bytes = result.generated_text.len() as u64;
143    let mut previous_len = 0u64;
144    for _ in 0..4 {
145        let json = serde_json::to_string_pretty(&result.manifest).map_err(|e| e.to_string())?;
146        let total = json.len() as u64 + token_bytes + token_piece_bytes + text_bytes;
147        result.manifest.receipt.artifacts = ArtifactCleanupCounters {
148            temp_created_bytes: total,
149            temp_removed_bytes: 0,
150            temp_retained_bytes: total,
151            temp_cleanup_failures: 0,
152        };
153        if total == previous_len {
154            return serde_json::to_string_pretty(&result.manifest).map_err(|e| e.to_string());
155        }
156        previous_len = total;
157    }
158    serde_json::to_string_pretty(&result.manifest).map_err(|e| e.to_string())
159}