qualia_core_db/inference/runtime/receipt/
manifest.rs1use serde::{Deserialize, Serialize};
2use sha2::{Digest, Sha256};
3use std::io::Read;
4use std::path::Path;
5
6use super::ExecutionReceipt;
7
8pub const MANIFEST_SCHEMA_VERSION: u16 = 3;
9pub const RAW_GREEDY_DECODE_POLICY: &str =
10 "greedy-argmax;temperature=0;top_k=1;top_p=1;repeat_penalty=1;repeat_last_n=0;eos=ignored";
11
12#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
13pub struct BenchmarkManifest {
14 pub schema_version: u16,
15 pub benchmark_kind: String,
16 pub executable_commit: String,
17 pub dirty_diff_hash: String,
18 pub executable_sha256: String,
19 pub model_path: String,
20 pub model_sha256: String,
21 pub prompt_token_sha256: String,
22 #[serde(default)]
24 pub prompt_tokens: u32,
25 #[serde(default)]
27 pub context_window: u32,
28 #[serde(default)]
30 pub decode_policy: String,
31 pub quantization: String,
32 pub decode_steps_requested: u32,
33 pub decode_steps_executed: u32,
34 pub warmup_runs: u16,
35 pub measured_runs: u16,
36 pub median_tok_s: f64,
37 pub p95_ms_per_token: f64,
38 pub receipt: ExecutionReceipt,
39}
40
41impl BenchmarkManifest {
42 pub fn validate(&self) -> Result<(), &'static str> {
44 if self.schema_version != MANIFEST_SCHEMA_VERSION {
45 return Err("unsupported benchmark manifest schema");
46 }
47 if self.benchmark_kind.is_empty() {
48 return Err("benchmark kind is required");
49 }
50 if self.model_sha256.len() != 64
51 || self.prompt_token_sha256.len() != 64
52 || self.executable_sha256.len() != 64
53 {
54 return Err("SHA-256 fields must contain 64 hexadecimal characters");
55 }
56 if self.measured_runs == 0 {
57 return Err("at least one measured run is required");
58 }
59 if self.decode_steps_executed == 0 {
60 return Err("at least one decode step is required");
61 }
62 if self.prompt_tokens == 0 || self.context_window == 0 {
63 return Err("prompt token count and context window are required");
64 }
65 if self.decode_policy.is_empty() {
66 return Err("decode policy is required");
67 }
68 if self
69 .prompt_tokens
70 .saturating_add(self.decode_steps_executed)
71 > self.context_window
72 {
73 return Err("prompt plus decode budget exceeds the prepared context window");
74 }
75 if !self.median_tok_s.is_finite() || !self.p95_ms_per_token.is_finite() {
76 return Err("benchmark rates must be finite");
77 }
78 Ok(())
79 }
80}
81
82pub fn sha256_file(path: &Path) -> Result<String, std::io::Error> {
84 let mut file = std::fs::File::open(path)?;
85 let mut hasher = Sha256::new();
86 let mut buffer = [0u8; 1024 * 1024];
87 loop {
88 let read = file.read(&mut buffer)?;
89 if read == 0 {
90 break;
91 }
92 hasher.update(&buffer[..read]);
93 }
94 Ok(hex::encode(hasher.finalize()))
95}
96
97pub fn sha256_token_ids(token_ids: &[u32]) -> String {
99 let mut hasher = Sha256::new();
100 for token_id in token_ids {
101 hasher.update(token_id.to_le_bytes());
102 }
103 hex::encode(hasher.finalize())
104}
105
106#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
108pub struct HardwareProfileManifest {
109 pub device_name: String,
110 pub compute_capability: (u32, u32),
111 pub total_vram_mb: u32,
112 pub usable_vram_mb: u32,
113 pub host_ram_mb: u32,
114 pub pcie_gen: u32,
115 pub pcie_lanes: u32,
116 pub driver_version: String,
117 pub cuda_version: String,
118}
119
120impl HardwareProfileManifest {
121 pub fn rtx_a2000_12gb() -> Self {
123 Self {
124 device_name: "NVIDIA RTX A2000 12GB".into(),
125 compute_capability: (8, 6),
126 total_vram_mb: 12288,
127 usable_vram_mb: 11500,
128 host_ram_mb: 32768,
129 pcie_gen: 4,
130 pcie_lanes: 16,
131 driver_version: "551.78".into(),
132 cuda_version: "12.4".into(),
133 }
134 }
135}
136
137#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
139pub struct OsrpBaselineManifest {
140 pub model_id: String,
141 pub model_sha256: String,
142 pub launch_invocation: String,
143 pub prompt_tokens: u32,
144 pub decode_tokens: u32,
145 pub prefill_tok_s: f64,
146 pub decode_tok_s: f64,
147 pub ttft_ms: f64,
148 pub p95_inter_token_ms: f64,
149 pub peak_vram_mb: u32,
150}
151
152impl OsrpBaselineManifest {
153 pub fn qwen3_6_35b_nvfp4_reference() -> Self {
155 Self {
156 model_id: "Qwen3.6-35B-A3B-NVFP4".into(),
157 model_sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".into(),
158 launch_invocation:
159 "osrp --model Qwen3.6-35B-A3B-NVFP4 --kv-cache-paged --gpu-memory-utilization 0.92"
160 .into(),
161 prompt_tokens: 512,
162 decode_tokens: 128,
163 prefill_tok_s: 310.0,
164 decode_tok_s: 38.5,
165 ttft_ms: 165.0,
166 p95_inter_token_ms: 27.5,
167 peak_vram_mb: 11200,
168 }
169 }
170
171 pub fn granite_4_0_h_tiny_q4km_reference() -> Self {
173 Self {
174 model_id: "granite-4.0-h-tiny-Q4_K_M".into(),
175 model_sha256: "064bea0136420b38d0b65697fa5e772e28b112eee1757aacc7f64eba6bf37810".into(),
176 launch_invocation: "qualia-cli.exe llm raw-decode-bench granite-4.0-h-tiny-Q4_K_M.gguf --quantization Q4_K_M".into(),
177 prompt_tokens: 8,
178 decode_tokens: 16,
179 prefill_tok_s: 15.0,
180 decode_tok_s: 0.15,
181 ttft_ms: 500.0,
182 p95_inter_token_ms: 7500.0,
183 peak_vram_mb: 4500,
184 }
185 }
186}
187
188#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
190pub struct ParityGateEvaluation {
191 pub gate1_baseline_captured: bool,
192 pub gate2_identical_workload: bool,
193 pub gate3_real_generation: bool,
194 pub gate4_memory_bounded: bool,
195 pub gate5_quality_correct: bool,
196 pub gate6_performance_demonstrated: bool,
197 pub gate7_same_or_better: bool,
198}
199
200impl ParityGateEvaluation {
201 pub fn all_passed(&self) -> bool {
203 self.gate1_baseline_captured
204 && self.gate2_identical_workload
205 && self.gate3_real_generation
206 && self.gate4_memory_bounded
207 && self.gate5_quality_correct
208 && self.gate6_performance_demonstrated
209 && self.gate7_same_or_better
210 }
211}
212
213#[cfg(test)]
214mod tests {
215 use super::*;
216 use crate::inference::runtime::receipt::BackendKind;
217
218 #[test]
219 fn manifest_round_trip_and_validation() {
220 let manifest = BenchmarkManifest {
221 schema_version: MANIFEST_SCHEMA_VERSION,
222 benchmark_kind: "raw-decode".into(),
223 executable_commit: "abc".into(),
224 dirty_diff_hash: "def".into(),
225 executable_sha256: "c".repeat(64),
226 model_path: "model.gguf".into(),
227 model_sha256: "a".repeat(64),
228 prompt_token_sha256: "b".repeat(64),
229 prompt_tokens: 32,
230 context_window: 1024,
231 decode_policy: RAW_GREEDY_DECODE_POLICY.into(),
232 quantization: "Q8_0".into(),
233 decode_steps_requested: 256,
234 decode_steps_executed: 256,
235 warmup_runs: 1,
236 measured_runs: 5,
237 median_tok_s: 100.0,
238 p95_ms_per_token: 11.0,
239 receipt: ExecutionReceipt::new(BackendKind::Cuda, BackendKind::Cuda, "model", "plan"),
240 };
241 manifest.validate().unwrap();
242 let json = serde_json::to_string(&manifest).unwrap();
243 let decoded: BenchmarkManifest = serde_json::from_str(&json).unwrap();
244 assert_eq!(decoded, manifest);
245 }
246
247 #[test]
248 fn manifest_rejects_undeclared_decode_policy() {
249 let mut manifest = BenchmarkManifest {
250 schema_version: MANIFEST_SCHEMA_VERSION,
251 benchmark_kind: "raw-decode".into(),
252 executable_commit: "abc".into(),
253 dirty_diff_hash: "def".into(),
254 executable_sha256: "c".repeat(64),
255 model_path: "model.gguf".into(),
256 model_sha256: "a".repeat(64),
257 prompt_token_sha256: "b".repeat(64),
258 prompt_tokens: 5,
259 context_window: 1024,
260 decode_policy: RAW_GREEDY_DECODE_POLICY.into(),
261 quantization: "Q8_0".into(),
262 decode_steps_requested: 256,
263 decode_steps_executed: 256,
264 warmup_runs: 1,
265 measured_runs: 5,
266 median_tok_s: 100.0,
267 p95_ms_per_token: 11.0,
268 receipt: ExecutionReceipt::new(BackendKind::Cuda, BackendKind::Cuda, "model", "plan"),
269 };
270 manifest.decode_policy.clear();
271 assert_eq!(manifest.validate(), Err("decode policy is required"));
272 }
273
274 #[test]
275 fn hardware_profile_and_osrp_baseline_roundtrip() {
276 let hw = HardwareProfileManifest::rtx_a2000_12gb();
277 assert_eq!(hw.compute_capability, (8, 6));
278 assert_eq!(hw.total_vram_mb, 12288);
279 let hw_json = serde_json::to_string(&hw).unwrap();
280 let hw_dec: HardwareProfileManifest = serde_json::from_str(&hw_json).unwrap();
281 assert_eq!(hw_dec, hw);
282
283 let osrp = OsrpBaselineManifest::qwen3_6_35b_nvfp4_reference();
284 assert_eq!(osrp.model_id, "Qwen3.6-35B-A3B-NVFP4");
285 assert!(osrp.decode_tok_s > 30.0);
286 let osrp_json = serde_json::to_string(&osrp).unwrap();
287 let osrp_dec: OsrpBaselineManifest = serde_json::from_str(&osrp_json).unwrap();
288 assert_eq!(osrp_dec, osrp);
289
290 let gates = ParityGateEvaluation {
291 gate1_baseline_captured: true,
292 gate2_identical_workload: true,
293 gate3_real_generation: true,
294 gate4_memory_bounded: true,
295 gate5_quality_correct: true,
296 gate6_performance_demonstrated: true,
297 gate7_same_or_better: true,
298 };
299 assert!(gates.all_passed());
300
301 let incomplete_gates = ParityGateEvaluation {
302 gate7_same_or_better: false,
303 ..gates
304 };
305 assert!(!incomplete_gates.all_passed());
306 }
307}