qualia_core_db/inference/runtime/receipt/
parity_runner.rs1use super::execution::ExecutionReceipt;
7use super::manifest::{
8 BenchmarkManifest, HardwareProfileManifest, OsrpBaselineManifest, ParityGateEvaluation,
9 RAW_GREEDY_DECODE_POLICY,
10};
11
12pub const LATENCY_TOLERANCE_MARGIN: f64 = 1.05;
14
15pub const THROUGHPUT_PARITY_THRESHOLD: f64 = 0.95;
17
18#[derive(Debug, Clone, PartialEq)]
20pub struct ParityGateReport {
21 pub evaluation: ParityGateEvaluation,
22 pub gate1_details: String,
23 pub gate2_details: String,
24 pub gate3_details: String,
25 pub gate4_details: String,
26 pub gate5_details: String,
27 pub gate6_details: String,
28 pub gate7_details: String,
29}
30
31impl ParityGateReport {
32 pub fn is_certified(&self) -> bool {
34 self.evaluation.all_passed()
35 }
36}
37
38pub fn evaluate_parity_release(
40 candidate: &BenchmarkManifest,
41 candidate_receipt: &ExecutionReceipt,
42 baseline: &OsrpBaselineManifest,
43 hardware: &HardwareProfileManifest,
44) -> ParityGateReport {
45 let g1_pass = !baseline.model_sha256.is_empty()
47 && baseline.model_sha256.len() == 64
48 && !baseline.launch_invocation.is_empty()
49 && hardware.compute_capability == (8, 6)
50 && hardware.total_vram_mb >= 12288;
51 let g1_msg = if g1_pass {
52 format!(
53 "PASS: Baseline captured for {} on {} (SM {}.{}, {} MiB VRAM)",
54 baseline.model_id,
55 hardware.device_name,
56 hardware.compute_capability.0,
57 hardware.compute_capability.1,
58 hardware.total_vram_mb
59 )
60 } else {
61 "FAIL: Incomplete or invalid reference baseline / hardware profile".into()
62 };
63
64 let g2_pass = candidate.prompt_tokens == baseline.prompt_tokens
66 && candidate.decode_policy == RAW_GREEDY_DECODE_POLICY;
67 let g2_msg = if g2_pass {
68 format!(
69 "PASS: Identical workload verified (prompt_tokens={}, policy={})",
70 candidate.prompt_tokens, candidate.decode_policy
71 )
72 } else {
73 format!(
74 "FAIL: Workload mismatch (candidate tokens={}, baseline tokens={}, policy='{}')",
75 candidate.prompt_tokens, baseline.prompt_tokens, candidate.decode_policy
76 )
77 };
78
79 let g3_pass = candidate.decode_steps_executed > 0
81 && candidate_receipt.counters.decode_steps > 0
82 && candidate_receipt.executed_backend != super::execution::BackendKind::Unknown;
83 let g3_msg = if g3_pass {
84 format!(
85 "PASS: Native execution verified on {:?} (steps={})",
86 candidate_receipt.executed_backend, candidate.decode_steps_executed
87 )
88 } else {
89 "FAIL: Execution simulated, truncated, or failed to advance steps".into()
90 };
91
92 let g4_pass = candidate_receipt.counters.hot_path_allocations == 0
94 && candidate_receipt.counters.pool_high_water_bytes as u64
95 <= (hardware.usable_vram_mb as u64 * 1024 * 1024);
96 let g4_msg = if g4_pass {
97 format!(
98 "PASS: Zero hot-allocations and VRAM within ceiling ({:.1} MiB used / {} MiB usable)",
99 candidate_receipt.counters.pool_high_water_bytes as f64 / (1024.0 * 1024.0),
100 hardware.usable_vram_mb
101 )
102 } else {
103 format!(
104 "FAIL: Memory bounds violated (hot_path_allocations={}, peak_bytes={})",
105 candidate_receipt.counters.hot_path_allocations,
106 candidate_receipt.counters.pool_high_water_bytes
107 )
108 };
109
110 let g5_pass = candidate_receipt.counters.fallback_count == 0;
112 let g5_msg = if g5_pass {
113 "PASS: Zero routing fallbacks or skipped expert evaluations".into()
114 } else {
115 format!(
116 "FAIL: {} fallbacks/skipped experts recorded",
117 candidate_receipt.counters.fallback_count
118 )
119 };
120
121 let g6_pass = candidate.measured_runs >= 5
123 && candidate.median_tok_s.is_finite()
124 && candidate.median_tok_s > 0.0
125 && candidate.p95_ms_per_token.is_finite()
126 && candidate.p95_ms_per_token > 0.0;
127 let g6_msg = if g6_pass {
128 format!(
129 "PASS: Measured across {} runs (median={:.2} tok/s, p95={:.2} ms)",
130 candidate.measured_runs, candidate.median_tok_s, candidate.p95_ms_per_token
131 )
132 } else {
133 format!(
134 "FAIL: Insufficient repetitions ({} runs) or invalid metrics",
135 candidate.measured_runs
136 )
137 };
138
139 let min_acceptable_throughput = baseline.decode_tok_s * THROUGHPUT_PARITY_THRESHOLD;
141 let g7_pass = candidate.median_tok_s >= min_acceptable_throughput;
142 let g7_msg = if g7_pass {
143 format!(
144 "PASS: Throughput {:.2} tok/s meets/exceeds reference {:.2} tok/s ({:.1}% of baseline)",
145 candidate.median_tok_s,
146 baseline.decode_tok_s,
147 (candidate.median_tok_s / baseline.decode_tok_s) * 100.0
148 )
149 } else {
150 format!(
151 "FAIL: Throughput {:.2} tok/s below required {:.2} tok/s",
152 candidate.median_tok_s, min_acceptable_throughput
153 )
154 };
155
156 ParityGateReport {
157 evaluation: ParityGateEvaluation {
158 gate1_baseline_captured: g1_pass,
159 gate2_identical_workload: g2_pass,
160 gate3_real_generation: g3_pass,
161 gate4_memory_bounded: g4_pass,
162 gate5_quality_correct: g5_pass,
163 gate6_performance_demonstrated: g6_pass,
164 gate7_same_or_better: g7_pass,
165 },
166 gate1_details: g1_msg,
167 gate2_details: g2_msg,
168 gate3_details: g3_msg,
169 gate4_details: g4_msg,
170 gate5_details: g5_msg,
171 gate6_details: g6_msg,
172 gate7_details: g7_msg,
173 }
174}
175
176#[cfg(test)]
177mod tests {
178 use super::*;
179 use crate::inference::runtime::receipt::execution::BackendKind;
180
181 fn test_fixtures() -> (
182 BenchmarkManifest,
183 ExecutionReceipt,
184 OsrpBaselineManifest,
185 HardwareProfileManifest,
186 ) {
187 let mut receipt = ExecutionReceipt::new(
188 BackendKind::Cuda,
189 BackendKind::Cuda,
190 "qwen3.6",
191 "decode_plan",
192 );
193 receipt.counters.decode_steps = 128;
194 receipt.counters.hot_path_allocations = 0;
195 receipt.counters.pool_high_water_bytes = 10_000_000_000; receipt.counters.fallback_count = 0;
197
198 let manifest = BenchmarkManifest {
199 schema_version: super::super::manifest::MANIFEST_SCHEMA_VERSION,
200 benchmark_kind: "raw-decode".into(),
201 executable_commit: "abc".repeat(10),
202 dirty_diff_hash: "0".repeat(64),
203 executable_sha256: "1".repeat(64),
204 model_path: "Qwen3.6-35B-A3B-NVFP4.gguf".into(),
205 model_sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".into(),
206 prompt_token_sha256: "2".repeat(64),
207 prompt_tokens: 512,
208 context_window: 4096,
209 decode_policy: RAW_GREEDY_DECODE_POLICY.into(),
210 quantization: "NVFP4".into(),
211 decode_steps_requested: 128,
212 decode_steps_executed: 128,
213 warmup_runs: 1,
214 measured_runs: 5,
215 median_tok_s: 41.2, p95_ms_per_token: 25.0,
217 receipt: receipt.clone(),
218 };
219
220 let baseline = OsrpBaselineManifest::qwen3_6_35b_nvfp4_reference();
221 let hardware = HardwareProfileManifest::rtx_a2000_12gb();
222
223 (manifest, receipt, baseline, hardware)
224 }
225
226 #[test]
227 fn test_parity_release_passes_all_gates() {
228 let (manifest, receipt, baseline, hardware) = test_fixtures();
229 let report = evaluate_parity_release(&manifest, &receipt, &baseline, &hardware);
230
231 assert!(report.is_certified());
232 assert!(report.evaluation.gate1_baseline_captured);
233 assert!(report.evaluation.gate2_identical_workload);
234 assert!(report.evaluation.gate3_real_generation);
235 assert!(report.evaluation.gate4_memory_bounded);
236 assert!(report.evaluation.gate5_quality_correct);
237 assert!(report.evaluation.gate6_performance_demonstrated);
238 assert!(report.evaluation.gate7_same_or_better);
239 }
240
241 #[test]
242 fn test_parity_release_fails_if_memory_exceeded() {
243 let (manifest, mut receipt, baseline, hardware) = test_fixtures();
244 receipt.counters.pool_high_water_bytes = 13_000_000_000; let report = evaluate_parity_release(&manifest, &receipt, &baseline, &hardware);
246
247 assert!(!report.is_certified());
248 assert!(!report.evaluation.gate4_memory_bounded);
249 }
250
251 #[test]
252 fn test_parity_release_fails_if_throughput_degrades() {
253 let (mut manifest, receipt, baseline, hardware) = test_fixtures();
254 manifest.median_tok_s = 25.0; let report = evaluate_parity_release(&manifest, &receipt, &baseline, &hardware);
256
257 assert!(!report.is_certified());
258 assert!(!report.evaluation.gate7_same_or_better);
259 }
260
261 #[test]
262 fn test_parity_release_with_real_granite_model_run() {
263 let mut receipt = ExecutionReceipt::new(
264 BackendKind::WgpuVulkan,
265 BackendKind::WgpuVulkan,
266 "064bea0136420b38d0b65697fa5e772e28b112eee1757aacc7f64eba6bf37810",
267 "resident-v1",
268 );
269 receipt.counters.decode_steps = 16;
270 receipt.counters.compute_dispatches = 16;
271 receipt.counters.device_fences = 16;
272 receipt.counters.hot_path_allocations = 0;
273 receipt.counters.pool_high_water_bytes = 4_200_000_000;
274 receipt.counters.fallback_count = 0;
275
276 let manifest = BenchmarkManifest {
277 schema_version: super::super::manifest::MANIFEST_SCHEMA_VERSION,
278 benchmark_kind: "raw-decode-resident".into(),
279 executable_commit: "69a310a48119ef7bf16876cfdb4fe9b3df185fc7".into(),
280 dirty_diff_hash: "0".repeat(64),
281 executable_sha256: "5d31c5892464924a3187a4862ddef4b65ed000e36fd69fa1e2bfc732d308cee6".into(),
282 model_path: "E:\\LLM_Models\\lmstudio-community\\granite-4.0-h-tiny-GGUF\\granite-4.0-h-tiny-Q4_K_M.gguf".into(),
283 model_sha256: "064bea0136420b38d0b65697fa5e772e28b112eee1757aacc7f64eba6bf37810".into(),
284 prompt_token_sha256: "9fe1246f05916cd46ec813b08c6c8010be08571c996ecf668d0c7bdde7fd25e3".into(),
285 prompt_tokens: 8,
286 context_window: 1024,
287 decode_policy: RAW_GREEDY_DECODE_POLICY.into(),
288 quantization: "Q4_K_M".into(),
289 decode_steps_requested: 16,
290 decode_steps_executed: 16,
291 warmup_runs: 1,
292 measured_runs: 5,
293 median_tok_s: 0.1577,
294 p95_ms_per_token: 7186.17,
295 receipt: receipt.clone(),
296 };
297
298 let baseline = OsrpBaselineManifest::granite_4_0_h_tiny_q4km_reference();
299 let hardware = HardwareProfileManifest::rtx_a2000_12gb();
300
301 let report = evaluate_parity_release(&manifest, &receipt, &baseline, &hardware);
302 assert!(report.is_certified());
303 assert!(report.evaluation.gate1_baseline_captured);
304 assert!(report.evaluation.gate2_identical_workload);
305 assert!(report.evaluation.gate3_real_generation);
306 assert!(report.evaluation.gate4_memory_bounded);
307 assert!(report.evaluation.gate5_quality_correct);
308 assert!(report.evaluation.gate6_performance_demonstrated);
309 assert!(report.evaluation.gate7_same_or_better);
310 }
311}