Skip to main content

qualia_core_db/inference/runtime/receipt/
parity_runner.rs

1//! Automated 7-Gate A2000 Parity Release Evaluator (Work Package F15).
2//!
3//! Compares production Qualia execution receipts against pinned reference baselines,
4//! validating the 7 mandatory gates required to certify OSRP parity on NVIDIA RTX A2000 12GB.
5
6use super::execution::ExecutionReceipt;
7use super::manifest::{
8    BenchmarkManifest, HardwareProfileManifest, OsrpBaselineManifest, ParityGateEvaluation,
9    RAW_GREEDY_DECODE_POLICY,
10};
11
12/// Maximum allowable latency margin (5% degradation tolerance).
13pub const LATENCY_TOLERANCE_MARGIN: f64 = 1.05;
14
15/// Minimum throughput threshold (95% of reference baseline).
16pub const THROUGHPUT_PARITY_THRESHOLD: f64 = 0.95;
17
18/// Detailed evaluation report for all 7 release gates.
19#[derive(Debug, Clone, PartialEq)]
20pub struct ParityGateReport {
21    pub evaluation: ParityGateEvaluation,
22    pub gate1_details: String,
23    pub gate2_details: String,
24    pub gate3_details: String,
25    pub gate4_details: String,
26    pub gate5_details: String,
27    pub gate6_details: String,
28    pub gate7_details: String,
29}
30
31impl ParityGateReport {
32    /// `true` when `certified`.
33    pub fn is_certified(&self) -> bool {
34        self.evaluation.all_passed()
35    }
36}
37
38/// Evaluate candidate execution against reference baseline and hardware profile.
39pub fn evaluate_parity_release(
40    candidate: &BenchmarkManifest,
41    candidate_receipt: &ExecutionReceipt,
42    baseline: &OsrpBaselineManifest,
43    hardware: &HardwareProfileManifest,
44) -> ParityGateReport {
45    // Gate 1: Baseline Captured
46    let g1_pass = !baseline.model_sha256.is_empty()
47        && baseline.model_sha256.len() == 64
48        && !baseline.launch_invocation.is_empty()
49        && hardware.compute_capability == (8, 6)
50        && hardware.total_vram_mb >= 12288;
51    let g1_msg = if g1_pass {
52        format!(
53            "PASS: Baseline captured for {} on {} (SM {}.{}, {} MiB VRAM)",
54            baseline.model_id,
55            hardware.device_name,
56            hardware.compute_capability.0,
57            hardware.compute_capability.1,
58            hardware.total_vram_mb
59        )
60    } else {
61        "FAIL: Incomplete or invalid reference baseline / hardware profile".into()
62    };
63
64    // Gate 2: Identical Workload
65    let g2_pass = candidate.prompt_tokens == baseline.prompt_tokens
66        && candidate.decode_policy == RAW_GREEDY_DECODE_POLICY;
67    let g2_msg = if g2_pass {
68        format!(
69            "PASS: Identical workload verified (prompt_tokens={}, policy={})",
70            candidate.prompt_tokens, candidate.decode_policy
71        )
72    } else {
73        format!(
74            "FAIL: Workload mismatch (candidate tokens={}, baseline tokens={}, policy='{}')",
75            candidate.prompt_tokens, baseline.prompt_tokens, candidate.decode_policy
76        )
77    };
78
79    // Gate 3: Real Generation through Native Routes
80    let g3_pass = candidate.decode_steps_executed > 0
81        && candidate_receipt.counters.decode_steps > 0
82        && candidate_receipt.executed_backend != super::execution::BackendKind::Unknown;
83    let g3_msg = if g3_pass {
84        format!(
85            "PASS: Native execution verified on {:?} (steps={})",
86            candidate_receipt.executed_backend, candidate.decode_steps_executed
87        )
88    } else {
89        "FAIL: Execution simulated, truncated, or failed to advance steps".into()
90    };
91
92    // Gate 4: Memory Bounded (peak within 12 GB, zero hot allocations)
93    let g4_pass = candidate_receipt.counters.hot_path_allocations == 0
94        && candidate_receipt.counters.pool_high_water_bytes as u64
95            <= (hardware.usable_vram_mb as u64 * 1024 * 1024);
96    let g4_msg = if g4_pass {
97        format!(
98            "PASS: Zero hot-allocations and VRAM within ceiling ({:.1} MiB used / {} MiB usable)",
99            candidate_receipt.counters.pool_high_water_bytes as f64 / (1024.0 * 1024.0),
100            hardware.usable_vram_mb
101        )
102    } else {
103        format!(
104            "FAIL: Memory bounds violated (hot_path_allocations={}, peak_bytes={})",
105            candidate_receipt.counters.hot_path_allocations,
106            candidate_receipt.counters.pool_high_water_bytes
107        )
108    };
109
110    // Gate 5: Quality Correct
111    let g5_pass = candidate_receipt.counters.fallback_count == 0;
112    let g5_msg = if g5_pass {
113        "PASS: Zero routing fallbacks or skipped expert evaluations".into()
114    } else {
115        format!(
116            "FAIL: {} fallbacks/skipped experts recorded",
117            candidate_receipt.counters.fallback_count
118        )
119    };
120
121    // Gate 6: Performance Demonstrated (warmup + measured runs >= 5)
122    let g6_pass = candidate.measured_runs >= 5
123        && candidate.median_tok_s.is_finite()
124        && candidate.median_tok_s > 0.0
125        && candidate.p95_ms_per_token.is_finite()
126        && candidate.p95_ms_per_token > 0.0;
127    let g6_msg = if g6_pass {
128        format!(
129            "PASS: Measured across {} runs (median={:.2} tok/s, p95={:.2} ms)",
130            candidate.measured_runs, candidate.median_tok_s, candidate.p95_ms_per_token
131        )
132    } else {
133        format!(
134            "FAIL: Insufficient repetitions ({} runs) or invalid metrics",
135            candidate.measured_runs
136        )
137    };
138
139    // Gate 7: Same or Better than OSRP Baseline
140    let min_acceptable_throughput = baseline.decode_tok_s * THROUGHPUT_PARITY_THRESHOLD;
141    let g7_pass = candidate.median_tok_s >= min_acceptable_throughput;
142    let g7_msg = if g7_pass {
143        format!(
144            "PASS: Throughput {:.2} tok/s meets/exceeds reference {:.2} tok/s ({:.1}% of baseline)",
145            candidate.median_tok_s,
146            baseline.decode_tok_s,
147            (candidate.median_tok_s / baseline.decode_tok_s) * 100.0
148        )
149    } else {
150        format!(
151            "FAIL: Throughput {:.2} tok/s below required {:.2} tok/s",
152            candidate.median_tok_s, min_acceptable_throughput
153        )
154    };
155
156    ParityGateReport {
157        evaluation: ParityGateEvaluation {
158            gate1_baseline_captured: g1_pass,
159            gate2_identical_workload: g2_pass,
160            gate3_real_generation: g3_pass,
161            gate4_memory_bounded: g4_pass,
162            gate5_quality_correct: g5_pass,
163            gate6_performance_demonstrated: g6_pass,
164            gate7_same_or_better: g7_pass,
165        },
166        gate1_details: g1_msg,
167        gate2_details: g2_msg,
168        gate3_details: g3_msg,
169        gate4_details: g4_msg,
170        gate5_details: g5_msg,
171        gate6_details: g6_msg,
172        gate7_details: g7_msg,
173    }
174}
175
176#[cfg(test)]
177mod tests {
178    use super::*;
179    use crate::inference::runtime::receipt::execution::BackendKind;
180
181    fn test_fixtures() -> (
182        BenchmarkManifest,
183        ExecutionReceipt,
184        OsrpBaselineManifest,
185        HardwareProfileManifest,
186    ) {
187        let mut receipt = ExecutionReceipt::new(
188            BackendKind::Cuda,
189            BackendKind::Cuda,
190            "qwen3.6",
191            "decode_plan",
192        );
193        receipt.counters.decode_steps = 128;
194        receipt.counters.hot_path_allocations = 0;
195        receipt.counters.pool_high_water_bytes = 10_000_000_000; // ~9.5 GB
196        receipt.counters.fallback_count = 0;
197
198        let manifest = BenchmarkManifest {
199            schema_version: super::super::manifest::MANIFEST_SCHEMA_VERSION,
200            benchmark_kind: "raw-decode".into(),
201            executable_commit: "abc".repeat(10),
202            dirty_diff_hash: "0".repeat(64),
203            executable_sha256: "1".repeat(64),
204            model_path: "Qwen3.6-35B-A3B-NVFP4.gguf".into(),
205            model_sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".into(),
206            prompt_token_sha256: "2".repeat(64),
207            prompt_tokens: 512,
208            context_window: 4096,
209            decode_policy: RAW_GREEDY_DECODE_POLICY.into(),
210            quantization: "NVFP4".into(),
211            decode_steps_requested: 128,
212            decode_steps_executed: 128,
213            warmup_runs: 1,
214            measured_runs: 5,
215            median_tok_s: 41.2, // Exceeds baseline of 38.5
216            p95_ms_per_token: 25.0,
217            receipt: receipt.clone(),
218        };
219
220        let baseline = OsrpBaselineManifest::qwen3_6_35b_nvfp4_reference();
221        let hardware = HardwareProfileManifest::rtx_a2000_12gb();
222
223        (manifest, receipt, baseline, hardware)
224    }
225
226    #[test]
227    fn test_parity_release_passes_all_gates() {
228        let (manifest, receipt, baseline, hardware) = test_fixtures();
229        let report = evaluate_parity_release(&manifest, &receipt, &baseline, &hardware);
230
231        assert!(report.is_certified());
232        assert!(report.evaluation.gate1_baseline_captured);
233        assert!(report.evaluation.gate2_identical_workload);
234        assert!(report.evaluation.gate3_real_generation);
235        assert!(report.evaluation.gate4_memory_bounded);
236        assert!(report.evaluation.gate5_quality_correct);
237        assert!(report.evaluation.gate6_performance_demonstrated);
238        assert!(report.evaluation.gate7_same_or_better);
239    }
240
241    #[test]
242    fn test_parity_release_fails_if_memory_exceeded() {
243        let (manifest, mut receipt, baseline, hardware) = test_fixtures();
244        receipt.counters.pool_high_water_bytes = 13_000_000_000; // > 11.5 GB usable
245        let report = evaluate_parity_release(&manifest, &receipt, &baseline, &hardware);
246
247        assert!(!report.is_certified());
248        assert!(!report.evaluation.gate4_memory_bounded);
249    }
250
251    #[test]
252    fn test_parity_release_fails_if_throughput_degrades() {
253        let (mut manifest, receipt, baseline, hardware) = test_fixtures();
254        manifest.median_tok_s = 25.0; // Well below 38.5 * 0.95 = 36.575
255        let report = evaluate_parity_release(&manifest, &receipt, &baseline, &hardware);
256
257        assert!(!report.is_certified());
258        assert!(!report.evaluation.gate7_same_or_better);
259    }
260
261    #[test]
262    fn test_parity_release_with_real_granite_model_run() {
263        let mut receipt = ExecutionReceipt::new(
264            BackendKind::WgpuVulkan,
265            BackendKind::WgpuVulkan,
266            "064bea0136420b38d0b65697fa5e772e28b112eee1757aacc7f64eba6bf37810",
267            "resident-v1",
268        );
269        receipt.counters.decode_steps = 16;
270        receipt.counters.compute_dispatches = 16;
271        receipt.counters.device_fences = 16;
272        receipt.counters.hot_path_allocations = 0;
273        receipt.counters.pool_high_water_bytes = 4_200_000_000;
274        receipt.counters.fallback_count = 0;
275
276        let manifest = BenchmarkManifest {
277            schema_version: super::super::manifest::MANIFEST_SCHEMA_VERSION,
278            benchmark_kind: "raw-decode-resident".into(),
279            executable_commit: "69a310a48119ef7bf16876cfdb4fe9b3df185fc7".into(),
280            dirty_diff_hash: "0".repeat(64),
281            executable_sha256: "5d31c5892464924a3187a4862ddef4b65ed000e36fd69fa1e2bfc732d308cee6".into(),
282            model_path: "E:\\LLM_Models\\lmstudio-community\\granite-4.0-h-tiny-GGUF\\granite-4.0-h-tiny-Q4_K_M.gguf".into(),
283            model_sha256: "064bea0136420b38d0b65697fa5e772e28b112eee1757aacc7f64eba6bf37810".into(),
284            prompt_token_sha256: "9fe1246f05916cd46ec813b08c6c8010be08571c996ecf668d0c7bdde7fd25e3".into(),
285            prompt_tokens: 8,
286            context_window: 1024,
287            decode_policy: RAW_GREEDY_DECODE_POLICY.into(),
288            quantization: "Q4_K_M".into(),
289            decode_steps_requested: 16,
290            decode_steps_executed: 16,
291            warmup_runs: 1,
292            measured_runs: 5,
293            median_tok_s: 0.1577,
294            p95_ms_per_token: 7186.17,
295            receipt: receipt.clone(),
296        };
297
298        let baseline = OsrpBaselineManifest::granite_4_0_h_tiny_q4km_reference();
299        let hardware = HardwareProfileManifest::rtx_a2000_12gb();
300
301        let report = evaluate_parity_release(&manifest, &receipt, &baseline, &hardware);
302        assert!(report.is_certified());
303        assert!(report.evaluation.gate1_baseline_captured);
304        assert!(report.evaluation.gate2_identical_workload);
305        assert!(report.evaluation.gate3_real_generation);
306        assert!(report.evaluation.gate4_memory_bounded);
307        assert!(report.evaluation.gate5_quality_correct);
308        assert!(report.evaluation.gate6_performance_demonstrated);
309        assert!(report.evaluation.gate7_same_or_better);
310    }
311}