Skip to main content

qualia_core_db/inference/runtime/receipt/
manifest.rs

1use serde::{Deserialize, Serialize};
2use sha2::{Digest, Sha256};
3use std::io::Read;
4use std::path::Path;
5
6use super::ExecutionReceipt;
7
8pub const MANIFEST_SCHEMA_VERSION: u16 = 3;
9pub const RAW_GREEDY_DECODE_POLICY: &str =
10    "greedy-argmax;temperature=0;top_k=1;top_p=1;repeat_penalty=1;repeat_last_n=0;eos=ignored";
11
12#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
13pub struct BenchmarkManifest {
14    pub schema_version: u16,
15    pub benchmark_kind: String,
16    pub executable_commit: String,
17    pub dirty_diff_hash: String,
18    pub executable_sha256: String,
19    pub model_path: String,
20    pub model_sha256: String,
21    pub prompt_token_sha256: String,
22    /// Number of tokens produced by the declared prompt before decode begins.
23    #[serde(default)]
24    pub prompt_tokens: u32,
25    /// Prepared runtime context capacity used for this run.
26    #[serde(default)]
27    pub context_window: u32,
28    /// Complete sampling/termination contract. Comparator runs must declare an identical policy.
29    #[serde(default)]
30    pub decode_policy: String,
31    pub quantization: String,
32    pub decode_steps_requested: u32,
33    pub decode_steps_executed: u32,
34    pub warmup_runs: u16,
35    pub measured_runs: u16,
36    pub median_tok_s: f64,
37    pub p95_ms_per_token: f64,
38    pub receipt: ExecutionReceipt,
39}
40
41impl BenchmarkManifest {
42    /// Validates manifest field coherence; static-str error on violation.
43    pub fn validate(&self) -> Result<(), &'static str> {
44        if self.schema_version != MANIFEST_SCHEMA_VERSION {
45            return Err("unsupported benchmark manifest schema");
46        }
47        if self.benchmark_kind.is_empty() {
48            return Err("benchmark kind is required");
49        }
50        if self.model_sha256.len() != 64
51            || self.prompt_token_sha256.len() != 64
52            || self.executable_sha256.len() != 64
53        {
54            return Err("SHA-256 fields must contain 64 hexadecimal characters");
55        }
56        if self.measured_runs == 0 {
57            return Err("at least one measured run is required");
58        }
59        if self.decode_steps_executed == 0 {
60            return Err("at least one decode step is required");
61        }
62        if self.prompt_tokens == 0 || self.context_window == 0 {
63            return Err("prompt token count and context window are required");
64        }
65        if self.decode_policy.is_empty() {
66            return Err("decode policy is required");
67        }
68        if self
69            .prompt_tokens
70            .saturating_add(self.decode_steps_executed)
71            > self.context_window
72        {
73            return Err("prompt plus decode budget exceeds the prepared context window");
74        }
75        if !self.median_tok_s.is_finite() || !self.p95_ms_per_token.is_finite() {
76            return Err("benchmark rates must be finite");
77        }
78        Ok(())
79    }
80}
81
82/// Streaming SHA-256 hex of a file.
83pub fn sha256_file(path: &Path) -> Result<String, std::io::Error> {
84    let mut file = std::fs::File::open(path)?;
85    let mut hasher = Sha256::new();
86    let mut buffer = [0u8; 1024 * 1024];
87    loop {
88        let read = file.read(&mut buffer)?;
89        if read == 0 {
90            break;
91        }
92        hasher.update(&buffer[..read]);
93    }
94    Ok(hex::encode(hasher.finalize()))
95}
96
97/// SHA-256 hex of a token-id sequence.
98pub fn sha256_token_ids(token_ids: &[u32]) -> String {
99    let mut hasher = Sha256::new();
100    for token_id in token_ids {
101        hasher.update(token_id.to_le_bytes());
102    }
103    hex::encode(hasher.finalize())
104}
105
106/// Captured hardware environment profile for validation and parity comparison (Gate 1).
107#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
108pub struct HardwareProfileManifest {
109    pub device_name: String,
110    pub compute_capability: (u32, u32),
111    pub total_vram_mb: u32,
112    pub usable_vram_mb: u32,
113    pub host_ram_mb: u32,
114    pub pcie_gen: u32,
115    pub pcie_lanes: u32,
116    pub driver_version: String,
117    pub cuda_version: String,
118}
119
120impl HardwareProfileManifest {
121    /// Reference manifest for the RTX A2000 12GB.
122    pub fn rtx_a2000_12gb() -> Self {
123        Self {
124            device_name: "NVIDIA RTX A2000 12GB".into(),
125            compute_capability: (8, 6),
126            total_vram_mb: 12288,
127            usable_vram_mb: 11500,
128            host_ram_mb: 32768,
129            pcie_gen: 4,
130            pcie_lanes: 16,
131            driver_version: "551.78".into(),
132            cuda_version: "12.4".into(),
133        }
134    }
135}
136
137/// Baseline benchmark manifest captured on reference OSRP implementation (Gate 1).
138#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
139pub struct OsrpBaselineManifest {
140    pub model_id: String,
141    pub model_sha256: String,
142    pub launch_invocation: String,
143    pub prompt_tokens: u32,
144    pub decode_tokens: u32,
145    pub prefill_tok_s: f64,
146    pub decode_tok_s: f64,
147    pub ttft_ms: f64,
148    pub p95_inter_token_ms: f64,
149    pub peak_vram_mb: u32,
150}
151
152impl OsrpBaselineManifest {
153    /// Reference baseline: Qwen3.6-35B NVFP4.
154    pub fn qwen3_6_35b_nvfp4_reference() -> Self {
155        Self {
156            model_id: "Qwen3.6-35B-A3B-NVFP4".into(),
157            model_sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef".into(),
158            launch_invocation:
159                "osrp --model Qwen3.6-35B-A3B-NVFP4 --kv-cache-paged --gpu-memory-utilization 0.92"
160                    .into(),
161            prompt_tokens: 512,
162            decode_tokens: 128,
163            prefill_tok_s: 310.0,
164            decode_tok_s: 38.5,
165            ttft_ms: 165.0,
166            p95_inter_token_ms: 27.5,
167            peak_vram_mb: 11200,
168        }
169    }
170
171    /// Reference baseline: Granite-4.0-H-tiny Q4_K_M.
172    pub fn granite_4_0_h_tiny_q4km_reference() -> Self {
173        Self {
174            model_id: "granite-4.0-h-tiny-Q4_K_M".into(),
175            model_sha256: "064bea0136420b38d0b65697fa5e772e28b112eee1757aacc7f64eba6bf37810".into(),
176            launch_invocation: "qualia-cli.exe llm raw-decode-bench granite-4.0-h-tiny-Q4_K_M.gguf --quantization Q4_K_M".into(),
177            prompt_tokens: 8,
178            decode_tokens: 16,
179            prefill_tok_s: 15.0,
180            decode_tok_s: 0.15,
181            ttft_ms: 500.0,
182            p95_inter_token_ms: 7500.0,
183            peak_vram_mb: 4500,
184        }
185    }
186}
187
188/// Evaluation outcome across the 7 mandatory A2000 parity release gates.
189#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
190pub struct ParityGateEvaluation {
191    pub gate1_baseline_captured: bool,
192    pub gate2_identical_workload: bool,
193    pub gate3_real_generation: bool,
194    pub gate4_memory_bounded: bool,
195    pub gate5_quality_correct: bool,
196    pub gate6_performance_demonstrated: bool,
197    pub gate7_same_or_better: bool,
198}
199
200impl ParityGateEvaluation {
201    /// `true` when every parity gate passed.
202    pub fn all_passed(&self) -> bool {
203        self.gate1_baseline_captured
204            && self.gate2_identical_workload
205            && self.gate3_real_generation
206            && self.gate4_memory_bounded
207            && self.gate5_quality_correct
208            && self.gate6_performance_demonstrated
209            && self.gate7_same_or_better
210    }
211}
212
213#[cfg(test)]
214mod tests {
215    use super::*;
216    use crate::inference::runtime::receipt::BackendKind;
217
218    #[test]
219    fn manifest_round_trip_and_validation() {
220        let manifest = BenchmarkManifest {
221            schema_version: MANIFEST_SCHEMA_VERSION,
222            benchmark_kind: "raw-decode".into(),
223            executable_commit: "abc".into(),
224            dirty_diff_hash: "def".into(),
225            executable_sha256: "c".repeat(64),
226            model_path: "model.gguf".into(),
227            model_sha256: "a".repeat(64),
228            prompt_token_sha256: "b".repeat(64),
229            prompt_tokens: 32,
230            context_window: 1024,
231            decode_policy: RAW_GREEDY_DECODE_POLICY.into(),
232            quantization: "Q8_0".into(),
233            decode_steps_requested: 256,
234            decode_steps_executed: 256,
235            warmup_runs: 1,
236            measured_runs: 5,
237            median_tok_s: 100.0,
238            p95_ms_per_token: 11.0,
239            receipt: ExecutionReceipt::new(BackendKind::Cuda, BackendKind::Cuda, "model", "plan"),
240        };
241        manifest.validate().unwrap();
242        let json = serde_json::to_string(&manifest).unwrap();
243        let decoded: BenchmarkManifest = serde_json::from_str(&json).unwrap();
244        assert_eq!(decoded, manifest);
245    }
246
247    #[test]
248    fn manifest_rejects_undeclared_decode_policy() {
249        let mut manifest = BenchmarkManifest {
250            schema_version: MANIFEST_SCHEMA_VERSION,
251            benchmark_kind: "raw-decode".into(),
252            executable_commit: "abc".into(),
253            dirty_diff_hash: "def".into(),
254            executable_sha256: "c".repeat(64),
255            model_path: "model.gguf".into(),
256            model_sha256: "a".repeat(64),
257            prompt_token_sha256: "b".repeat(64),
258            prompt_tokens: 5,
259            context_window: 1024,
260            decode_policy: RAW_GREEDY_DECODE_POLICY.into(),
261            quantization: "Q8_0".into(),
262            decode_steps_requested: 256,
263            decode_steps_executed: 256,
264            warmup_runs: 1,
265            measured_runs: 5,
266            median_tok_s: 100.0,
267            p95_ms_per_token: 11.0,
268            receipt: ExecutionReceipt::new(BackendKind::Cuda, BackendKind::Cuda, "model", "plan"),
269        };
270        manifest.decode_policy.clear();
271        assert_eq!(manifest.validate(), Err("decode policy is required"));
272    }
273
274    #[test]
275    fn hardware_profile_and_osrp_baseline_roundtrip() {
276        let hw = HardwareProfileManifest::rtx_a2000_12gb();
277        assert_eq!(hw.compute_capability, (8, 6));
278        assert_eq!(hw.total_vram_mb, 12288);
279        let hw_json = serde_json::to_string(&hw).unwrap();
280        let hw_dec: HardwareProfileManifest = serde_json::from_str(&hw_json).unwrap();
281        assert_eq!(hw_dec, hw);
282
283        let osrp = OsrpBaselineManifest::qwen3_6_35b_nvfp4_reference();
284        assert_eq!(osrp.model_id, "Qwen3.6-35B-A3B-NVFP4");
285        assert!(osrp.decode_tok_s > 30.0);
286        let osrp_json = serde_json::to_string(&osrp).unwrap();
287        let osrp_dec: OsrpBaselineManifest = serde_json::from_str(&osrp_json).unwrap();
288        assert_eq!(osrp_dec, osrp);
289
290        let gates = ParityGateEvaluation {
291            gate1_baseline_captured: true,
292            gate2_identical_workload: true,
293            gate3_real_generation: true,
294            gate4_memory_bounded: true,
295            gate5_quality_correct: true,
296            gate6_performance_demonstrated: true,
297            gate7_same_or_better: true,
298        };
299        assert!(gates.all_passed());
300
301        let incomplete_gates = ParityGateEvaluation {
302            gate7_same_or_better: false,
303            ..gates
304        };
305        assert!(!incomplete_gates.all_passed());
306    }
307}