qualia_core_db/query/ingest_job/
job.rs1use serde::{Deserialize, Serialize};
4use std::fs::{self, File};
5use std::io::{self, Write};
6use std::path::{Path, PathBuf};
7
8use super::source::{IngestEncoding, IngestRdfFormat, IngestSourceKind, WINDOW_BYTES};
9use crate::query::ingest::IngestMode;
10
11pub const JOB_SPEC: &str = "job.json";
12pub const JOB_CHECKPOINT: &str = "checkpoint.json";
13pub const JOB_ATTESTATION: &str = "source-attestation.json";
14pub const JOB_WINDOWS: &str = "windows.sha256";
15pub const JOB_PROGRESS: &str = "progress.json";
16pub const JOB_RUNS: &str = "runs";
17
18#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "snake_case")]
20pub enum IngestJobPhase {
21 Starting,
22 Parsing,
23 Sorting,
24 Publishing,
25 Complete,
26 Failed,
27}
28
29#[derive(Clone, Debug, Serialize, Deserialize)]
30pub struct IngestJobSpec {
31 pub schema: u32,
32 pub source: IngestSourceKind,
33 pub encoding: IngestEncoding,
34 pub format: IngestRdfFormat,
35 pub mode: String,
36 pub segment_mib: Option<u64>,
37 pub output: String,
38 pub created_unix: u64,
39}
40
41#[derive(Clone, Debug, Serialize, Deserialize)]
42pub struct IngestCheckpoint {
43 pub schema: u32,
44 pub phase: IngestJobPhase,
45 pub triples: u64,
46 pub uncompressed_bytes: u64,
47 pub wire_bytes: u64,
48 pub quin_chunks: u64,
49 pub lex_runs: u64,
50 pub last_error: Option<String>,
51 pub updated_unix: u64,
52}
53
54#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
55pub struct SourceAttestation {
56 pub locator: String,
57 pub encoding: IngestEncoding,
58 pub format: IngestRdfFormat,
59 pub uncompressed_bytes: u64,
60 pub wire_bytes: Option<u64>,
61 pub triples: u64,
62 pub uncompressed_sha256_hex: Option<String>,
65 pub wire_sha256_hex: Option<String>,
67 pub window_commitment_hex: Option<String>,
70 pub window_bytes: u64,
71 pub window_count: u64,
72 pub etag: Option<String>,
73 pub content_length: Option<u64>,
74 pub retrieved_unix: u64,
75}
76
77impl SourceAttestation {
78 pub fn verify_story(&self) -> &'static str {
80 if self.uncompressed_sha256_hex.is_some() {
81 "re-stream the locator (file or URL), decompress the same way, SHA-256 the bytes Rio saw, compare uncompressed_sha256_hex. The source file does not need to stay on disk."
82 } else if self.window_commitment_hex.is_some() {
83 "job did not finish a single-pass digest. Re-stream and compare 16 MiB window hashes to windows.sha256; full file SHA-256 is unknown until a complete pass."
84 } else {
85 "no digest recorded yet — nothing to verify against."
86 }
87 }
88}
89
90#[derive(Clone, Debug)]
91pub struct IngestJob {
92 pub dir: PathBuf,
93 pub spec: IngestJobSpec,
94 pub checkpoint: IngestCheckpoint,
95}
96
97impl IngestJob {
98 pub fn create(
100 dir: PathBuf,
101 source: IngestSourceKind,
102 encoding: IngestEncoding,
103 format: IngestRdfFormat,
104 mode: IngestMode,
105 segment_mib: Option<u64>,
106 output: &Path,
107 ) -> io::Result<Self> {
108 fs::create_dir_all(dir.join(JOB_RUNS))?;
109 let spec = IngestJobSpec {
110 schema: 1,
111 source,
112 encoding,
113 format,
114 mode: match mode {
115 IngestMode::Complete => "complete".into(),
116 IngestMode::StripLiterals => "strip_literals".into(),
117 },
118 segment_mib,
119 output: output.to_string_lossy().into_owned(),
120 created_unix: unix_now(),
121 };
122 let checkpoint = IngestCheckpoint {
123 schema: 1,
124 phase: IngestJobPhase::Starting,
125 triples: 0,
126 uncompressed_bytes: 0,
127 wire_bytes: 0,
128 quin_chunks: 0,
129 lex_runs: 0,
130 last_error: None,
131 updated_unix: unix_now(),
132 };
133 let job = Self {
134 dir,
135 spec,
136 checkpoint,
137 };
138 job.write_spec()?;
139 job.write_checkpoint()?;
140 Ok(job)
141 }
142
143 pub fn open(dir: PathBuf) -> io::Result<Self> {
145 let spec = read_json(&dir.join(JOB_SPEC))?;
146 let checkpoint = read_json(&dir.join(JOB_CHECKPOINT))?;
147 Ok(Self {
148 dir,
149 spec,
150 checkpoint,
151 })
152 }
153
154 pub fn runs_dir(&self) -> PathBuf {
156 self.dir.join(JOB_RUNS)
157 }
158
159 pub fn is_complete(&self) -> bool {
161 self.checkpoint.phase == IngestJobPhase::Complete
162 }
163
164 pub fn write_spec(&self) -> io::Result<()> {
166 write_json_atomic(&self.dir.join(JOB_SPEC), &self.spec)
167 }
168
169 pub fn write_checkpoint(&self) -> io::Result<()> {
171 write_json_atomic(&self.dir.join(JOB_CHECKPOINT), &self.checkpoint)
172 }
173
174 pub fn set_phase(&mut self, phase: IngestJobPhase) -> io::Result<()> {
176 self.checkpoint.phase = phase;
177 self.checkpoint.updated_unix = unix_now();
178 self.write_checkpoint()
179 }
180
181 pub fn record_progress(
183 &mut self,
184 triples: u64,
185 uncompressed_bytes: u64,
186 quin_chunks: u64,
187 lex_runs: u64,
188 ) -> io::Result<()> {
189 self.checkpoint.phase = IngestJobPhase::Parsing;
190 self.checkpoint.triples = triples;
191 self.checkpoint.uncompressed_bytes = uncompressed_bytes;
192 self.checkpoint.quin_chunks = quin_chunks;
193 self.checkpoint.lex_runs = lex_runs;
194 self.checkpoint.updated_unix = unix_now();
195 self.write_checkpoint()
196 }
197
198 pub fn mark_failed(&mut self, err: impl ToString) -> io::Result<()> {
200 self.checkpoint.phase = IngestJobPhase::Failed;
201 self.checkpoint.last_error = Some(err.to_string());
202 self.checkpoint.updated_unix = unix_now();
203 self.write_checkpoint()
204 }
205
206 pub fn write_attestation(&self, att: &SourceAttestation) -> io::Result<()> {
208 write_json_atomic(&self.dir.join(JOB_ATTESTATION), att)
209 }
210
211 pub fn attestation_path(&self) -> PathBuf {
213 self.dir.join(JOB_ATTESTATION)
214 }
215
216 pub fn windows_path(&self) -> PathBuf {
218 self.dir.join(JOB_WINDOWS)
219 }
220
221 pub fn load_attestation(&self) -> io::Result<Option<SourceAttestation>> {
223 let path = self.attestation_path();
224 if !path.is_file() {
225 return Ok(None);
226 }
227 Ok(Some(read_json(&path)?))
228 }
229}
230
231pub fn unix_now() -> u64 {
233 std::time::SystemTime::now()
234 .duration_since(std::time::UNIX_EPOCH)
235 .unwrap_or_default()
236 .as_secs()
237}
238
239pub fn write_json_atomic(path: &Path, value: &impl Serialize) -> io::Result<()> {
241 if let Some(parent) = path.parent() {
242 fs::create_dir_all(parent)?;
243 }
244 let tmp = path.with_extension("json.tmp");
245 let mut f = File::create(&tmp)?;
246 serde_json::to_writer_pretty(&mut f, value)
247 .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
248 f.write_all(b"\n")?;
249 f.flush()?;
250 fs::rename(tmp, path)
251}
252
253pub fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> io::Result<T> {
255 let bytes = fs::read(path)?;
256 serde_json::from_slice(&bytes).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
257}
258
259pub fn append_window_hash(path: &Path, hash: &[u8; 32]) -> io::Result<()> {
261 use std::io::Write as _;
262 if let Some(parent) = path.parent() {
263 fs::create_dir_all(parent)?;
264 }
265 let mut f = fs::OpenOptions::new()
266 .create(true)
267 .append(true)
268 .open(path)?;
269 writeln!(f, "{}", hex_encode(hash))
270}
271
272pub fn read_window_hashes(path: &Path) -> io::Result<Vec<[u8; 32]>> {
274 if !path.is_file() {
275 return Ok(Vec::new());
276 }
277 let text = fs::read_to_string(path)?;
278 let mut out = Vec::new();
279 for line in text.lines() {
280 let line = line.trim();
281 if line.is_empty() {
282 continue;
283 }
284 out.push(hex_decode32(line)?);
285 }
286 Ok(out)
287}
288
289pub fn window_commitment(windows: &[[u8; 32]]) -> [u8; 32] {
291 use sha2::{Digest, Sha256};
292 let mut h = Sha256::new();
293 h.update((WINDOW_BYTES as u64).to_le_bytes());
294 h.update((windows.len() as u64).to_le_bytes());
295 for w in windows {
296 h.update(w);
297 }
298 h.finalize().into()
299}
300
301pub fn hex_encode(bytes: &[u8]) -> String {
303 const HEX: &[u8; 16] = b"0123456789abcdef";
304 let mut s = String::with_capacity(bytes.len() * 2);
305 for b in bytes {
306 s.push(HEX[(b >> 4) as usize] as char);
307 s.push(HEX[(b & 0x0f) as usize] as char);
308 }
309 s
310}
311
312pub fn hex_decode32(s: &str) -> io::Result<[u8; 32]> {
314 let s = s.trim();
315 if s.len() != 64 {
316 return Err(io::Error::new(
317 io::ErrorKind::InvalidData,
318 "expected 64 hex chars",
319 ));
320 }
321 let mut out = [0u8; 32];
322 for i in 0..32 {
323 let hi = hex_nibble(s.as_bytes()[i * 2])?;
324 let lo = hex_nibble(s.as_bytes()[i * 2 + 1])?;
325 out[i] = (hi << 4) | lo;
326 }
327 Ok(out)
328}
329
330#[cfg(test)]
331mod tests {
332 use super::*;
333 use crate::query::ingest::IngestMode;
334 use crate::query::ingest_job::source::{IngestEncoding, IngestRdfFormat, IngestSourceKind};
335
336 #[test]
337 fn job_round_trip_and_window_commitment() {
338 let dir = tempfile::TempDir::new().unwrap();
339 let job = IngestJob::create(
340 dir.path().to_path_buf(),
341 IngestSourceKind::File {
342 path: "x.nt".into(),
343 },
344 IngestEncoding::Identity,
345 IngestRdfFormat::NTriples,
346 IngestMode::Complete,
347 Some(512),
348 Path::new("out.q42"),
349 )
350 .unwrap();
351 assert!(job.runs_dir().is_dir());
352 append_window_hash(&job.windows_path(), &[1u8; 32]).unwrap();
353 append_window_hash(&job.windows_path(), &[2u8; 32]).unwrap();
354 let wins = read_window_hashes(&job.windows_path()).unwrap();
355 assert_eq!(wins.len(), 2);
356 let c = window_commitment(&wins);
357 assert_ne!(c, [0u8; 32]);
358 let opened = IngestJob::open(dir.path().to_path_buf()).unwrap();
359 assert_eq!(opened.spec.output, "out.q42");
360 }
361}
362
363fn hex_nibble(b: u8) -> io::Result<u8> {
364 match b {
365 b'0'..=b'9' => Ok(b - b'0'),
366 b'a'..=b'f' => Ok(b - b'a' + 10),
367 b'A'..=b'F' => Ok(b - b'A' + 10),
368 _ => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid hex")),
369 }
370}