Skip to main content

qualia_core_db/query/ingest_job/
job.rs

1//! On-disk job specification, checkpoint, and source attestation.
2
3use serde::{Deserialize, Serialize};
4use std::fs::{self, File};
5use std::io::{self, Write};
6use std::path::{Path, PathBuf};
7
8use super::source::{IngestEncoding, IngestRdfFormat, IngestSourceKind, WINDOW_BYTES};
9use crate::query::ingest::IngestMode;
10
11pub const JOB_SPEC: &str = "job.json";
12pub const JOB_CHECKPOINT: &str = "checkpoint.json";
13pub const JOB_ATTESTATION: &str = "source-attestation.json";
14pub const JOB_WINDOWS: &str = "windows.sha256";
15pub const JOB_PROGRESS: &str = "progress.json";
16pub const JOB_RUNS: &str = "runs";
17
18#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "snake_case")]
20pub enum IngestJobPhase {
21    Starting,
22    Parsing,
23    Sorting,
24    Publishing,
25    Complete,
26    Failed,
27}
28
29#[derive(Clone, Debug, Serialize, Deserialize)]
30pub struct IngestJobSpec {
31    pub schema: u32,
32    pub source: IngestSourceKind,
33    pub encoding: IngestEncoding,
34    pub format: IngestRdfFormat,
35    pub mode: String,
36    pub segment_mib: Option<u64>,
37    pub output: String,
38    pub created_unix: u64,
39}
40
41#[derive(Clone, Debug, Serialize, Deserialize)]
42pub struct IngestCheckpoint {
43    pub schema: u32,
44    pub phase: IngestJobPhase,
45    pub triples: u64,
46    pub uncompressed_bytes: u64,
47    pub wire_bytes: u64,
48    pub quin_chunks: u64,
49    pub lex_runs: u64,
50    pub last_error: Option<String>,
51    pub updated_unix: u64,
52}
53
54#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
55pub struct SourceAttestation {
56    pub locator: String,
57    pub encoding: IngestEncoding,
58    pub format: IngestRdfFormat,
59    pub uncompressed_bytes: u64,
60    pub wire_bytes: Option<u64>,
61    pub triples: u64,
62    /// SHA-256 of the full decompressed source. Present only after a complete
63    /// single-pass hash (no crash mid-stream without resume).
64    pub uncompressed_sha256_hex: Option<String>,
65    /// SHA-256 of the on-the-wire bytes (compressed if the server sent gzip/zstd).
66    pub wire_sha256_hex: Option<String>,
67    /// SHA-256 of the concatenation of 16 MiB window hashes. Always available
68    /// after at least one full window; used to verify a prefix of an incomplete job.
69    pub window_commitment_hex: Option<String>,
70    pub window_bytes: u64,
71    pub window_count: u64,
72    pub etag: Option<String>,
73    pub content_length: Option<u64>,
74    pub retrieved_unix: u64,
75}
76
77impl SourceAttestation {
78    /// Human-readable verification instructions depending on which digest was recorded (full SHA-256, 16 MiB window commitment, or none yet).
79    pub fn verify_story(&self) -> &'static str {
80        if self.uncompressed_sha256_hex.is_some() {
81            "re-stream the locator (file or URL), decompress the same way, SHA-256 the bytes Rio saw, compare uncompressed_sha256_hex. The source file does not need to stay on disk."
82        } else if self.window_commitment_hex.is_some() {
83            "job did not finish a single-pass digest. Re-stream and compare 16 MiB window hashes to windows.sha256; full file SHA-256 is unknown until a complete pass."
84        } else {
85            "no digest recorded yet — nothing to verify against."
86        }
87    }
88}
89
90#[derive(Clone, Debug)]
91pub struct IngestJob {
92    pub dir: PathBuf,
93    pub spec: IngestJobSpec,
94    pub checkpoint: IngestCheckpoint,
95}
96
97impl IngestJob {
98    /// Creates a new ingest job: makes the runs dir, writes the spec (source/encoding/format/mode/segment size), and initializes the checkpoint.
99    pub fn create(
100        dir: PathBuf,
101        source: IngestSourceKind,
102        encoding: IngestEncoding,
103        format: IngestRdfFormat,
104        mode: IngestMode,
105        segment_mib: Option<u64>,
106        output: &Path,
107    ) -> io::Result<Self> {
108        fs::create_dir_all(dir.join(JOB_RUNS))?;
109        let spec = IngestJobSpec {
110            schema: 1,
111            source,
112            encoding,
113            format,
114            mode: match mode {
115                IngestMode::Complete => "complete".into(),
116                IngestMode::StripLiterals => "strip_literals".into(),
117            },
118            segment_mib,
119            output: output.to_string_lossy().into_owned(),
120            created_unix: unix_now(),
121        };
122        let checkpoint = IngestCheckpoint {
123            schema: 1,
124            phase: IngestJobPhase::Starting,
125            triples: 0,
126            uncompressed_bytes: 0,
127            wire_bytes: 0,
128            quin_chunks: 0,
129            lex_runs: 0,
130            last_error: None,
131            updated_unix: unix_now(),
132        };
133        let job = Self {
134            dir,
135            spec,
136            checkpoint,
137        };
138        job.write_spec()?;
139        job.write_checkpoint()?;
140        Ok(job)
141    }
142
143    /// Reopens a job dir, loading spec and checkpoint; `io` errors propagate.
144    pub fn open(dir: PathBuf) -> io::Result<Self> {
145        let spec = read_json(&dir.join(JOB_SPEC))?;
146        let checkpoint = read_json(&dir.join(JOB_CHECKPOINT))?;
147        Ok(Self {
148            dir,
149            spec,
150            checkpoint,
151        })
152    }
153
154    /// Path to the job's `runs/` subdirectory.
155    pub fn runs_dir(&self) -> PathBuf {
156        self.dir.join(JOB_RUNS)
157    }
158
159    /// `true` once the checkpoint phase is `Complete`.
160    pub fn is_complete(&self) -> bool {
161        self.checkpoint.phase == IngestJobPhase::Complete
162    }
163
164    /// Persists the job spec JSON atomically.
165    pub fn write_spec(&self) -> io::Result<()> {
166        write_json_atomic(&self.dir.join(JOB_SPEC), &self.spec)
167    }
168
169    /// Persists the checkpoint JSON atomically.
170    pub fn write_checkpoint(&self) -> io::Result<()> {
171        write_json_atomic(&self.dir.join(JOB_CHECKPOINT), &self.checkpoint)
172    }
173
174    /// Updates the checkpoint phase and persists it.
175    pub fn set_phase(&mut self, phase: IngestJobPhase) -> io::Result<()> {
176        self.checkpoint.phase = phase;
177        self.checkpoint.updated_unix = unix_now();
178        self.write_checkpoint()
179    }
180
181    /// Records bytes/quins processed plus optional window hash and persists the checkpoint.
182    pub fn record_progress(
183        &mut self,
184        triples: u64,
185        uncompressed_bytes: u64,
186        quin_chunks: u64,
187        lex_runs: u64,
188    ) -> io::Result<()> {
189        self.checkpoint.phase = IngestJobPhase::Parsing;
190        self.checkpoint.triples = triples;
191        self.checkpoint.uncompressed_bytes = uncompressed_bytes;
192        self.checkpoint.quin_chunks = quin_chunks;
193        self.checkpoint.lex_runs = lex_runs;
194        self.checkpoint.updated_unix = unix_now();
195        self.write_checkpoint()
196    }
197
198    /// Marks the job failed with `err` and persists the checkpoint.
199    pub fn mark_failed(&mut self, err: impl ToString) -> io::Result<()> {
200        self.checkpoint.phase = IngestJobPhase::Failed;
201        self.checkpoint.last_error = Some(err.to_string());
202        self.checkpoint.updated_unix = unix_now();
203        self.write_checkpoint()
204    }
205
206    /// Writes the source attestation JSON atomically.
207    pub fn write_attestation(&self, att: &SourceAttestation) -> io::Result<()> {
208        write_json_atomic(&self.dir.join(JOB_ATTESTATION), att)
209    }
210
211    /// Path to the attestation file inside the job dir.
212    pub fn attestation_path(&self) -> PathBuf {
213        self.dir.join(JOB_ATTESTATION)
214    }
215
216    /// Path to the per-window hash file.
217    pub fn windows_path(&self) -> PathBuf {
218        self.dir.join(JOB_WINDOWS)
219    }
220
221    /// Reads the attestation JSON; `Ok(None)` when absent.
222    pub fn load_attestation(&self) -> io::Result<Option<SourceAttestation>> {
223        let path = self.attestation_path();
224        if !path.is_file() {
225            return Ok(None);
226        }
227        Ok(Some(read_json(&path)?))
228    }
229}
230
231/// Current UNIX epoch seconds (saturating at 0 on clock skew).
232pub fn unix_now() -> u64 {
233    std::time::SystemTime::now()
234        .duration_since(std::time::UNIX_EPOCH)
235        .unwrap_or_default()
236        .as_secs()
237}
238
239/// Serializes `value` to `<path>.json.tmp` then renames — readers never see a partial file.
240pub fn write_json_atomic(path: &Path, value: &impl Serialize) -> io::Result<()> {
241    if let Some(parent) = path.parent() {
242        fs::create_dir_all(parent)?;
243    }
244    let tmp = path.with_extension("json.tmp");
245    let mut f = File::create(&tmp)?;
246    serde_json::to_writer_pretty(&mut f, value)
247        .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
248    f.write_all(b"\n")?;
249    f.flush()?;
250    fs::rename(tmp, path)
251}
252
253/// Reads and deserializes a JSON file; `InvalidData` on parse failure.
254pub fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> io::Result<T> {
255    let bytes = fs::read(path)?;
256    serde_json::from_slice(&bytes).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
257}
258
259/// Appends one hex-encoded 32-byte window hash line.
260pub fn append_window_hash(path: &Path, hash: &[u8; 32]) -> io::Result<()> {
261    use std::io::Write as _;
262    if let Some(parent) = path.parent() {
263        fs::create_dir_all(parent)?;
264    }
265    let mut f = fs::OpenOptions::new()
266        .create(true)
267        .append(true)
268        .open(path)?;
269    writeln!(f, "{}", hex_encode(hash))
270}
271
272/// Reads all window-hash lines, hex-decoding each; empty vec when absent.
273pub fn read_window_hashes(path: &Path) -> io::Result<Vec<[u8; 32]>> {
274    if !path.is_file() {
275        return Ok(Vec::new());
276    }
277    let text = fs::read_to_string(path)?;
278    let mut out = Vec::new();
279    for line in text.lines() {
280        let line = line.trim();
281        if line.is_empty() {
282            continue;
283        }
284        out.push(hex_decode32(line)?);
285    }
286    Ok(out)
287}
288
289/// SHA-256 over `WINDOW_BYTES` + window count + each hash — binds the commitment to the segmentation scheme.
290pub fn window_commitment(windows: &[[u8; 32]]) -> [u8; 32] {
291    use sha2::{Digest, Sha256};
292    let mut h = Sha256::new();
293    h.update((WINDOW_BYTES as u64).to_le_bytes());
294    h.update((windows.len() as u64).to_le_bytes());
295    for w in windows {
296        h.update(w);
297    }
298    h.finalize().into()
299}
300
301/// Lowercase hex encoding (no external deps).
302pub fn hex_encode(bytes: &[u8]) -> String {
303    const HEX: &[u8; 16] = b"0123456789abcdef";
304    let mut s = String::with_capacity(bytes.len() * 2);
305    for b in bytes {
306        s.push(HEX[(b >> 4) as usize] as char);
307        s.push(HEX[(b & 0x0f) as usize] as char);
308    }
309    s
310}
311
312/// Decodes exactly 64 hex chars into `[u8; 32]`; `InvalidData` otherwise.
313pub fn hex_decode32(s: &str) -> io::Result<[u8; 32]> {
314    let s = s.trim();
315    if s.len() != 64 {
316        return Err(io::Error::new(
317            io::ErrorKind::InvalidData,
318            "expected 64 hex chars",
319        ));
320    }
321    let mut out = [0u8; 32];
322    for i in 0..32 {
323        let hi = hex_nibble(s.as_bytes()[i * 2])?;
324        let lo = hex_nibble(s.as_bytes()[i * 2 + 1])?;
325        out[i] = (hi << 4) | lo;
326    }
327    Ok(out)
328}
329
330#[cfg(test)]
331mod tests {
332    use super::*;
333    use crate::query::ingest::IngestMode;
334    use crate::query::ingest_job::source::{IngestEncoding, IngestRdfFormat, IngestSourceKind};
335
336    #[test]
337    fn job_round_trip_and_window_commitment() {
338        let dir = tempfile::TempDir::new().unwrap();
339        let job = IngestJob::create(
340            dir.path().to_path_buf(),
341            IngestSourceKind::File {
342                path: "x.nt".into(),
343            },
344            IngestEncoding::Identity,
345            IngestRdfFormat::NTriples,
346            IngestMode::Complete,
347            Some(512),
348            Path::new("out.q42"),
349        )
350        .unwrap();
351        assert!(job.runs_dir().is_dir());
352        append_window_hash(&job.windows_path(), &[1u8; 32]).unwrap();
353        append_window_hash(&job.windows_path(), &[2u8; 32]).unwrap();
354        let wins = read_window_hashes(&job.windows_path()).unwrap();
355        assert_eq!(wins.len(), 2);
356        let c = window_commitment(&wins);
357        assert_ne!(c, [0u8; 32]);
358        let opened = IngestJob::open(dir.path().to_path_buf()).unwrap();
359        assert_eq!(opened.spec.output, "out.q42");
360    }
361}
362
363fn hex_nibble(b: u8) -> io::Result<u8> {
364    match b {
365        b'0'..=b'9' => Ok(b - b'0'),
366        b'a'..=b'f' => Ok(b - b'a' + 10),
367        b'A'..=b'F' => Ok(b - b'A' + 10),
368        _ => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid hex")),
369    }
370}