Skip to main content

qualia_core_db/query/ingest_job/
status.rs

1//! Human/agent status for a durable ingest job: checkpoint + live progress.json.
2
3use serde::Serialize;
4use std::fs;
5use std::io;
6use std::path::Path;
7
8use super::job::{IngestJob, IngestJobPhase, JOB_PROGRESS};
9use super::resume::resume_is_supported;
10use crate::query::ingest_report::{format_bytes, format_count, IngestSnapshot};
11
12#[derive(Clone, Debug, Serialize)]
13pub struct IngestJobStatus {
14    pub dir: String,
15    pub phase: IngestJobPhase,
16    pub accepted_triples: u64,
17    pub quin_chunks: u64,
18    pub lex_runs: u64,
19    pub run_bytes: u64,
20    pub source: String,
21    pub output: String,
22    pub output_exists: bool,
23    pub resumable: bool,
24    pub complete: bool,
25    /// Live tick from `progress.json` (this process or a still-running continue).
26    pub live: Option<IngestSnapshot>,
27    pub summary: String,
28}
29
30/// `?::job_status`.
31pub fn job_status(dir: &Path) -> io::Result<IngestJobStatus> {
32    let job = IngestJob::open(dir.to_path_buf())?;
33    let live = read_progress_json(&dir.join(JOB_PROGRESS));
34    let run_bytes = dir_bytes(&job.runs_dir());
35    let output_exists = Path::new(&job.spec.output).is_file();
36    let summary = format_status_line(&job, live.as_ref(), run_bytes, output_exists);
37    Ok(IngestJobStatus {
38        dir: dir.display().to_string(),
39        phase: job.checkpoint.phase,
40        accepted_triples: job.checkpoint.triples,
41        quin_chunks: job.checkpoint.quin_chunks,
42        lex_runs: job.checkpoint.lex_runs,
43        run_bytes,
44        source: job.spec.source.locator().to_string(),
45        output: job.spec.output.clone(),
46        output_exists,
47        resumable: resume_is_supported(&job),
48        complete: job.is_complete(),
49        live,
50        summary,
51    })
52}
53
54/// Reads `progress-json`.
55pub fn read_progress_json(path: &Path) -> Option<IngestSnapshot> {
56    let raw = fs::read(path).ok()?;
57    serde_json::from_slice(&raw).ok()
58}
59
60fn dir_bytes(dir: &Path) -> u64 {
61    let rd = match fs::read_dir(dir) {
62        Ok(rd) => rd,
63        Err(_) => return 0,
64    };
65    rd.flatten()
66        .filter_map(|e| e.metadata().ok())
67        .map(|m| m.len())
68        .sum()
69}
70
71fn format_status_line(
72    job: &IngestJob,
73    live: Option<&IngestSnapshot>,
74    run_bytes: u64,
75    output_exists: bool,
76) -> String {
77    if let Some(snap) = live {
78        let extra = if snap.skip_target > 0 && snap.phase.as_str() == "skip" {
79            format!(
80                "  skip {} / {}",
81                format_count(snap.triples),
82                format_count(snap.skip_target)
83            )
84        } else {
85            String::new()
86        };
87        return format!(
88            "{}  {}  {:.1}%  {:.1} MB/s  eta {}  runs {}{}",
89            snap.phase.as_str(),
90            format_count(snap.triples),
91            snap.pct,
92            snap.mb_per_sec,
93            snap.eta_secs
94                .map(|s| format!("{:.0}s", s))
95                .unwrap_or_else(|| "—".into()),
96            format_bytes(run_bytes),
97            extra
98        );
99    }
100    format!(
101        "checkpoint {:?}  accepted {}  chunks {}  lex {}  runs {}  output {}",
102        job.checkpoint.phase,
103        format_count(job.checkpoint.triples),
104        job.checkpoint.quin_chunks,
105        job.checkpoint.lex_runs,
106        format_bytes(run_bytes),
107        if output_exists { "present" } else { "none" }
108    )
109}
110
111#[cfg(test)]
112mod tests {
113    use super::*;
114    use crate::query::ingest::IngestMode;
115    use crate::query::ingest_job::{IngestEncoding, IngestRdfFormat, IngestSourceKind};
116    use crate::query::ingest_report::{IngestPhase, IngestReport};
117
118    #[test]
119    fn status_reads_live_progress_file() {
120        let dir = tempfile::TempDir::new().unwrap();
121        let job_dir = dir.path().join("job");
122        let src = dir.path().join("a.nt");
123        let out = dir.path().join("a.q42");
124        fs::write(&src, b"").unwrap();
125        IngestJob::create(
126            job_dir.clone(),
127            IngestSourceKind::File {
128                path: src.to_string_lossy().into_owned(),
129            },
130            IngestEncoding::Identity,
131            IngestRdfFormat::NTriples,
132            IngestMode::Complete,
133            None,
134            &out,
135        )
136        .unwrap();
137        let report = IngestReport::silent();
138        report.attach_progress_file(job_dir.join(JOB_PROGRESS));
139        report.set_skip_target(1_808_000_000);
140        report.set_source_bytes(100);
141        report.add_bytes_read(46);
142        report.maybe_tick_skip(2_000_000, 1_808_000_000);
143        let st = job_status(&job_dir).unwrap();
144        assert!(st.resumable);
145        assert!(!st.complete);
146        let live = st.live.expect("progress.json");
147        assert_eq!(live.phase, IngestPhase::Skipping);
148        assert!(st.summary.contains("skip"), "{}", st.summary);
149    }
150}