qualia_core_db/query/ingest_job/
status.rs1use serde::Serialize;
4use std::fs;
5use std::io;
6use std::path::Path;
7
8use super::job::{IngestJob, IngestJobPhase, JOB_PROGRESS};
9use super::resume::resume_is_supported;
10use crate::query::ingest_report::{format_bytes, format_count, IngestSnapshot};
11
12#[derive(Clone, Debug, Serialize)]
13pub struct IngestJobStatus {
14 pub dir: String,
15 pub phase: IngestJobPhase,
16 pub accepted_triples: u64,
17 pub quin_chunks: u64,
18 pub lex_runs: u64,
19 pub run_bytes: u64,
20 pub source: String,
21 pub output: String,
22 pub output_exists: bool,
23 pub resumable: bool,
24 pub complete: bool,
25 pub live: Option<IngestSnapshot>,
27 pub summary: String,
28}
29
30pub fn job_status(dir: &Path) -> io::Result<IngestJobStatus> {
32 let job = IngestJob::open(dir.to_path_buf())?;
33 let live = read_progress_json(&dir.join(JOB_PROGRESS));
34 let run_bytes = dir_bytes(&job.runs_dir());
35 let output_exists = Path::new(&job.spec.output).is_file();
36 let summary = format_status_line(&job, live.as_ref(), run_bytes, output_exists);
37 Ok(IngestJobStatus {
38 dir: dir.display().to_string(),
39 phase: job.checkpoint.phase,
40 accepted_triples: job.checkpoint.triples,
41 quin_chunks: job.checkpoint.quin_chunks,
42 lex_runs: job.checkpoint.lex_runs,
43 run_bytes,
44 source: job.spec.source.locator().to_string(),
45 output: job.spec.output.clone(),
46 output_exists,
47 resumable: resume_is_supported(&job),
48 complete: job.is_complete(),
49 live,
50 summary,
51 })
52}
53
54pub fn read_progress_json(path: &Path) -> Option<IngestSnapshot> {
56 let raw = fs::read(path).ok()?;
57 serde_json::from_slice(&raw).ok()
58}
59
60fn dir_bytes(dir: &Path) -> u64 {
61 let rd = match fs::read_dir(dir) {
62 Ok(rd) => rd,
63 Err(_) => return 0,
64 };
65 rd.flatten()
66 .filter_map(|e| e.metadata().ok())
67 .map(|m| m.len())
68 .sum()
69}
70
71fn format_status_line(
72 job: &IngestJob,
73 live: Option<&IngestSnapshot>,
74 run_bytes: u64,
75 output_exists: bool,
76) -> String {
77 if let Some(snap) = live {
78 let extra = if snap.skip_target > 0 && snap.phase.as_str() == "skip" {
79 format!(
80 " skip {} / {}",
81 format_count(snap.triples),
82 format_count(snap.skip_target)
83 )
84 } else {
85 String::new()
86 };
87 return format!(
88 "{} {} {:.1}% {:.1} MB/s eta {} runs {}{}",
89 snap.phase.as_str(),
90 format_count(snap.triples),
91 snap.pct,
92 snap.mb_per_sec,
93 snap.eta_secs
94 .map(|s| format!("{:.0}s", s))
95 .unwrap_or_else(|| "—".into()),
96 format_bytes(run_bytes),
97 extra
98 );
99 }
100 format!(
101 "checkpoint {:?} accepted {} chunks {} lex {} runs {} output {}",
102 job.checkpoint.phase,
103 format_count(job.checkpoint.triples),
104 job.checkpoint.quin_chunks,
105 job.checkpoint.lex_runs,
106 format_bytes(run_bytes),
107 if output_exists { "present" } else { "none" }
108 )
109}
110
111#[cfg(test)]
112mod tests {
113 use super::*;
114 use crate::query::ingest::IngestMode;
115 use crate::query::ingest_job::{IngestEncoding, IngestRdfFormat, IngestSourceKind};
116 use crate::query::ingest_report::{IngestPhase, IngestReport};
117
118 #[test]
119 fn status_reads_live_progress_file() {
120 let dir = tempfile::TempDir::new().unwrap();
121 let job_dir = dir.path().join("job");
122 let src = dir.path().join("a.nt");
123 let out = dir.path().join("a.q42");
124 fs::write(&src, b"").unwrap();
125 IngestJob::create(
126 job_dir.clone(),
127 IngestSourceKind::File {
128 path: src.to_string_lossy().into_owned(),
129 },
130 IngestEncoding::Identity,
131 IngestRdfFormat::NTriples,
132 IngestMode::Complete,
133 None,
134 &out,
135 )
136 .unwrap();
137 let report = IngestReport::silent();
138 report.attach_progress_file(job_dir.join(JOB_PROGRESS));
139 report.set_skip_target(1_808_000_000);
140 report.set_source_bytes(100);
141 report.add_bytes_read(46);
142 report.maybe_tick_skip(2_000_000, 1_808_000_000);
143 let st = job_status(&job_dir).unwrap();
144 assert!(st.resumable);
145 assert!(!st.complete);
146 let live = st.live.expect("progress.json");
147 assert_eq!(live.phase, IngestPhase::Skipping);
148 assert!(st.summary.contains("skip"), "{}", st.summary);
149 }
150}