Skip to main content

qualia_core_db/query/ingest_job/
inspect.rs

1//! Review a job directory, a legacy scratch tree, or a volume root.
2
3use serde::Serialize;
4use std::fs;
5use std::io;
6use std::path::{Path, PathBuf};
7
8use super::job::{IngestJob, IngestJobPhase, SourceAttestation};
9use crate::q42_volume::Q42Volume;
10
11#[derive(Clone, Debug, Serialize)]
12pub struct IngestInspectReport {
13    pub kind: String,
14    pub path: String,
15    pub phase: Option<IngestJobPhase>,
16    pub triples: u64,
17    pub quin_chunks: u64,
18    pub lex_runs: u64,
19    pub run_bytes: u64,
20    pub last_chunk: Option<String>,
21    pub last_write_unix: Option<u64>,
22    pub complete: bool,
23    pub resumable: bool,
24    pub notes: Vec<String>,
25    pub attestation: Option<SourceAttestation>,
26    pub volume_segments: Option<u64>,
27    pub volume_lex_shards: Option<u64>,
28}
29
30/// Inspects an ingest job.
31pub fn inspect_job(dir: &Path) -> io::Result<IngestInspectReport> {
32    let job = IngestJob::open(dir.to_path_buf())?;
33    let runs = summarize_runs(&job.runs_dir())?;
34    let resumable = super::resume::resume_is_supported(&job);
35    let mut notes = Vec::new();
36    notes.push(format!("source {}", job.spec.source.locator()));
37    notes.push(format!(
38        "encoding {:?} format {:?} mode {}",
39        job.spec.encoding, job.spec.format, job.spec.mode
40    ));
41    if let Some(att) = job.load_attestation()? {
42        notes.push(att.verify_story().to_string());
43    }
44    if !job.is_complete() {
45        notes.push("job is incomplete — use ingest-job continue if resumable".into());
46    }
47    if let Some(live) = super::status::read_progress_json(&dir.join(super::job::JOB_PROGRESS)) {
48        notes.push(format!(
49            "live {}  {:.1}%  {:.1} MB/s  {} triples",
50            live.phase.as_str(),
51            live.pct,
52            live.mb_per_sec,
53            live.triples
54        ));
55    }
56    Ok(IngestInspectReport {
57        kind: "job".into(),
58        path: dir.display().to_string(),
59        phase: Some(job.checkpoint.phase),
60        triples: job.checkpoint.triples.max(runs.triples_estimate),
61        quin_chunks: runs.quin_chunks,
62        lex_runs: runs.lex_runs,
63        run_bytes: runs.bytes,
64        last_chunk: runs.last_name,
65        last_write_unix: runs.last_write,
66        complete: job.is_complete(),
67        resumable,
68        notes,
69        attestation: job.load_attestation()?,
70        volume_segments: None,
71        volume_lex_shards: None,
72    })
73}
74
75/// Review a TempDir-style scratch (the live OSM job): chunk_*.tmp + lexrun_*.tmp.
76pub fn inspect_legacy_scratch(dir: &Path) -> io::Result<IngestInspectReport> {
77    let search = find_run_dir(dir)?;
78    let runs = summarize_runs(&search)?;
79    let mut notes = vec![
80        "legacy scratch (no job.json) — this is what the pre-job OSM ingest writes".into(),
81        "cannot resume this layout as a job until `ingest-job adopt-scratch` copies it".into(),
82        format!(
83            "quin estimate ≈ {} (1M quins per full 45.8 MiB chunk)",
84            runs.triples_estimate
85        ),
86    ];
87    if runs.partial_tail {
88        notes.push(
89            "last chunk is not a multiple of 48 bytes — treat as torn; do not adopt it".into(),
90        );
91    }
92    Ok(IngestInspectReport {
93        kind: "legacy_scratch".into(),
94        path: search.display().to_string(),
95        phase: Some(IngestJobPhase::Parsing),
96        triples: runs.triples_estimate,
97        quin_chunks: runs.quin_chunks,
98        lex_runs: runs.lex_runs,
99        run_bytes: runs.bytes,
100        last_chunk: runs.last_name,
101        last_write_unix: runs.last_write,
102        complete: false,
103        resumable: false,
104        notes,
105        attestation: None,
106        volume_segments: None,
107        volume_lex_shards: None,
108    })
109}
110
111/// Inspects a volume root.
112pub fn inspect_volume_root(path: &Path) -> io::Result<IngestInspectReport> {
113    let vol = Q42Volume::open(path)?;
114    let mut notes = Vec::new();
115    let (segs, lex) = match vol.volume_manifest()? {
116        Some(m) => {
117            notes.push(format!("generation {}", m.generation));
118            (m.segments.len() as u64, m.lexicon_segments.len() as u64)
119        }
120        None => {
121            notes.push("single-file volume (no volume-set manifest)".into());
122            (0, 0)
123        }
124    };
125    let att_path = path.with_extension("q42.source.json");
126    let attestation = if att_path.is_file() {
127        Some(super::job::read_json(&att_path)?)
128    } else {
129        notes.push("no sidecar source-attestation (*.q42.source.json)".into());
130        None
131    };
132    Ok(IngestInspectReport {
133        kind: "volume".into(),
134        path: path.display().to_string(),
135        phase: Some(IngestJobPhase::Complete),
136        triples: 0,
137        quin_chunks: 0,
138        lex_runs: 0,
139        run_bytes: fs::metadata(path).map(|m| m.len()).unwrap_or(0),
140        last_chunk: None,
141        last_write_unix: None,
142        complete: true,
143        resumable: false,
144        notes,
145        attestation,
146        volume_segments: Some(segs),
147        volume_lex_shards: Some(lex),
148    })
149}
150
151struct RunSummary {
152    quin_chunks: u64,
153    lex_runs: u64,
154    bytes: u64,
155    triples_estimate: u64,
156    last_name: Option<String>,
157    last_write: Option<u64>,
158    partial_tail: bool,
159}
160
161fn summarize_runs(dir: &Path) -> io::Result<RunSummary> {
162    let mut quin_chunks = 0u64;
163    let mut lex_runs = 0u64;
164    let mut bytes = 0u64;
165    let mut triples_estimate = 0u64;
166    let mut last_name = None;
167    let mut last_write = None::<u64>;
168    let mut last_mtime = 0u64;
169    let mut partial_tail = false;
170    if !dir.is_dir() {
171        return Ok(RunSummary {
172            quin_chunks,
173            lex_runs,
174            bytes,
175            triples_estimate,
176            last_name,
177            last_write,
178            partial_tail,
179        });
180    }
181    let mut chunks: Vec<PathBuf> = Vec::new();
182    for ent in fs::read_dir(dir)? {
183        let ent = ent?;
184        let path = ent.path();
185        let name = path
186            .file_name()
187            .and_then(|n| n.to_str())
188            .unwrap_or("")
189            .to_string();
190        let meta = ent.metadata()?;
191        let len = meta.len();
192        bytes += len;
193        let mtime = meta
194            .modified()
195            .ok()
196            .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
197            .map(|d| d.as_secs())
198            .unwrap_or(0);
199        if mtime >= last_mtime {
200            last_mtime = mtime;
201            last_write = Some(mtime);
202            last_name = Some(name.clone());
203        }
204        if name.starts_with("chunk_") && name.ends_with(".tmp") {
205            quin_chunks += 1;
206            chunks.push(path);
207            if len % 48 != 0 {
208                partial_tail = true;
209            } else {
210                triples_estimate += len / 48;
211            }
212        } else if name.starts_with("lexrun_") && name.ends_with(".tmp") {
213            lex_runs += 1;
214        }
215    }
216    let _ = chunks;
217    Ok(RunSummary {
218        quin_chunks,
219        lex_runs,
220        bytes,
221        triples_estimate,
222        last_name,
223        last_write,
224        partial_tail,
225    })
226}
227
228fn find_run_dir(dir: &Path) -> io::Result<PathBuf> {
229    if dir.join("job.json").is_file() {
230        return Ok(dir.join("runs"));
231    }
232    // TempDir nest: scratch/.tmpXXXX/chunk_0.tmp
233    let mut best = dir.to_path_buf();
234    let mut best_n = count_chunks(dir);
235    if let Ok(rd) = fs::read_dir(dir) {
236        for ent in rd.flatten() {
237            if ent.path().is_dir() {
238                let n = count_chunks(&ent.path());
239                if n > best_n {
240                    best_n = n;
241                    best = ent.path();
242                }
243            }
244        }
245    }
246    Ok(best)
247}
248
249fn count_chunks(dir: &Path) -> u64 {
250    fs::read_dir(dir)
251        .map(|rd| {
252            rd.flatten()
253                .filter(|e| e.file_name().to_string_lossy().starts_with("chunk_"))
254                .count() as u64
255        })
256        .unwrap_or(0)
257}