qualia_core_db/query/ingest_job/
inspect.rs1use serde::Serialize;
4use std::fs;
5use std::io;
6use std::path::{Path, PathBuf};
7
8use super::job::{IngestJob, IngestJobPhase, SourceAttestation};
9use crate::q42_volume::Q42Volume;
10
11#[derive(Clone, Debug, Serialize)]
12pub struct IngestInspectReport {
13 pub kind: String,
14 pub path: String,
15 pub phase: Option<IngestJobPhase>,
16 pub triples: u64,
17 pub quin_chunks: u64,
18 pub lex_runs: u64,
19 pub run_bytes: u64,
20 pub last_chunk: Option<String>,
21 pub last_write_unix: Option<u64>,
22 pub complete: bool,
23 pub resumable: bool,
24 pub notes: Vec<String>,
25 pub attestation: Option<SourceAttestation>,
26 pub volume_segments: Option<u64>,
27 pub volume_lex_shards: Option<u64>,
28}
29
30pub fn inspect_job(dir: &Path) -> io::Result<IngestInspectReport> {
32 let job = IngestJob::open(dir.to_path_buf())?;
33 let runs = summarize_runs(&job.runs_dir())?;
34 let resumable = super::resume::resume_is_supported(&job);
35 let mut notes = Vec::new();
36 notes.push(format!("source {}", job.spec.source.locator()));
37 notes.push(format!(
38 "encoding {:?} format {:?} mode {}",
39 job.spec.encoding, job.spec.format, job.spec.mode
40 ));
41 if let Some(att) = job.load_attestation()? {
42 notes.push(att.verify_story().to_string());
43 }
44 if !job.is_complete() {
45 notes.push("job is incomplete — use ingest-job continue if resumable".into());
46 }
47 if let Some(live) = super::status::read_progress_json(&dir.join(super::job::JOB_PROGRESS)) {
48 notes.push(format!(
49 "live {} {:.1}% {:.1} MB/s {} triples",
50 live.phase.as_str(),
51 live.pct,
52 live.mb_per_sec,
53 live.triples
54 ));
55 }
56 Ok(IngestInspectReport {
57 kind: "job".into(),
58 path: dir.display().to_string(),
59 phase: Some(job.checkpoint.phase),
60 triples: job.checkpoint.triples.max(runs.triples_estimate),
61 quin_chunks: runs.quin_chunks,
62 lex_runs: runs.lex_runs,
63 run_bytes: runs.bytes,
64 last_chunk: runs.last_name,
65 last_write_unix: runs.last_write,
66 complete: job.is_complete(),
67 resumable,
68 notes,
69 attestation: job.load_attestation()?,
70 volume_segments: None,
71 volume_lex_shards: None,
72 })
73}
74
75pub fn inspect_legacy_scratch(dir: &Path) -> io::Result<IngestInspectReport> {
77 let search = find_run_dir(dir)?;
78 let runs = summarize_runs(&search)?;
79 let mut notes = vec![
80 "legacy scratch (no job.json) — this is what the pre-job OSM ingest writes".into(),
81 "cannot resume this layout as a job until `ingest-job adopt-scratch` copies it".into(),
82 format!(
83 "quin estimate ≈ {} (1M quins per full 45.8 MiB chunk)",
84 runs.triples_estimate
85 ),
86 ];
87 if runs.partial_tail {
88 notes.push(
89 "last chunk is not a multiple of 48 bytes — treat as torn; do not adopt it".into(),
90 );
91 }
92 Ok(IngestInspectReport {
93 kind: "legacy_scratch".into(),
94 path: search.display().to_string(),
95 phase: Some(IngestJobPhase::Parsing),
96 triples: runs.triples_estimate,
97 quin_chunks: runs.quin_chunks,
98 lex_runs: runs.lex_runs,
99 run_bytes: runs.bytes,
100 last_chunk: runs.last_name,
101 last_write_unix: runs.last_write,
102 complete: false,
103 resumable: false,
104 notes,
105 attestation: None,
106 volume_segments: None,
107 volume_lex_shards: None,
108 })
109}
110
111pub fn inspect_volume_root(path: &Path) -> io::Result<IngestInspectReport> {
113 let vol = Q42Volume::open(path)?;
114 let mut notes = Vec::new();
115 let (segs, lex) = match vol.volume_manifest()? {
116 Some(m) => {
117 notes.push(format!("generation {}", m.generation));
118 (m.segments.len() as u64, m.lexicon_segments.len() as u64)
119 }
120 None => {
121 notes.push("single-file volume (no volume-set manifest)".into());
122 (0, 0)
123 }
124 };
125 let att_path = path.with_extension("q42.source.json");
126 let attestation = if att_path.is_file() {
127 Some(super::job::read_json(&att_path)?)
128 } else {
129 notes.push("no sidecar source-attestation (*.q42.source.json)".into());
130 None
131 };
132 Ok(IngestInspectReport {
133 kind: "volume".into(),
134 path: path.display().to_string(),
135 phase: Some(IngestJobPhase::Complete),
136 triples: 0,
137 quin_chunks: 0,
138 lex_runs: 0,
139 run_bytes: fs::metadata(path).map(|m| m.len()).unwrap_or(0),
140 last_chunk: None,
141 last_write_unix: None,
142 complete: true,
143 resumable: false,
144 notes,
145 attestation,
146 volume_segments: Some(segs),
147 volume_lex_shards: Some(lex),
148 })
149}
150
151struct RunSummary {
152 quin_chunks: u64,
153 lex_runs: u64,
154 bytes: u64,
155 triples_estimate: u64,
156 last_name: Option<String>,
157 last_write: Option<u64>,
158 partial_tail: bool,
159}
160
161fn summarize_runs(dir: &Path) -> io::Result<RunSummary> {
162 let mut quin_chunks = 0u64;
163 let mut lex_runs = 0u64;
164 let mut bytes = 0u64;
165 let mut triples_estimate = 0u64;
166 let mut last_name = None;
167 let mut last_write = None::<u64>;
168 let mut last_mtime = 0u64;
169 let mut partial_tail = false;
170 if !dir.is_dir() {
171 return Ok(RunSummary {
172 quin_chunks,
173 lex_runs,
174 bytes,
175 triples_estimate,
176 last_name,
177 last_write,
178 partial_tail,
179 });
180 }
181 let mut chunks: Vec<PathBuf> = Vec::new();
182 for ent in fs::read_dir(dir)? {
183 let ent = ent?;
184 let path = ent.path();
185 let name = path
186 .file_name()
187 .and_then(|n| n.to_str())
188 .unwrap_or("")
189 .to_string();
190 let meta = ent.metadata()?;
191 let len = meta.len();
192 bytes += len;
193 let mtime = meta
194 .modified()
195 .ok()
196 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
197 .map(|d| d.as_secs())
198 .unwrap_or(0);
199 if mtime >= last_mtime {
200 last_mtime = mtime;
201 last_write = Some(mtime);
202 last_name = Some(name.clone());
203 }
204 if name.starts_with("chunk_") && name.ends_with(".tmp") {
205 quin_chunks += 1;
206 chunks.push(path);
207 if len % 48 != 0 {
208 partial_tail = true;
209 } else {
210 triples_estimate += len / 48;
211 }
212 } else if name.starts_with("lexrun_") && name.ends_with(".tmp") {
213 lex_runs += 1;
214 }
215 }
216 let _ = chunks;
217 Ok(RunSummary {
218 quin_chunks,
219 lex_runs,
220 bytes,
221 triples_estimate,
222 last_name,
223 last_write,
224 partial_tail,
225 })
226}
227
228fn find_run_dir(dir: &Path) -> io::Result<PathBuf> {
229 if dir.join("job.json").is_file() {
230 return Ok(dir.join("runs"));
231 }
232 let mut best = dir.to_path_buf();
234 let mut best_n = count_chunks(dir);
235 if let Ok(rd) = fs::read_dir(dir) {
236 for ent in rd.flatten() {
237 if ent.path().is_dir() {
238 let n = count_chunks(&ent.path());
239 if n > best_n {
240 best_n = n;
241 best = ent.path();
242 }
243 }
244 }
245 }
246 Ok(best)
247}
248
249fn count_chunks(dir: &Path) -> u64 {
250 fs::read_dir(dir)
251 .map(|rd| {
252 rd.flatten()
253 .filter(|e| e.file_name().to_string_lossy().starts_with("chunk_"))
254 .count() as u64
255 })
256 .unwrap_or(0)
257}