qualia_core_db/query/ingest_job/
resume.rs1use std::fs;
4use std::io;
5use std::path::{Path, PathBuf};
6
7use super::job::{IngestJob, JOB_RUNS};
8use super::source::IngestEncoding;
9use super::source::IngestSourceKind;
10use crate::query::ingest::IngestMode;
11
12pub fn resume_is_supported(job: &IngestJob) -> bool {
17 !job.is_complete()
18}
19
20pub fn adopt_legacy_scratch(
23 scratch: &Path,
24 job_dir: PathBuf,
25 source_path: &Path,
26 output: &Path,
27 mode: IngestMode,
28 segment_mib: Option<u64>,
29) -> io::Result<IngestJob> {
30 let run_src = find_legacy_run_root(scratch)?;
31 let mut job = IngestJob::create(
32 job_dir,
33 IngestSourceKind::File {
34 path: source_path.to_string_lossy().into_owned(),
35 },
36 IngestEncoding::Identity,
37 super::source::infer_rdf_format(&source_path.to_string_lossy()),
38 mode,
39 segment_mib,
40 output,
41 )?;
42 let dest = job.runs_dir();
43 let mut chunks = 0u64;
44 let mut lex = 0u64;
45 let mut triples = 0u64;
46 for ent in fs::read_dir(&run_src)? {
47 let ent = ent?;
48 let name = ent.file_name();
49 let name_s = name.to_string_lossy();
50 if !(name_s.starts_with("chunk_") || name_s.starts_with("lexrun_")) {
51 continue;
52 }
53 if name_s.starts_with("chunk_") {
54 let len = ent.metadata()?.len();
55 if len % 48 != 0 {
56 continue;
58 }
59 chunks += 1;
60 triples += len / 48;
61 } else {
62 lex += 1;
63 }
64 let dest_path = dest.join(&name);
66 if let Err(err) = fs::rename(ent.path(), &dest_path) {
67 fs::copy(ent.path(), &dest_path)?;
68 let _ = err;
69 }
70 }
71 job.record_progress(triples, 0, chunks, lex)?;
72 Ok(job)
73}
74
75pub fn publish_job(
78 job_dir: &Path,
79 report: crate::query::ingest_report::IngestReport,
80) -> io::Result<u64> {
81 let job = IngestJob::open(job_dir.to_path_buf())?;
82 let mut sorter = crate::external_sort::ExternalSorter::adopt_existing(job.runs_dir())?;
83 sorter.set_note_sink(report.note_sink());
84 let cap = job
85 .spec
86 .segment_mib
87 .unwrap_or(512)
88 .saturating_mul(1024 * 1024);
89 if cap == 0 {
90 return Err(io::Error::new(
91 io::ErrorKind::InvalidInput,
92 "segment_mib must be greater than zero",
93 ));
94 }
95 report.emit(
96 crate::query::ingest_report::IngestPhase::Publishing,
97 format!(
98 "publishing {} quin run(s) + {} lex run(s) → {}",
99 sorter.quin_run_count(),
100 sorter.lex_run_count(),
101 job.spec.output
102 ),
103 None,
104 );
105 let stats = sorter.merge_volume_set(Path::new(&job.spec.output), cap)?;
106 Ok(stats.blocks_written)
107}
108
109pub fn continue_job(job: &IngestJob) -> io::Result<u64> {
111 crate::query::ingest::streaming_import_rdf_with_job(
112 &job.dir,
113 crate::query::ingest_report::IngestReport::silent(),
114 )
115}
116
117fn find_legacy_run_root(scratch: &Path) -> io::Result<PathBuf> {
118 if scratch.join(JOB_RUNS).is_dir() {
119 return Ok(scratch.join(JOB_RUNS));
120 }
121 let mut best = scratch.to_path_buf();
122 let mut best_n = 0u64;
123 if let Ok(rd) = fs::read_dir(scratch) {
124 for ent in rd.flatten() {
125 let p = ent.path();
126 if !p.is_dir() {
127 continue;
128 }
129 let n = fs::read_dir(&p)
130 .map(|rd| {
131 rd.flatten()
132 .filter(|e| e.file_name().to_string_lossy().starts_with("chunk_"))
133 .count() as u64
134 })
135 .unwrap_or(0);
136 if n > best_n {
137 best_n = n;
138 best = p;
139 }
140 }
141 }
142 if best_n == 0
143 && fs::read_dir(scratch)
144 .map(|rd| {
145 rd.flatten()
146 .any(|e| e.file_name().to_string_lossy().starts_with("chunk_"))
147 })
148 .unwrap_or(false)
149 {
150 return Ok(scratch.to_path_buf());
151 }
152 if best_n == 0 {
153 return Err(io::Error::new(
154 io::ErrorKind::NotFound,
155 format!("no chunk_*.tmp under {}", scratch.display()),
156 ));
157 }
158 Ok(best)
159}
160
161#[cfg(test)]
162mod tests {
163 use super::*;
164 use crate::NQuin;
165
166 #[test]
167 fn adopt_moves_chunks_on_same_volume() {
168 let scratch = tempfile::TempDir::new().unwrap();
169 let inner = scratch.path().join("run");
170 fs::create_dir_all(&inner).unwrap();
171 let q = NQuin {
172 subject: 1,
173 predicate: 2,
174 object: 3,
175 context: 0,
176 metadata: 0,
177 parity: 1 ^ 2 ^ 3,
178 };
179 let bytes = bytemuck::bytes_of(&q);
180 fs::write(inner.join("chunk_0.tmp"), bytes).unwrap();
181 fs::write(inner.join("lexrun_0.tmp"), b"x").unwrap();
182 let job_dir = scratch.path().join("job");
183 let src = scratch.path().join("src.ttl");
184 fs::write(&src, b"").unwrap();
185 let out = scratch.path().join("out.q42");
186 let job = adopt_legacy_scratch(
187 scratch.path(),
188 job_dir,
189 &src,
190 &out,
191 IngestMode::Complete,
192 Some(512),
193 )
194 .unwrap();
195 assert_eq!(job.checkpoint.triples, 1);
196 assert_eq!(job.checkpoint.quin_chunks, 1);
197 assert!(!inner.join("chunk_0.tmp").exists());
198 assert!(job.runs_dir().join("chunk_0.tmp").exists());
199 }
200}