Skip to main content

qualia_core_db/query/ingest_job/
resume.rs

1//! Adopt existing quin/lex runs and decide whether a job can continue.
2
3use std::fs;
4use std::io;
5use std::path::{Path, PathBuf};
6
7use super::job::{IngestJob, JOB_RUNS};
8use super::source::IngestEncoding;
9use super::source::IngestSourceKind;
10use crate::query::ingest::IngestMode;
11
12/// A job can continue when it is not complete and we still have the source
13/// locator. Turtle/HTTP resume re-reads from the start and skips already
14/// accepted triples (correct; prefixes stay valid). Gzip/zstd HTTP costs
15/// another download. There is no mid-stream inflater checkpoint in v1.
16pub fn resume_is_supported(job: &IngestJob) -> bool {
17    !job.is_complete()
18}
19
20/// Copy a live/legacy scratch tree into a new job directory so it can be inspected
21/// and, if the source file is still there and uncompressed, continued.
22pub fn adopt_legacy_scratch(
23    scratch: &Path,
24    job_dir: PathBuf,
25    source_path: &Path,
26    output: &Path,
27    mode: IngestMode,
28    segment_mib: Option<u64>,
29) -> io::Result<IngestJob> {
30    let run_src = find_legacy_run_root(scratch)?;
31    let mut job = IngestJob::create(
32        job_dir,
33        IngestSourceKind::File {
34            path: source_path.to_string_lossy().into_owned(),
35        },
36        IngestEncoding::Identity,
37        super::source::infer_rdf_format(&source_path.to_string_lossy()),
38        mode,
39        segment_mib,
40        output,
41    )?;
42    let dest = job.runs_dir();
43    let mut chunks = 0u64;
44    let mut lex = 0u64;
45    let mut triples = 0u64;
46    for ent in fs::read_dir(&run_src)? {
47        let ent = ent?;
48        let name = ent.file_name();
49        let name_s = name.to_string_lossy();
50        if !(name_s.starts_with("chunk_") || name_s.starts_with("lexrun_")) {
51            continue;
52        }
53        if name_s.starts_with("chunk_") {
54            let len = ent.metadata()?.len();
55            if len % 48 != 0 {
56                // Torn last write — skip; resume will re-parse that tail.
57                continue;
58            }
59            chunks += 1;
60            triples += len / 48;
61        } else {
62            lex += 1;
63        }
64        // Same-volume rename: OSM-scale scratch is 100+ GiB; copy would not fit.
65        let dest_path = dest.join(&name);
66        if let Err(err) = fs::rename(ent.path(), &dest_path) {
67            fs::copy(ent.path(), &dest_path)?;
68            let _ = err;
69        }
70    }
71    job.record_progress(triples, 0, chunks, lex)?;
72    Ok(job)
73}
74
75/// Publish already-hashed quin/lex runs to the job's `.q42` root. Does **not**
76/// re-read the Turtle (the OSM continue died on disk during this step).
77pub fn publish_job(
78    job_dir: &Path,
79    report: crate::query::ingest_report::IngestReport,
80) -> io::Result<u64> {
81    let job = IngestJob::open(job_dir.to_path_buf())?;
82    let mut sorter = crate::external_sort::ExternalSorter::adopt_existing(job.runs_dir())?;
83    sorter.set_note_sink(report.note_sink());
84    let cap = job
85        .spec
86        .segment_mib
87        .unwrap_or(512)
88        .saturating_mul(1024 * 1024);
89    if cap == 0 {
90        return Err(io::Error::new(
91            io::ErrorKind::InvalidInput,
92            "segment_mib must be greater than zero",
93        ));
94    }
95    report.emit(
96        crate::query::ingest_report::IngestPhase::Publishing,
97        format!(
98            "publishing {} quin run(s) + {} lex run(s) → {}",
99            sorter.quin_run_count(),
100            sorter.lex_run_count(),
101            job.spec.output
102        ),
103        None,
104    );
105    let stats = sorter.merge_volume_set(Path::new(&job.spec.output), cap)?;
106    Ok(stats.blocks_written)
107}
108
109/// Continue is implemented by `streaming_import_rdf_with_job` (adopt + skip-N).
110pub fn continue_job(job: &IngestJob) -> io::Result<u64> {
111    crate::query::ingest::streaming_import_rdf_with_job(
112        &job.dir,
113        crate::query::ingest_report::IngestReport::silent(),
114    )
115}
116
117fn find_legacy_run_root(scratch: &Path) -> io::Result<PathBuf> {
118    if scratch.join(JOB_RUNS).is_dir() {
119        return Ok(scratch.join(JOB_RUNS));
120    }
121    let mut best = scratch.to_path_buf();
122    let mut best_n = 0u64;
123    if let Ok(rd) = fs::read_dir(scratch) {
124        for ent in rd.flatten() {
125            let p = ent.path();
126            if !p.is_dir() {
127                continue;
128            }
129            let n = fs::read_dir(&p)
130                .map(|rd| {
131                    rd.flatten()
132                        .filter(|e| e.file_name().to_string_lossy().starts_with("chunk_"))
133                        .count() as u64
134                })
135                .unwrap_or(0);
136            if n > best_n {
137                best_n = n;
138                best = p;
139            }
140        }
141    }
142    if best_n == 0
143        && fs::read_dir(scratch)
144            .map(|rd| {
145                rd.flatten()
146                    .any(|e| e.file_name().to_string_lossy().starts_with("chunk_"))
147            })
148            .unwrap_or(false)
149    {
150        return Ok(scratch.to_path_buf());
151    }
152    if best_n == 0 {
153        return Err(io::Error::new(
154            io::ErrorKind::NotFound,
155            format!("no chunk_*.tmp under {}", scratch.display()),
156        ));
157    }
158    Ok(best)
159}
160
161#[cfg(test)]
162mod tests {
163    use super::*;
164    use crate::NQuin;
165
166    #[test]
167    fn adopt_moves_chunks_on_same_volume() {
168        let scratch = tempfile::TempDir::new().unwrap();
169        let inner = scratch.path().join("run");
170        fs::create_dir_all(&inner).unwrap();
171        let q = NQuin {
172            subject: 1,
173            predicate: 2,
174            object: 3,
175            context: 0,
176            metadata: 0,
177            parity: 1 ^ 2 ^ 3,
178        };
179        let bytes = bytemuck::bytes_of(&q);
180        fs::write(inner.join("chunk_0.tmp"), bytes).unwrap();
181        fs::write(inner.join("lexrun_0.tmp"), b"x").unwrap();
182        let job_dir = scratch.path().join("job");
183        let src = scratch.path().join("src.ttl");
184        fs::write(&src, b"").unwrap();
185        let out = scratch.path().join("out.q42");
186        let job = adopt_legacy_scratch(
187            scratch.path(),
188            job_dir,
189            &src,
190            &out,
191            IngestMode::Complete,
192            Some(512),
193        )
194        .unwrap();
195        assert_eq!(job.checkpoint.triples, 1);
196        assert_eq!(job.checkpoint.quin_chunks, 1);
197        assert!(!inner.join("chunk_0.tmp").exists());
198        assert!(job.runs_dir().join("chunk_0.tmp").exists());
199    }
200}