Skip to main content

qualia_core_db/q42/asset_import/
job.rs

1//! Bounded import job: unique TempDir, streaming chunks, promote-on-success.
2
3use super::budgets::ImportBudgets;
4use super::error::ImportError;
5use super::status::{ChunkOutcome, FeedProgress, ImportStatus, PromotedArtifact};
6use crate::q42::asset_envelope::sha256_of;
7use std::fs::{self, File};
8use std::io::{self, Read, Write};
9use std::path::{Path, PathBuf};
10use tempfile::TempDir;
11
12const RAW_NAME: &str = "raw.bin";
13const OUT_NAME: &str = "accepted.bin";
14const PROMOTE_NAME: &str = "promoted.bin";
15
16/// Cold-construction import job with RAII scratch cleanup.
17///
18/// The caller's `raw_path` is never opened for write. A copy (or same-volume
19/// hardlink) lives under a unique [`TempDir`]; only that directory is cleaned
20/// on drop / unwind — no broad temp sweeps.
21pub struct ImportJob {
22    scratch: TempDir,
23    raw_scratch: PathBuf,
24    out_scratch: PathBuf,
25    reader: File,
26    budgets: ImportBudgets,
27    cancelled: bool,
28    bytes_read: u64,
29    accepted: u64,
30    quarantined: u64,
31    output_bytes: u64,
32    status: ImportStatus,
33    /// Scratch root path retained for cleanup tests after `into_parts` / drop.
34    scratch_path: PathBuf,
35}
36
37impl ImportJob {
38    /// Begin a job: validate budgets, create unique TempDir, stage immutable raw.
39    pub fn begin(raw_path: &Path, budgets: ImportBudgets) -> Result<Self, ImportError> {
40        let budgets = budgets.validate()?;
41        if !raw_path.is_file() {
42            return Err(ImportError::RawNotFound(raw_path.to_path_buf()));
43        }
44
45        let scratch = TempDir::new().map_err(ImportError::Io)?;
46        let scratch_path = scratch.path().to_path_buf();
47        let raw_scratch = scratch_path.join(RAW_NAME);
48        let out_scratch = scratch_path.join(OUT_NAME);
49
50        stage_immutable_raw(raw_path, &raw_scratch)?;
51        File::create(&out_scratch).map_err(ImportError::Io)?;
52
53        let reader = File::open(&raw_scratch).map_err(ImportError::Io)?;
54
55        Ok(Self {
56            scratch,
57            raw_scratch,
58            out_scratch,
59            reader,
60            budgets,
61            cancelled: false,
62            bytes_read: 0,
63            accepted: 0,
64            quarantined: 0,
65            output_bytes: 0,
66            status: ImportStatus::Running,
67            scratch_path,
68        })
69    }
70
71    /// Absolute path of this job's TempDir (for tests / diagnostics only).
72    pub fn scratch_path(&self) -> &Path {
73        &self.scratch_path
74    }
75
76    /// Path of the immutable staged raw copy inside scratch.
77    pub fn raw_scratch_path(&self) -> &Path {
78        &self.raw_scratch
79    }
80
81    /// `ImportJob::status`.
82    pub fn status(&self) -> &ImportStatus {
83        &self.status
84    }
85
86    /// Accepted assets.
87    pub fn accepted(&self) -> u64 {
88        self.accepted
89    }
90
91    /// Quarantined assets.
92    pub fn quarantined(&self) -> u64 {
93        self.quarantined
94    }
95
96    /// Import budgets.
97    pub fn budgets(&self) -> ImportBudgets {
98        self.budgets
99    }
100
101    /// Request cancellation; subsequent `feed_chunk` / `promote` fail closed.
102    pub fn cancel(&mut self) {
103        self.cancelled = true;
104        if matches!(self.status, ImportStatus::Running) {
105            self.status = ImportStatus::Cancelled;
106        }
107    }
108
109    /// Stream the next chunk under `chunk_byte_budget`, invoke `f`, update counts.
110    ///
111    /// `f` receives the chunk bytes and a writer for accepted output bytes.
112    /// On EOF after a successful callback the job transitions to [`ImportStatus::Succeeded`].
113    pub fn feed_chunk<F>(&mut self, f: F) -> Result<FeedProgress, ImportError>
114    where
115        F: FnOnce(&[u8], &mut dyn Write) -> Result<ChunkOutcome, ImportError>,
116    {
117        if self.cancelled || matches!(self.status, ImportStatus::Cancelled) {
118            self.status = ImportStatus::Cancelled;
119            return Err(ImportError::Cancelled);
120        }
121        if let ImportStatus::Failed(msg) = &self.status {
122            return Err(ImportError::Failed(msg.clone()));
123        }
124        if self.status.is_succeeded() {
125            return Ok(FeedProgress {
126                bytes_this_chunk: 0,
127                bytes_read_total: self.bytes_read,
128                eof: true,
129            });
130        }
131
132        let remaining_bytes = self.budgets.max_bytes.saturating_sub(self.bytes_read);
133        if remaining_bytes == 0 {
134            return self.fail_budget_bytes();
135        }
136
137        let to_read = (self.budgets.chunk_byte_budget as usize).min(remaining_bytes as usize);
138
139        let mut buf = vec![0u8; to_read];
140        let n = self.reader.read(&mut buf).map_err(|e| {
141            self.status = ImportStatus::Failed(e.to_string());
142            ImportError::Io(e)
143        })?;
144        buf.truncate(n);
145
146        if n == 0 {
147            self.status = ImportStatus::Succeeded {
148                bytes_read: self.bytes_read,
149                accepted: self.accepted,
150                quarantined: self.quarantined,
151                output_bytes: self.output_bytes,
152            };
153            return Ok(FeedProgress {
154                bytes_this_chunk: 0,
155                bytes_read_total: self.bytes_read,
156                eof: true,
157            });
158        }
159
160        // Hard fail if this chunk alone would exceed max_bytes (partial last chunk OK).
161        let next_total = self.bytes_read.saturating_add(n as u64);
162        if next_total > self.budgets.max_bytes {
163            return self.fail_budget_bytes();
164        }
165
166        let mut out = File::options()
167            .append(true)
168            .open(&self.out_scratch)
169            .map_err(|e| {
170                self.status = ImportStatus::Failed(e.to_string());
171                ImportError::Io(e)
172            })?;
173
174        let before_len = self.output_bytes;
175        let outcome = match f(&buf, &mut out) {
176            Ok(o) => o,
177            Err(e) => {
178                self.status = ImportStatus::Failed(e.to_string());
179                return Err(e);
180            }
181        };
182        out.flush().map_err(|e| {
183            self.status = ImportStatus::Failed(e.to_string());
184            ImportError::Io(e)
185        })?;
186        drop(out);
187
188        let written = fs::metadata(&self.out_scratch)
189            .map(|m| m.len())
190            .unwrap_or(before_len);
191        self.output_bytes = written;
192
193        let added = outcome.accepted.saturating_add(outcome.quarantined);
194        let next_records = self
195            .accepted
196            .saturating_add(self.quarantined)
197            .saturating_add(added);
198        if next_records > self.budgets.max_records {
199            self.status = ImportStatus::Failed("record budget exceeded".into());
200            return Err(ImportError::RecordBudgetExceeded {
201                used: next_records,
202                max: self.budgets.max_records,
203            });
204        }
205
206        self.bytes_read = next_total;
207        self.accepted = self.accepted.saturating_add(outcome.accepted);
208        self.quarantined = self.quarantined.saturating_add(outcome.quarantined);
209
210        // Peek one byte to detect EOF without consuming a future chunk budget incorrectly.
211        let mut peek = [0u8; 1];
212        let eof = match self.reader.read(&mut peek) {
213            Ok(0) => true,
214            Ok(1) => {
215                // Put the byte back by seeking -1; File is Seek for regular files.
216                use std::io::Seek;
217                self.reader.seek(io::SeekFrom::Current(-1)).map_err(|e| {
218                    self.status = ImportStatus::Failed(e.to_string());
219                    ImportError::Io(e)
220                })?;
221                false
222            }
223            Ok(_) => false,
224            Err(e) => {
225                self.status = ImportStatus::Failed(e.to_string());
226                return Err(ImportError::Io(e));
227            }
228        };
229
230        // If we hit max_bytes exactly mid-stream without EOF, fail closed on next feed;
231        // if at max_bytes and more data remains, that is a budget violation.
232        if !eof && self.bytes_read >= self.budgets.max_bytes {
233            return self.fail_budget_bytes();
234        }
235
236        if eof {
237            self.status = ImportStatus::Succeeded {
238                bytes_read: self.bytes_read,
239                accepted: self.accepted,
240                quarantined: self.quarantined,
241                output_bytes: self.output_bytes,
242            };
243        }
244
245        Ok(FeedProgress {
246            bytes_this_chunk: n as u64,
247            bytes_read_total: self.bytes_read,
248            eof,
249        })
250    }
251
252    /// Promote staged accepted output into `dest_dir` (only when status is Succeeded).
253    ///
254    /// Consumes `self` so the TempDir drops after the promote copy completes.
255    pub fn promote(self, dest_dir: &Path) -> Result<PromotedArtifact, ImportError> {
256        if self.cancelled || matches!(self.status, ImportStatus::Cancelled) {
257            return Err(ImportError::Cancelled);
258        }
259        let (_bytes_read, accepted, quarantined, output_bytes) = self.status.require_succeeded()?;
260
261        fs::create_dir_all(dest_dir).map_err(ImportError::Io)?;
262        let dest = dest_dir.join(PROMOTE_NAME);
263        if dest.exists() {
264            return Err(ImportError::DestinationExists(dest));
265        }
266
267        fs::copy(&self.out_scratch, &dest).map_err(ImportError::Io)?;
268        let bytes = fs::metadata(&dest).map(|m| m.len()).unwrap_or(output_bytes);
269        let data = fs::read(&dest).map_err(ImportError::Io)?;
270        let digest = sha256_of(&data);
271
272        // Drop scratch via self.scratch Drop at end of scope.
273        let _scratch = self.scratch;
274
275        Ok(PromotedArtifact {
276            path: dest,
277            sha256: digest,
278            bytes,
279            accepted,
280            quarantined,
281        })
282    }
283
284    fn fail_budget_bytes(&mut self) -> Result<FeedProgress, ImportError> {
285        self.status = ImportStatus::Failed("byte budget exceeded".into());
286        Err(ImportError::ByteBudgetExceeded {
287            used: self.bytes_read,
288            max: self.budgets.max_bytes,
289        })
290    }
291}
292
293/// Copy or hardlink `src` into `dst` without mutating `src`.
294fn stage_immutable_raw(src: &Path, dst: &Path) -> Result<(), ImportError> {
295    // Prefer hardlink when same volume (zero-copy stage); fall back to full copy.
296    match fs::hard_link(src, dst) {
297        Ok(()) => Ok(()),
298        Err(_) => {
299            fs::copy(src, dst).map_err(ImportError::Io)?;
300            Ok(())
301        }
302    }
303}