qualia_core_db/q42/asset_import/
job.rs1use super::budgets::ImportBudgets;
4use super::error::ImportError;
5use super::status::{ChunkOutcome, FeedProgress, ImportStatus, PromotedArtifact};
6use crate::q42::asset_envelope::sha256_of;
7use std::fs::{self, File};
8use std::io::{self, Read, Write};
9use std::path::{Path, PathBuf};
10use tempfile::TempDir;
11
12const RAW_NAME: &str = "raw.bin";
13const OUT_NAME: &str = "accepted.bin";
14const PROMOTE_NAME: &str = "promoted.bin";
15
16pub struct ImportJob {
22 scratch: TempDir,
23 raw_scratch: PathBuf,
24 out_scratch: PathBuf,
25 reader: File,
26 budgets: ImportBudgets,
27 cancelled: bool,
28 bytes_read: u64,
29 accepted: u64,
30 quarantined: u64,
31 output_bytes: u64,
32 status: ImportStatus,
33 scratch_path: PathBuf,
35}
36
37impl ImportJob {
38 pub fn begin(raw_path: &Path, budgets: ImportBudgets) -> Result<Self, ImportError> {
40 let budgets = budgets.validate()?;
41 if !raw_path.is_file() {
42 return Err(ImportError::RawNotFound(raw_path.to_path_buf()));
43 }
44
45 let scratch = TempDir::new().map_err(ImportError::Io)?;
46 let scratch_path = scratch.path().to_path_buf();
47 let raw_scratch = scratch_path.join(RAW_NAME);
48 let out_scratch = scratch_path.join(OUT_NAME);
49
50 stage_immutable_raw(raw_path, &raw_scratch)?;
51 File::create(&out_scratch).map_err(ImportError::Io)?;
52
53 let reader = File::open(&raw_scratch).map_err(ImportError::Io)?;
54
55 Ok(Self {
56 scratch,
57 raw_scratch,
58 out_scratch,
59 reader,
60 budgets,
61 cancelled: false,
62 bytes_read: 0,
63 accepted: 0,
64 quarantined: 0,
65 output_bytes: 0,
66 status: ImportStatus::Running,
67 scratch_path,
68 })
69 }
70
71 pub fn scratch_path(&self) -> &Path {
73 &self.scratch_path
74 }
75
76 pub fn raw_scratch_path(&self) -> &Path {
78 &self.raw_scratch
79 }
80
81 pub fn status(&self) -> &ImportStatus {
83 &self.status
84 }
85
86 pub fn accepted(&self) -> u64 {
88 self.accepted
89 }
90
91 pub fn quarantined(&self) -> u64 {
93 self.quarantined
94 }
95
96 pub fn budgets(&self) -> ImportBudgets {
98 self.budgets
99 }
100
101 pub fn cancel(&mut self) {
103 self.cancelled = true;
104 if matches!(self.status, ImportStatus::Running) {
105 self.status = ImportStatus::Cancelled;
106 }
107 }
108
109 pub fn feed_chunk<F>(&mut self, f: F) -> Result<FeedProgress, ImportError>
114 where
115 F: FnOnce(&[u8], &mut dyn Write) -> Result<ChunkOutcome, ImportError>,
116 {
117 if self.cancelled || matches!(self.status, ImportStatus::Cancelled) {
118 self.status = ImportStatus::Cancelled;
119 return Err(ImportError::Cancelled);
120 }
121 if let ImportStatus::Failed(msg) = &self.status {
122 return Err(ImportError::Failed(msg.clone()));
123 }
124 if self.status.is_succeeded() {
125 return Ok(FeedProgress {
126 bytes_this_chunk: 0,
127 bytes_read_total: self.bytes_read,
128 eof: true,
129 });
130 }
131
132 let remaining_bytes = self.budgets.max_bytes.saturating_sub(self.bytes_read);
133 if remaining_bytes == 0 {
134 return self.fail_budget_bytes();
135 }
136
137 let to_read = (self.budgets.chunk_byte_budget as usize).min(remaining_bytes as usize);
138
139 let mut buf = vec![0u8; to_read];
140 let n = self.reader.read(&mut buf).map_err(|e| {
141 self.status = ImportStatus::Failed(e.to_string());
142 ImportError::Io(e)
143 })?;
144 buf.truncate(n);
145
146 if n == 0 {
147 self.status = ImportStatus::Succeeded {
148 bytes_read: self.bytes_read,
149 accepted: self.accepted,
150 quarantined: self.quarantined,
151 output_bytes: self.output_bytes,
152 };
153 return Ok(FeedProgress {
154 bytes_this_chunk: 0,
155 bytes_read_total: self.bytes_read,
156 eof: true,
157 });
158 }
159
160 let next_total = self.bytes_read.saturating_add(n as u64);
162 if next_total > self.budgets.max_bytes {
163 return self.fail_budget_bytes();
164 }
165
166 let mut out = File::options()
167 .append(true)
168 .open(&self.out_scratch)
169 .map_err(|e| {
170 self.status = ImportStatus::Failed(e.to_string());
171 ImportError::Io(e)
172 })?;
173
174 let before_len = self.output_bytes;
175 let outcome = match f(&buf, &mut out) {
176 Ok(o) => o,
177 Err(e) => {
178 self.status = ImportStatus::Failed(e.to_string());
179 return Err(e);
180 }
181 };
182 out.flush().map_err(|e| {
183 self.status = ImportStatus::Failed(e.to_string());
184 ImportError::Io(e)
185 })?;
186 drop(out);
187
188 let written = fs::metadata(&self.out_scratch)
189 .map(|m| m.len())
190 .unwrap_or(before_len);
191 self.output_bytes = written;
192
193 let added = outcome.accepted.saturating_add(outcome.quarantined);
194 let next_records = self
195 .accepted
196 .saturating_add(self.quarantined)
197 .saturating_add(added);
198 if next_records > self.budgets.max_records {
199 self.status = ImportStatus::Failed("record budget exceeded".into());
200 return Err(ImportError::RecordBudgetExceeded {
201 used: next_records,
202 max: self.budgets.max_records,
203 });
204 }
205
206 self.bytes_read = next_total;
207 self.accepted = self.accepted.saturating_add(outcome.accepted);
208 self.quarantined = self.quarantined.saturating_add(outcome.quarantined);
209
210 let mut peek = [0u8; 1];
212 let eof = match self.reader.read(&mut peek) {
213 Ok(0) => true,
214 Ok(1) => {
215 use std::io::Seek;
217 self.reader.seek(io::SeekFrom::Current(-1)).map_err(|e| {
218 self.status = ImportStatus::Failed(e.to_string());
219 ImportError::Io(e)
220 })?;
221 false
222 }
223 Ok(_) => false,
224 Err(e) => {
225 self.status = ImportStatus::Failed(e.to_string());
226 return Err(ImportError::Io(e));
227 }
228 };
229
230 if !eof && self.bytes_read >= self.budgets.max_bytes {
233 return self.fail_budget_bytes();
234 }
235
236 if eof {
237 self.status = ImportStatus::Succeeded {
238 bytes_read: self.bytes_read,
239 accepted: self.accepted,
240 quarantined: self.quarantined,
241 output_bytes: self.output_bytes,
242 };
243 }
244
245 Ok(FeedProgress {
246 bytes_this_chunk: n as u64,
247 bytes_read_total: self.bytes_read,
248 eof,
249 })
250 }
251
252 pub fn promote(self, dest_dir: &Path) -> Result<PromotedArtifact, ImportError> {
256 if self.cancelled || matches!(self.status, ImportStatus::Cancelled) {
257 return Err(ImportError::Cancelled);
258 }
259 let (_bytes_read, accepted, quarantined, output_bytes) = self.status.require_succeeded()?;
260
261 fs::create_dir_all(dest_dir).map_err(ImportError::Io)?;
262 let dest = dest_dir.join(PROMOTE_NAME);
263 if dest.exists() {
264 return Err(ImportError::DestinationExists(dest));
265 }
266
267 fs::copy(&self.out_scratch, &dest).map_err(ImportError::Io)?;
268 let bytes = fs::metadata(&dest).map(|m| m.len()).unwrap_or(output_bytes);
269 let data = fs::read(&dest).map_err(ImportError::Io)?;
270 let digest = sha256_of(&data);
271
272 let _scratch = self.scratch;
274
275 Ok(PromotedArtifact {
276 path: dest,
277 sha256: digest,
278 bytes,
279 accepted,
280 quarantined,
281 })
282 }
283
284 fn fail_budget_bytes(&mut self) -> Result<FeedProgress, ImportError> {
285 self.status = ImportStatus::Failed("byte budget exceeded".into());
286 Err(ImportError::ByteBudgetExceeded {
287 used: self.bytes_read,
288 max: self.budgets.max_bytes,
289 })
290 }
291}
292
293fn stage_immutable_raw(src: &Path, dst: &Path) -> Result<(), ImportError> {
295 match fs::hard_link(src, dst) {
297 Ok(()) => Ok(()),
298 Err(_) => {
299 fs::copy(src, dst).map_err(ImportError::Io)?;
300 Ok(())
301 }
302 }
303}