qualia_core_db/q42/asset_import/
mod.rs1mod budgets;
10mod error;
11mod job;
12mod status;
13
14pub use budgets::ImportBudgets;
15pub use error::ImportError;
16pub use job::ImportJob;
17pub use status::{ChunkOutcome, FeedProgress, ImportStatus, PromotedArtifact};
18
19pub use crate::q42::asset_envelope::SENTINEL_PASS_BUDGET_BYTES;
20
21#[cfg(test)]
22mod tests {
23 use super::*;
24 use crate::q42::asset_envelope::sha256_of;
25 use std::fs;
26 use std::path::PathBuf;
27
28 fn write_raw(dir: &tempfile::TempDir, name: &str, bytes: &[u8]) -> PathBuf {
29 let path = dir.path().join(name);
30 fs::write(&path, bytes).unwrap();
31 path
32 }
33
34 fn small_budgets(max_bytes: u64, max_records: u64, chunk: u64) -> ImportBudgets {
35 ImportBudgets {
36 max_bytes,
37 max_records,
38 chunk_byte_budget: chunk,
39 }
40 }
41
42 fn feed_passthrough(job: &mut ImportJob) -> Result<FeedProgress, ImportError> {
43 job.feed_chunk(|chunk, out| {
44 out.write_all(chunk).map_err(ImportError::Io)?;
45 Ok(ChunkOutcome {
46 accepted: 1,
47 quarantined: 0,
48 })
49 })
50 }
51
52 fn drain_to_success(job: &mut ImportJob) -> Result<(), ImportError> {
53 loop {
54 let progress = feed_passthrough(job)?;
55 if progress.eof {
56 return Ok(());
57 }
58 }
59 }
60
61 #[test]
62 fn chunk_over_sentinel_rejected_at_begin() {
63 let dir = tempfile::TempDir::new().unwrap();
64 let raw = write_raw(&dir, "a.bin", b"abc");
65 let budgets = ImportBudgets {
66 max_bytes: SENTINEL_PASS_BUDGET_BYTES,
67 max_records: 10,
68 chunk_byte_budget: SENTINEL_PASS_BUDGET_BYTES + 1,
69 };
70 assert_eq!(
71 ImportJob::begin(&raw, budgets).err(),
72 Some(ImportError::ChunkBudgetExceeded)
73 );
74 }
75
76 #[test]
77 fn budget_exceeded_when_raw_larger_than_max_bytes() {
78 let dir = tempfile::TempDir::new().unwrap();
79 let payload = b"0123456789ABCDEF"; let raw = write_raw(&dir, "big.bin", payload);
81 let mut job = ImportJob::begin(&raw, small_budgets(8, 100, 4)).unwrap();
82 let p1 = feed_passthrough(&mut job).unwrap();
84 assert_eq!(p1.bytes_this_chunk, 4);
85 assert!(!p1.eof);
86 let err = feed_passthrough(&mut job).unwrap_err();
88 assert_eq!(err, ImportError::ByteBudgetExceeded { used: 8, max: 8 });
89 assert!(matches!(job.status(), ImportStatus::Failed(_)));
90 }
91
92 #[test]
93 fn cleanup_on_success() {
94 let dir = tempfile::TempDir::new().unwrap();
95 let payload = b"promote-me";
96 let raw = write_raw(&dir, "in.bin", payload);
97 let dest = dir.path().join("out");
98 let scratch_path = {
99 let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 32)).unwrap();
100 let scratch = job.scratch_path().to_path_buf();
101 assert!(scratch.exists());
102 drain_to_success(&mut job).unwrap();
103 assert!(job.status().is_succeeded());
104 let promoted = job.promote(&dest).unwrap();
105 assert_eq!(promoted.bytes, payload.len() as u64);
106 assert_eq!(promoted.sha256, sha256_of(payload));
107 assert!(promoted.path.exists());
108 scratch
109 };
110 assert!(
111 !scratch_path.exists(),
112 "TempDir must be gone after successful promote"
113 );
114 assert_eq!(fs::read(&raw).unwrap(), payload);
116 }
117
118 #[test]
119 fn cleanup_on_error() {
120 let dir = tempfile::TempDir::new().unwrap();
121 let raw = write_raw(&dir, "in.bin", b"data");
122 let scratch_path = {
123 let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 32)).unwrap();
124 let scratch = job.scratch_path().to_path_buf();
125 let err = job.feed_chunk(|_, _| Err(ImportError::Failed("parse".into())));
126 assert_eq!(err, Err(ImportError::Failed("parse".into())));
127 assert!(scratch.exists(), "scratch lives while job is held");
128 scratch
129 };
130 assert!(
131 !scratch_path.exists(),
132 "TempDir RAII cleanup on drop after error"
133 );
134 }
135
136 #[test]
137 fn cleanup_on_unwind() {
138 let dir = tempfile::TempDir::new().unwrap();
139 let raw = write_raw(&dir, "in.bin", b"unwind-payload");
140 let scratch_gone = {
141 let job = ImportJob::begin(&raw, small_budgets(64, 10, 32)).unwrap();
142 let scratch = job.scratch_path().to_path_buf();
143 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
144 let mut job = job;
145 let _ = job.feed_chunk(|_, _| panic!("forced unwind"));
146 }));
147 assert!(result.is_err());
148 scratch
149 };
150 assert!(!scratch_gone.exists(), "TempDir must clean up on unwind");
151 }
152
153 #[test]
154 fn cancellation_blocks_feed_and_promote() {
155 let dir = tempfile::TempDir::new().unwrap();
156 let raw = write_raw(&dir, "in.bin", b"0123456789");
157 let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 4)).unwrap();
158 let _ = feed_passthrough(&mut job).unwrap();
159 job.cancel();
160 assert_eq!(job.status(), &ImportStatus::Cancelled);
161 assert_eq!(
162 feed_passthrough(&mut job).err(),
163 Some(ImportError::Cancelled)
164 );
165 let dest = dir.path().join("cancelled-out");
166 assert_eq!(job.promote(&dest).err(), Some(ImportError::Cancelled));
167 }
168
169 #[test]
170 fn partial_input_leaves_running_and_cleans_on_drop() {
171 let dir = tempfile::TempDir::new().unwrap();
172 let raw = write_raw(&dir, "in.bin", b"abcdefghij"); let dest = dir.path().join("partial-out");
174
175 let early = ImportJob::begin(&raw, small_budgets(64, 10, 4)).unwrap();
177 assert_eq!(early.promote(&dest).err(), Some(ImportError::NotSucceeded));
178
179 let scratch_path = {
180 let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 4)).unwrap();
181 let scratch = job.scratch_path().to_path_buf();
182 let p = feed_passthrough(&mut job).unwrap();
183 assert_eq!(p.bytes_this_chunk, 4);
184 assert!(!p.eof);
185 assert_eq!(job.status(), &ImportStatus::Running);
186 scratch
188 };
189 assert!(!scratch_path.exists());
190 }
191
192 #[test]
193 fn deterministic_output_hash_of_promoted_bytes() {
194 let dir = tempfile::TempDir::new().unwrap();
195 let payload = b"hash-stable-fixture-v1";
196 let raw = write_raw(&dir, "in.bin", payload);
197 let dest_a = dir.path().join("a");
198 let dest_b = dir.path().join("b");
199
200 let hash_a = {
201 let mut job = ImportJob::begin(&raw, ImportBudgets::sentinel_pass()).unwrap();
202 drain_to_success(&mut job).unwrap();
203 job.promote(&dest_a).unwrap().sha256
204 };
205 let hash_b = {
206 let mut job = ImportJob::begin(&raw, ImportBudgets::sentinel_pass()).unwrap();
207 drain_to_success(&mut job).unwrap();
208 job.promote(&dest_b).unwrap().sha256
209 };
210 assert_eq!(hash_a, hash_b);
211 assert_eq!(hash_a, sha256_of(payload));
212 }
213
214 #[test]
215 fn raw_caller_path_not_mutated() {
216 let dir = tempfile::TempDir::new().unwrap();
217 let payload = b"immutable-raw";
218 let raw = write_raw(&dir, "caller.bin", payload);
219 let before = fs::metadata(&raw).unwrap().len();
220 let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 32)).unwrap();
221 assert_ne!(job.raw_scratch_path(), raw.as_path());
223 drain_to_success(&mut job).unwrap();
224 let _ = job.promote(&dir.path().join("promoted-dir")).unwrap();
225 assert_eq!(fs::read(&raw).unwrap(), payload);
226 assert_eq!(fs::metadata(&raw).unwrap().len(), before);
227 }
228
229 #[test]
230 fn quarantine_counts_accumulate() {
231 let dir = tempfile::TempDir::new().unwrap();
232 let raw = write_raw(&dir, "in.bin", b"ABCDEFGH");
233 let mut job = ImportJob::begin(&raw, small_budgets(64, 100, 4)).unwrap();
234 job.feed_chunk(|chunk, out| {
235 out.write_all(&chunk[..2]).map_err(ImportError::Io)?;
236 Ok(ChunkOutcome {
237 accepted: 2,
238 quarantined: 2,
239 })
240 })
241 .unwrap();
242 assert_eq!(job.accepted(), 2);
243 assert_eq!(job.quarantined(), 2);
244 }
245
246 #[test]
247 fn record_budget_exceeded() {
248 let dir = tempfile::TempDir::new().unwrap();
249 let raw = write_raw(&dir, "in.bin", b"ABCDEFGH");
250 let mut job = ImportJob::begin(&raw, small_budgets(64, 3, 8)).unwrap();
251 let err = job
252 .feed_chunk(|chunk, out| {
253 out.write_all(chunk).map_err(ImportError::Io)?;
254 Ok(ChunkOutcome {
255 accepted: 2,
256 quarantined: 2,
257 })
258 })
259 .unwrap_err();
260 assert_eq!(err, ImportError::RecordBudgetExceeded { used: 4, max: 3 });
261 }
262}