Skip to main content

qualia_core_db/q42/asset_import/
mod.rs

1//! Bounded Q42 asset import jobs (AST-02).
2//!
3//! Cold-construction framework: unique [`tempfile::TempDir`], explicit byte /
4//! record / chunk budgets (chunk pass ≤ `SENTINEL_PASS_BUDGET_BYTES`), streaming
5//! `Read`, cancellation, quarantine counts, and promote-on-success. Caller
6//! supplies a local `Path` — no network downloader. The raw caller path is
7//! never mutated; only the job TempDir is cleaned via RAII.
8
9mod budgets;
10mod error;
11mod job;
12mod status;
13
14pub use budgets::ImportBudgets;
15pub use error::ImportError;
16pub use job::ImportJob;
17pub use status::{ChunkOutcome, FeedProgress, ImportStatus, PromotedArtifact};
18
19pub use crate::q42::asset_envelope::SENTINEL_PASS_BUDGET_BYTES;
20
21#[cfg(test)]
22mod tests {
23    use super::*;
24    use crate::q42::asset_envelope::sha256_of;
25    use std::fs;
26    use std::path::PathBuf;
27
28    fn write_raw(dir: &tempfile::TempDir, name: &str, bytes: &[u8]) -> PathBuf {
29        let path = dir.path().join(name);
30        fs::write(&path, bytes).unwrap();
31        path
32    }
33
34    fn small_budgets(max_bytes: u64, max_records: u64, chunk: u64) -> ImportBudgets {
35        ImportBudgets {
36            max_bytes,
37            max_records,
38            chunk_byte_budget: chunk,
39        }
40    }
41
42    fn feed_passthrough(job: &mut ImportJob) -> Result<FeedProgress, ImportError> {
43        job.feed_chunk(|chunk, out| {
44            out.write_all(chunk).map_err(ImportError::Io)?;
45            Ok(ChunkOutcome {
46                accepted: 1,
47                quarantined: 0,
48            })
49        })
50    }
51
52    fn drain_to_success(job: &mut ImportJob) -> Result<(), ImportError> {
53        loop {
54            let progress = feed_passthrough(job)?;
55            if progress.eof {
56                return Ok(());
57            }
58        }
59    }
60
61    #[test]
62    fn chunk_over_sentinel_rejected_at_begin() {
63        let dir = tempfile::TempDir::new().unwrap();
64        let raw = write_raw(&dir, "a.bin", b"abc");
65        let budgets = ImportBudgets {
66            max_bytes: SENTINEL_PASS_BUDGET_BYTES,
67            max_records: 10,
68            chunk_byte_budget: SENTINEL_PASS_BUDGET_BYTES + 1,
69        };
70        assert_eq!(
71            ImportJob::begin(&raw, budgets).err(),
72            Some(ImportError::ChunkBudgetExceeded)
73        );
74    }
75
76    #[test]
77    fn budget_exceeded_when_raw_larger_than_max_bytes() {
78        let dir = tempfile::TempDir::new().unwrap();
79        let payload = b"0123456789ABCDEF"; // 16 bytes
80        let raw = write_raw(&dir, "big.bin", payload);
81        let mut job = ImportJob::begin(&raw, small_budgets(8, 100, 4)).unwrap();
82        // First chunk: 4 bytes OK
83        let p1 = feed_passthrough(&mut job).unwrap();
84        assert_eq!(p1.bytes_this_chunk, 4);
85        assert!(!p1.eof);
86        // Second chunk: 4 bytes → at max_bytes with more remaining → fail
87        let err = feed_passthrough(&mut job).unwrap_err();
88        assert_eq!(err, ImportError::ByteBudgetExceeded { used: 8, max: 8 });
89        assert!(matches!(job.status(), ImportStatus::Failed(_)));
90    }
91
92    #[test]
93    fn cleanup_on_success() {
94        let dir = tempfile::TempDir::new().unwrap();
95        let payload = b"promote-me";
96        let raw = write_raw(&dir, "in.bin", payload);
97        let dest = dir.path().join("out");
98        let scratch_path = {
99            let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 32)).unwrap();
100            let scratch = job.scratch_path().to_path_buf();
101            assert!(scratch.exists());
102            drain_to_success(&mut job).unwrap();
103            assert!(job.status().is_succeeded());
104            let promoted = job.promote(&dest).unwrap();
105            assert_eq!(promoted.bytes, payload.len() as u64);
106            assert_eq!(promoted.sha256, sha256_of(payload));
107            assert!(promoted.path.exists());
108            scratch
109        };
110        assert!(
111            !scratch_path.exists(),
112            "TempDir must be gone after successful promote"
113        );
114        // Caller raw path unchanged and still present.
115        assert_eq!(fs::read(&raw).unwrap(), payload);
116    }
117
118    #[test]
119    fn cleanup_on_error() {
120        let dir = tempfile::TempDir::new().unwrap();
121        let raw = write_raw(&dir, "in.bin", b"data");
122        let scratch_path = {
123            let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 32)).unwrap();
124            let scratch = job.scratch_path().to_path_buf();
125            let err = job.feed_chunk(|_, _| Err(ImportError::Failed("parse".into())));
126            assert_eq!(err, Err(ImportError::Failed("parse".into())));
127            assert!(scratch.exists(), "scratch lives while job is held");
128            scratch
129        };
130        assert!(
131            !scratch_path.exists(),
132            "TempDir RAII cleanup on drop after error"
133        );
134    }
135
136    #[test]
137    fn cleanup_on_unwind() {
138        let dir = tempfile::TempDir::new().unwrap();
139        let raw = write_raw(&dir, "in.bin", b"unwind-payload");
140        let scratch_gone = {
141            let job = ImportJob::begin(&raw, small_budgets(64, 10, 32)).unwrap();
142            let scratch = job.scratch_path().to_path_buf();
143            let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
144                let mut job = job;
145                let _ = job.feed_chunk(|_, _| panic!("forced unwind"));
146            }));
147            assert!(result.is_err());
148            scratch
149        };
150        assert!(!scratch_gone.exists(), "TempDir must clean up on unwind");
151    }
152
153    #[test]
154    fn cancellation_blocks_feed_and_promote() {
155        let dir = tempfile::TempDir::new().unwrap();
156        let raw = write_raw(&dir, "in.bin", b"0123456789");
157        let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 4)).unwrap();
158        let _ = feed_passthrough(&mut job).unwrap();
159        job.cancel();
160        assert_eq!(job.status(), &ImportStatus::Cancelled);
161        assert_eq!(
162            feed_passthrough(&mut job).err(),
163            Some(ImportError::Cancelled)
164        );
165        let dest = dir.path().join("cancelled-out");
166        assert_eq!(job.promote(&dest).err(), Some(ImportError::Cancelled));
167    }
168
169    #[test]
170    fn partial_input_leaves_running_and_cleans_on_drop() {
171        let dir = tempfile::TempDir::new().unwrap();
172        let raw = write_raw(&dir, "in.bin", b"abcdefghij"); // 10 bytes
173        let dest = dir.path().join("partial-out");
174
175        // Promote with no successful feed must fail closed.
176        let early = ImportJob::begin(&raw, small_budgets(64, 10, 4)).unwrap();
177        assert_eq!(early.promote(&dest).err(), Some(ImportError::NotSucceeded));
178
179        let scratch_path = {
180            let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 4)).unwrap();
181            let scratch = job.scratch_path().to_path_buf();
182            let p = feed_passthrough(&mut job).unwrap();
183            assert_eq!(p.bytes_this_chunk, 4);
184            assert!(!p.eof);
185            assert_eq!(job.status(), &ImportStatus::Running);
186            // Drop without promote — RAII cleans scratch.
187            scratch
188        };
189        assert!(!scratch_path.exists());
190    }
191
192    #[test]
193    fn deterministic_output_hash_of_promoted_bytes() {
194        let dir = tempfile::TempDir::new().unwrap();
195        let payload = b"hash-stable-fixture-v1";
196        let raw = write_raw(&dir, "in.bin", payload);
197        let dest_a = dir.path().join("a");
198        let dest_b = dir.path().join("b");
199
200        let hash_a = {
201            let mut job = ImportJob::begin(&raw, ImportBudgets::sentinel_pass()).unwrap();
202            drain_to_success(&mut job).unwrap();
203            job.promote(&dest_a).unwrap().sha256
204        };
205        let hash_b = {
206            let mut job = ImportJob::begin(&raw, ImportBudgets::sentinel_pass()).unwrap();
207            drain_to_success(&mut job).unwrap();
208            job.promote(&dest_b).unwrap().sha256
209        };
210        assert_eq!(hash_a, hash_b);
211        assert_eq!(hash_a, sha256_of(payload));
212    }
213
214    #[test]
215    fn raw_caller_path_not_mutated() {
216        let dir = tempfile::TempDir::new().unwrap();
217        let payload = b"immutable-raw";
218        let raw = write_raw(&dir, "caller.bin", payload);
219        let before = fs::metadata(&raw).unwrap().len();
220        let mut job = ImportJob::begin(&raw, small_budgets(64, 10, 32)).unwrap();
221        // Scratch raw must differ in path from caller.
222        assert_ne!(job.raw_scratch_path(), raw.as_path());
223        drain_to_success(&mut job).unwrap();
224        let _ = job.promote(&dir.path().join("promoted-dir")).unwrap();
225        assert_eq!(fs::read(&raw).unwrap(), payload);
226        assert_eq!(fs::metadata(&raw).unwrap().len(), before);
227    }
228
229    #[test]
230    fn quarantine_counts_accumulate() {
231        let dir = tempfile::TempDir::new().unwrap();
232        let raw = write_raw(&dir, "in.bin", b"ABCDEFGH");
233        let mut job = ImportJob::begin(&raw, small_budgets(64, 100, 4)).unwrap();
234        job.feed_chunk(|chunk, out| {
235            out.write_all(&chunk[..2]).map_err(ImportError::Io)?;
236            Ok(ChunkOutcome {
237                accepted: 2,
238                quarantined: 2,
239            })
240        })
241        .unwrap();
242        assert_eq!(job.accepted(), 2);
243        assert_eq!(job.quarantined(), 2);
244    }
245
246    #[test]
247    fn record_budget_exceeded() {
248        let dir = tempfile::TempDir::new().unwrap();
249        let raw = write_raw(&dir, "in.bin", b"ABCDEFGH");
250        let mut job = ImportJob::begin(&raw, small_budgets(64, 3, 8)).unwrap();
251        let err = job
252            .feed_chunk(|chunk, out| {
253                out.write_all(chunk).map_err(ImportError::Io)?;
254                Ok(ChunkOutcome {
255                    accepted: 2,
256                    quarantined: 2,
257                })
258            })
259            .unwrap_err();
260        assert_eq!(err, ImportError::RecordBudgetExceeded { used: 4, max: 3 });
261    }
262}