Skip to main content

qualia_core_db/q42/chebi_parse/
parse.rs

1//! High-level ChEBI `compounds.tsv` parse with budgets and cancel.
2
3use std::path::Path;
4use std::sync::atomic::{AtomicBool, Ordering};
5
6use super::error::ParseError;
7use super::record::{ParseBudgets, ParseReport};
8use super::tsv::{header_matches, parse_data_row, split_tsv_fields};
9
10/// Parse ChEBI FTP-style `compounds.tsv` bytes into accepted records and
11/// quarantined rows.
12///
13/// # Format
14///
15/// Tab-separated, header row required:
16/// `ID STATUS CHEBI_ACCESSION SOURCE PARENT_ID NAME DEFINITION`
17/// (tabs between columns). Optional trailing columns are ignored.
18///
19/// # Budgets
20///
21/// - Whole input larger than `budgets.max_bytes` → [`ParseError::ByteBudgetExceeded`].
22/// - Any line longer than `budgets.max_line_bytes` → [`ParseError::LineTooLong`].
23/// - Accepted + quarantined count exceeding `max_records` →
24///   [`ParseError::RecordBudgetExceeded`].
25///
26/// # Cancellation
27///
28/// When `cancel` is set, returns [`ParseError::Cancelled`] at the next
29/// non-empty data-line boundary (header is still validated first).
30pub fn parse_compounds_tsv(
31    input: &[u8],
32    budgets: ParseBudgets,
33    release_label: &str,
34    cancel: &AtomicBool,
35) -> Result<ParseReport, ParseError> {
36    let budgets = budgets.validate()?;
37    let size = input.len() as u64;
38    if size > budgets.max_bytes {
39        return Err(ParseError::ByteBudgetExceeded {
40            size,
41            max: budgets.max_bytes,
42        });
43    }
44
45    match std::str::from_utf8(input) {
46        Ok(text) => parse_compounds_text(text, size, budgets, release_label, cancel),
47        Err(_) => {
48            let owned = String::from_utf8_lossy(input).into_owned();
49            parse_compounds_text(&owned, size, budgets, release_label, cancel)
50        }
51    }
52}
53
54/// Convenience: read a caller-selected local path then parse (no network).
55pub fn parse_compounds_tsv_path(
56    path: &Path,
57    budgets: ParseBudgets,
58    release_label: &str,
59    cancel: &AtomicBool,
60) -> Result<ParseReport, ParseError> {
61    if cancel.load(Ordering::Relaxed) {
62        return Err(ParseError::Cancelled);
63    }
64    let budgets = budgets.validate()?;
65    let meta_len = std::fs::metadata(path)?.len();
66    if meta_len > budgets.max_bytes {
67        return Err(ParseError::ByteBudgetExceeded {
68            size: meta_len,
69            max: budgets.max_bytes,
70        });
71    }
72    let bytes = std::fs::read(path)?;
73    parse_compounds_tsv(&bytes, budgets, release_label, cancel)
74}
75
76fn parse_compounds_text(
77    text: &str,
78    bytes_read: u64,
79    budgets: ParseBudgets,
80    release_label: &str,
81    cancel: &AtomicBool,
82) -> Result<ParseReport, ParseError> {
83    let mut lines = text.split('\n').enumerate();
84    let Some((header_idx, header_raw)) = lines.next() else {
85        return Err(ParseError::BadHeader);
86    };
87    let header_line = strip_cr(header_raw);
88    if header_line.is_empty() {
89        return Err(ParseError::BadHeader);
90    }
91    check_line_len(header_idx as u64 + 1, header_line, budgets.max_line_bytes)?;
92    let header_fields = split_tsv_fields(header_line);
93    if !header_matches(&header_fields) {
94        return Err(ParseError::BadHeader);
95    }
96
97    let mut accepted = Vec::new();
98    let mut quarantined = Vec::new();
99
100    for (idx, raw) in lines {
101        let source_line = idx as u64 + 1;
102        let line = strip_cr(raw);
103        if line.is_empty() {
104            continue;
105        }
106        if cancel.load(Ordering::Relaxed) {
107            return Err(ParseError::Cancelled);
108        }
109        check_line_len(source_line, line, budgets.max_line_bytes)?;
110
111        let used = (accepted.len() + quarantined.len()) as u64;
112        if used >= budgets.max_records {
113            return Err(ParseError::RecordBudgetExceeded {
114                used,
115                max: budgets.max_records,
116            });
117        }
118
119        match parse_data_row(source_line, line) {
120            Ok(rec) => accepted.push(rec),
121            Err(q) => quarantined.push(q),
122        }
123    }
124
125    Ok(ParseReport {
126        accepted,
127        quarantined,
128        bytes_read,
129        release_label: release_label.to_string(),
130    })
131}
132
133fn strip_cr(line: &str) -> &str {
134    line.strip_suffix('\r').unwrap_or(line)
135}
136
137fn check_line_len(source_line: u64, line: &str, max: u64) -> Result<(), ParseError> {
138    let len = line.len() as u64;
139    if len > max {
140        return Err(ParseError::LineTooLong {
141            source_line,
142            len,
143            max,
144        });
145    }
146    Ok(())
147}
148
149#[cfg(test)]
150mod tests {
151    use super::*;
152
153    fn budgets(max_bytes: u64, max_records: u64, max_line_bytes: u64) -> ParseBudgets {
154        ParseBudgets {
155            max_bytes,
156            max_records,
157            max_line_bytes,
158        }
159    }
160
161    fn header() -> &'static str {
162        "ID\tSTATUS\tCHEBI_ACCESSION\tSOURCE\tPARENT_ID\tNAME\tDEFINITION"
163    }
164
165    fn cancel_flag(v: bool) -> AtomicBool {
166        AtomicBool::new(v)
167    }
168
169    #[test]
170    fn happy_path_accepts_synthetic_compounds() {
171        let tsv = format!(
172            "{}\n\
173             15377\tC\tCHEBI:15377\tChEBI\t\twater\toxidane\n\
174             16236\tC\tCHEBI:16236\tChEBI\t15377\tethanol\t\n\
175             17790\tE\tCHEBI:17790\t\t\tmethanol\ta simple alcohol\n",
176            header()
177        );
178        let report = parse_compounds_tsv(
179            tsv.as_bytes(),
180            budgets(10_000, 100, 2_000),
181            "chebi-rel-synthetic-1",
182            &cancel_flag(false),
183        )
184        .unwrap();
185        assert_eq!(report.accepted.len(), 3);
186        assert!(report.quarantined.is_empty());
187        assert_eq!(report.total_rows(), 3);
188        assert_eq!(report.release_label, "chebi-rel-synthetic-1");
189        assert_eq!(report.bytes_read, tsv.len() as u64);
190        assert_eq!(report.accepted[0].id, 15377);
191        assert_eq!(report.accepted[0].parent_id, None);
192        assert_eq!(report.accepted[1].parent_id, Some(15377));
193        assert_eq!(report.accepted[2].definition, "a simple alcohol");
194    }
195
196    #[test]
197    fn malformed_row_quarantined_others_accepted() {
198        let tsv = format!(
199            "{}\n\
200             1\tC\tCHEBI:1\ts\t\talpha\td1\n\
201             not-an-id\tC\tCHEBI:2\ts\t\tbeta\td2\n\
202             3\tC\tCHEBI:3\ts\t\tgamma\td3\n",
203            header()
204        );
205        let report = parse_compounds_tsv(
206            tsv.as_bytes(),
207            budgets(10_000, 100, 2_000),
208            "rel",
209            &cancel_flag(false),
210        )
211        .unwrap();
212        assert_eq!(report.accepted.len(), 2);
213        assert_eq!(report.quarantined.len(), 1);
214        assert_eq!(report.quarantined[0].reason, "malformed_id");
215        assert_eq!(report.quarantined[0].source_line, 3);
216        assert_eq!(report.total_rows(), 3);
217    }
218
219    #[test]
220    fn deleted_status_quarantined() {
221        let tsv = format!(
222            "{}\n\
223             9\tD\tCHEBI:9\ts\t\tgone\told\n\
224             10\tC\tCHEBI:10\ts\t\tkept\t\n",
225            header()
226        );
227        let report = parse_compounds_tsv(
228            tsv.as_bytes(),
229            budgets(10_000, 100, 2_000),
230            "rel",
231            &cancel_flag(false),
232        )
233        .unwrap();
234        assert_eq!(report.accepted.len(), 1);
235        assert_eq!(report.accepted[0].id, 10);
236        assert_eq!(report.quarantined.len(), 1);
237        assert_eq!(report.quarantined[0].reason, "deleted_status");
238    }
239
240    #[test]
241    fn oversize_max_bytes_errors() {
242        let tsv = format!("{}\n1\tC\tCHEBI:1\t\t\tn\td\n", header());
243        let err = parse_compounds_tsv(
244            tsv.as_bytes(),
245            budgets(8, 100, 2_000),
246            "rel",
247            &cancel_flag(false),
248        )
249        .unwrap_err();
250        assert_eq!(
251            err,
252            ParseError::ByteBudgetExceeded {
253                size: tsv.len() as u64,
254                max: 8
255            }
256        );
257    }
258
259    #[test]
260    fn cancellation_mid_parse() {
261        // Cancel is observed at the data-line boundary after a valid header
262        // (deterministic mid-parse; no thread race).
263        let tsv = format!(
264            "{}\n\
265             1\tC\tCHEBI:1\t\t\ta\t\n\
266             2\tC\tCHEBI:2\t\t\tb\t\n",
267            header()
268        );
269        assert_eq!(
270            parse_compounds_tsv(
271                tsv.as_bytes(),
272                budgets(10_000, 100, 2_000),
273                "rel",
274                &cancel_flag(true),
275            )
276            .unwrap_err(),
277            ParseError::Cancelled
278        );
279
280        // Header-only input with cancel set still succeeds (no data boundary).
281        let header_only = format!("{}\n", header());
282        let report = parse_compounds_tsv(
283            header_only.as_bytes(),
284            budgets(10_000, 100, 2_000),
285            "rel",
286            &cancel_flag(true),
287        )
288        .unwrap();
289        assert!(report.accepted.is_empty());
290        assert!(report.quarantined.is_empty());
291    }
292
293    #[test]
294    fn deterministic_mapping() {
295        let tsv = format!(
296            "{}\n\
297             5\tC\tCHEBI:5\tx\t\tok\td\n\
298             bad\tC\tCHEBI:6\tx\t\tnook\td\n\
299             7\tD\tCHEBI:7\tx\t\tdel\td\n",
300            header()
301        );
302        let a = parse_compounds_tsv(
303            tsv.as_bytes(),
304            budgets(10_000, 100, 2_000),
305            "same-label",
306            &cancel_flag(false),
307        )
308        .unwrap();
309        let b = parse_compounds_tsv(
310            tsv.as_bytes(),
311            budgets(10_000, 100, 2_000),
312            "same-label",
313            &cancel_flag(false),
314        )
315        .unwrap();
316        assert_eq!(a, b);
317    }
318
319    #[test]
320    fn empty_parent_id_is_none() {
321        let tsv = format!("{}\n42\tC\tCHEBI:42\t\t\tnamed\t\n", header());
322        let report = parse_compounds_tsv(
323            tsv.as_bytes(),
324            budgets(10_000, 100, 2_000),
325            "rel",
326            &cancel_flag(false),
327        )
328        .unwrap();
329        assert_eq!(report.accepted[0].parent_id, None);
330    }
331
332    #[test]
333    fn missing_header_is_bad_header() {
334        let tsv = "1\tC\tCHEBI:1\t\t\tn\td\n";
335        let err = parse_compounds_tsv(
336            tsv.as_bytes(),
337            budgets(10_000, 100, 2_000),
338            "rel",
339            &cancel_flag(false),
340        )
341        .unwrap_err();
342        assert_eq!(err, ParseError::BadHeader);
343
344        let empty = b"";
345        assert_eq!(
346            parse_compounds_tsv(
347                empty,
348                budgets(10_000, 100, 2_000),
349                "rel",
350                &cancel_flag(false)
351            )
352            .unwrap_err(),
353            ParseError::BadHeader
354        );
355    }
356
357    #[test]
358    fn trailing_columns_ignored() {
359        let tsv = format!(
360            "{}\tEXTRA\n\
361             1\tC\tCHEBI:1\ts\t\tn\td\tx\n",
362            header()
363        );
364        let report = parse_compounds_tsv(
365            tsv.as_bytes(),
366            budgets(10_000, 100, 2_000),
367            "rel",
368            &cancel_flag(false),
369        )
370        .unwrap();
371        assert_eq!(report.accepted.len(), 1);
372        assert_eq!(report.accepted[0].name, "n");
373    }
374}