Skip to main content

qualia_core_db/query/
ingest_resume.rs

1//! Format-aware resume: newline skip for NT/NQ, prefix+offset for Turtle/TriG/N3.
2//! Gzip/XML/JSON-LD stay skip-N (cannot seek honestly).
3
4use serde::{Deserialize, Serialize};
5use std::fs::File;
6use std::io::{self, Read, Seek, SeekFrom};
7use std::path::Path;
8use std::sync::atomic::{AtomicU64, Ordering};
9use std::sync::Arc;
10
11use crate::query::ingest_job::IngestRdfFormat;
12
13pub const RESUME_FILE: &str = "resume.json";
14
15#[derive(Clone, Debug, Serialize, Deserialize)]
16pub struct ResumeCursor {
17    pub schema: u32,
18    pub format: IngestRdfFormat,
19    pub byte_offset: u64,
20    pub triples: u64,
21    pub prolog: String,
22    pub seekable: bool,
23}
24
25impl ResumeCursor {
26    /// Loads a resumable cursor from `dir`; `None` when absent.
27    pub fn load(dir: &Path) -> io::Result<Option<Self>> {
28        let path = dir.join(RESUME_FILE);
29        if !path.is_file() {
30            return Ok(None);
31        }
32        Ok(Some(
33            serde_json::from_slice(&std::fs::read(path)?)
34                .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?,
35        ))
36    }
37
38    /// Atomically persists the cursor into `dir`.
39    pub fn store(&self, dir: &Path) -> io::Result<()> {
40        let path = dir.join(RESUME_FILE);
41        let tmp = path.with_extension("json.tmp");
42        let body =
43            serde_json::to_vec(self).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
44        std::fs::write(&tmp, body)?;
45        std::fs::rename(tmp, path)
46    }
47}
48
49/// `true` when `fmt` parses line-by-line (ND/NT/NQ).
50pub fn format_is_line_oriented(fmt: IngestRdfFormat) -> bool {
51    matches!(fmt, IngestRdfFormat::NTriples | IngestRdfFormat::NQuads)
52}
53
54/// `true` when `fmt` parses through the Prolog engine.
55pub fn format_uses_prolog(fmt: IngestRdfFormat) -> bool {
56    matches!(
57        fmt,
58        IngestRdfFormat::Turtle | IngestRdfFormat::TriG | IngestRdfFormat::N3
59    )
60}
61
62/// `true` when `fmt` supports byte-offset seeking.
63pub fn format_can_seek(fmt: IngestRdfFormat) -> bool {
64    format_is_line_oriented(fmt) || format_uses_prolog(fmt)
65}
66
67/// Skip `n` newline-terminated records. Returns (bytes consumed, unread tail
68/// after the nth newline so the next parser does not lose triples).
69pub fn skip_newlines<R: Read>(
70    reader: &mut R,
71    n: u64,
72    tick: impl Fn(u64, u64),
73) -> io::Result<(u64, Vec<u8>)> {
74    if n == 0 {
75        return Ok((0, Vec::new()));
76    }
77    let mut buf = [0u8; 64 * 1024];
78    let mut lines = 0u64;
79    let mut bytes = 0u64;
80    let mut carry = Vec::new();
81    while lines < n {
82        let k = reader.read(&mut buf)?;
83        if k == 0 {
84            break;
85        }
86        for (i, &b) in buf[..k].iter().enumerate() {
87            bytes += 1;
88            if b == b'\n' {
89                lines += 1;
90                if lines % 2_000_000 == 0 {
91                    tick(lines, bytes);
92                }
93                if lines == n {
94                    carry.extend_from_slice(&buf[i + 1..k]);
95                    tick(lines, bytes);
96                    return Ok((bytes, carry));
97                }
98            }
99        }
100    }
101    tick(lines, bytes);
102    Ok((bytes, carry))
103}
104
105/// Counts bytes the inner reader actually delivers (parser-facing).
106pub struct ByteCounter<R> {
107    inner: R,
108    count: Arc<AtomicU64>,
109}
110
111impl<R> ByteCounter<R> {
112    /// Wraps `inner`, counting bytes read into `count`.
113    pub fn new(inner: R, count: Arc<AtomicU64>) -> Self {
114        Self { inner, count }
115    }
116
117    /// Bytes read so far.
118    pub fn count(&self) -> u64 {
119        self.count.load(Ordering::Relaxed)
120    }
121}
122
123impl<R: Read> Read for ByteCounter<R> {
124    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
125        let n = self.inner.read(buf)?;
126        if n > 0 {
127            self.count.fetch_add(n as u64, Ordering::Relaxed);
128        }
129        Ok(n)
130    }
131}
132
133/// Copies `@prefix` / `@base` / `PREFIX` / `BASE` statements into `prolog`.
134pub struct PrefixCapture<R> {
135    inner: R,
136    pub prolog: String,
137    line: Vec<u8>,
138    finished_header: bool,
139}
140
141impl<R> PrefixCapture<R> {
142    /// Wraps `inner`, capturing a bounded prefix for format sniffing.
143    pub fn new(inner: R) -> Self {
144        Self {
145            inner,
146            prolog: String::new(),
147            line: Vec::new(),
148            finished_header: false,
149        }
150    }
151}
152
153impl<R: Read> Read for PrefixCapture<R> {
154    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
155        let n = self.inner.read(buf)?;
156        if n == 0 || self.finished_header {
157            return Ok(n);
158        }
159        for &b in &buf[..n] {
160            self.line.push(b);
161            if b == b'\n' {
162                consider_line(&mut self.prolog, &self.line, &mut self.finished_header);
163                self.line.clear();
164            }
165        }
166        Ok(n)
167    }
168}
169
170fn consider_line(prolog: &mut String, line: &[u8], finished: &mut bool) {
171    let s = std::str::from_utf8(line).unwrap_or("");
172    let t = s.trim_start();
173    if t.is_empty() || t.starts_with('#') {
174        if !*finished {
175            prolog.push_str(s);
176        }
177        return;
178    }
179    let lower = t.to_ascii_lowercase();
180    if lower.starts_with("@prefix")
181        || lower.starts_with("@base")
182        || lower.starts_with("prefix ")
183        || lower.starts_with("base ")
184    {
185        prolog.push_str(s);
186        return;
187    }
188    *finished = true;
189}
190
191/// Open a seekable file, optionally resume at `cursor`.
192pub fn open_resumed_file(
193    path: &Path,
194    cursor: Option<&ResumeCursor>,
195) -> io::Result<(Box<dyn Read + Send>, u64, String)> {
196    let mut file = File::open(path)?;
197    let Some(cur) = cursor.filter(|c| c.seekable && c.byte_offset > 0) else {
198        return Ok((Box::new(file), 0, String::new()));
199    };
200    file.seek(SeekFrom::Start(cur.byte_offset))?;
201    if cur.prolog.is_empty() {
202        return Ok((Box::new(file), cur.byte_offset, String::new()));
203    }
204    let header = std::io::Cursor::new(cur.prolog.clone().into_bytes());
205    Ok((
206        Box::new(header.chain(file)),
207        cur.byte_offset,
208        cur.prolog.clone(),
209    ))
210}
211
212#[cfg(test)]
213mod tests {
214    use super::*;
215
216    #[test]
217    fn skip_newlines_stops_after_n() {
218        let src = b"a\nb\nc\nd\n";
219        let mut cur = std::io::Cursor::new(&src[..]);
220        let (bytes, tail) = skip_newlines(&mut cur, 2, |_, _| {}).unwrap();
221        assert_eq!(bytes, 4);
222        let mut rest = String::from_utf8(tail).unwrap();
223        cur.read_to_string(&mut rest).unwrap();
224        assert_eq!(rest, "c\nd\n");
225    }
226
227    #[test]
228    fn prefix_capture_keeps_prefixes_only() {
229        let src = b"@prefix ex: <http://ex/> .\n# c\nex:a ex:b ex:c .\n";
230        let mut cap = PrefixCapture::new(&src[..]);
231        let mut out = Vec::new();
232        cap.read_to_end(&mut out).unwrap();
233        assert!(cap.prolog.contains("@prefix ex:"));
234        assert!(!cap.prolog.contains("ex:a ex:b"));
235    }
236}