qualia_core_db/query/
ingest_resume.rs1use serde::{Deserialize, Serialize};
5use std::fs::File;
6use std::io::{self, Read, Seek, SeekFrom};
7use std::path::Path;
8use std::sync::atomic::{AtomicU64, Ordering};
9use std::sync::Arc;
10
11use crate::query::ingest_job::IngestRdfFormat;
12
13pub const RESUME_FILE: &str = "resume.json";
14
15#[derive(Clone, Debug, Serialize, Deserialize)]
16pub struct ResumeCursor {
17 pub schema: u32,
18 pub format: IngestRdfFormat,
19 pub byte_offset: u64,
20 pub triples: u64,
21 pub prolog: String,
22 pub seekable: bool,
23}
24
25impl ResumeCursor {
26 pub fn load(dir: &Path) -> io::Result<Option<Self>> {
28 let path = dir.join(RESUME_FILE);
29 if !path.is_file() {
30 return Ok(None);
31 }
32 Ok(Some(
33 serde_json::from_slice(&std::fs::read(path)?)
34 .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?,
35 ))
36 }
37
38 pub fn store(&self, dir: &Path) -> io::Result<()> {
40 let path = dir.join(RESUME_FILE);
41 let tmp = path.with_extension("json.tmp");
42 let body =
43 serde_json::to_vec(self).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
44 std::fs::write(&tmp, body)?;
45 std::fs::rename(tmp, path)
46 }
47}
48
49pub fn format_is_line_oriented(fmt: IngestRdfFormat) -> bool {
51 matches!(fmt, IngestRdfFormat::NTriples | IngestRdfFormat::NQuads)
52}
53
54pub fn format_uses_prolog(fmt: IngestRdfFormat) -> bool {
56 matches!(
57 fmt,
58 IngestRdfFormat::Turtle | IngestRdfFormat::TriG | IngestRdfFormat::N3
59 )
60}
61
62pub fn format_can_seek(fmt: IngestRdfFormat) -> bool {
64 format_is_line_oriented(fmt) || format_uses_prolog(fmt)
65}
66
67pub fn skip_newlines<R: Read>(
70 reader: &mut R,
71 n: u64,
72 tick: impl Fn(u64, u64),
73) -> io::Result<(u64, Vec<u8>)> {
74 if n == 0 {
75 return Ok((0, Vec::new()));
76 }
77 let mut buf = [0u8; 64 * 1024];
78 let mut lines = 0u64;
79 let mut bytes = 0u64;
80 let mut carry = Vec::new();
81 while lines < n {
82 let k = reader.read(&mut buf)?;
83 if k == 0 {
84 break;
85 }
86 for (i, &b) in buf[..k].iter().enumerate() {
87 bytes += 1;
88 if b == b'\n' {
89 lines += 1;
90 if lines % 2_000_000 == 0 {
91 tick(lines, bytes);
92 }
93 if lines == n {
94 carry.extend_from_slice(&buf[i + 1..k]);
95 tick(lines, bytes);
96 return Ok((bytes, carry));
97 }
98 }
99 }
100 }
101 tick(lines, bytes);
102 Ok((bytes, carry))
103}
104
105pub struct ByteCounter<R> {
107 inner: R,
108 count: Arc<AtomicU64>,
109}
110
111impl<R> ByteCounter<R> {
112 pub fn new(inner: R, count: Arc<AtomicU64>) -> Self {
114 Self { inner, count }
115 }
116
117 pub fn count(&self) -> u64 {
119 self.count.load(Ordering::Relaxed)
120 }
121}
122
123impl<R: Read> Read for ByteCounter<R> {
124 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
125 let n = self.inner.read(buf)?;
126 if n > 0 {
127 self.count.fetch_add(n as u64, Ordering::Relaxed);
128 }
129 Ok(n)
130 }
131}
132
133pub struct PrefixCapture<R> {
135 inner: R,
136 pub prolog: String,
137 line: Vec<u8>,
138 finished_header: bool,
139}
140
141impl<R> PrefixCapture<R> {
142 pub fn new(inner: R) -> Self {
144 Self {
145 inner,
146 prolog: String::new(),
147 line: Vec::new(),
148 finished_header: false,
149 }
150 }
151}
152
153impl<R: Read> Read for PrefixCapture<R> {
154 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
155 let n = self.inner.read(buf)?;
156 if n == 0 || self.finished_header {
157 return Ok(n);
158 }
159 for &b in &buf[..n] {
160 self.line.push(b);
161 if b == b'\n' {
162 consider_line(&mut self.prolog, &self.line, &mut self.finished_header);
163 self.line.clear();
164 }
165 }
166 Ok(n)
167 }
168}
169
170fn consider_line(prolog: &mut String, line: &[u8], finished: &mut bool) {
171 let s = std::str::from_utf8(line).unwrap_or("");
172 let t = s.trim_start();
173 if t.is_empty() || t.starts_with('#') {
174 if !*finished {
175 prolog.push_str(s);
176 }
177 return;
178 }
179 let lower = t.to_ascii_lowercase();
180 if lower.starts_with("@prefix")
181 || lower.starts_with("@base")
182 || lower.starts_with("prefix ")
183 || lower.starts_with("base ")
184 {
185 prolog.push_str(s);
186 return;
187 }
188 *finished = true;
189}
190
191pub fn open_resumed_file(
193 path: &Path,
194 cursor: Option<&ResumeCursor>,
195) -> io::Result<(Box<dyn Read + Send>, u64, String)> {
196 let mut file = File::open(path)?;
197 let Some(cur) = cursor.filter(|c| c.seekable && c.byte_offset > 0) else {
198 return Ok((Box::new(file), 0, String::new()));
199 };
200 file.seek(SeekFrom::Start(cur.byte_offset))?;
201 if cur.prolog.is_empty() {
202 return Ok((Box::new(file), cur.byte_offset, String::new()));
203 }
204 let header = std::io::Cursor::new(cur.prolog.clone().into_bytes());
205 Ok((
206 Box::new(header.chain(file)),
207 cur.byte_offset,
208 cur.prolog.clone(),
209 ))
210}
211
212#[cfg(test)]
213mod tests {
214 use super::*;
215
216 #[test]
217 fn skip_newlines_stops_after_n() {
218 let src = b"a\nb\nc\nd\n";
219 let mut cur = std::io::Cursor::new(&src[..]);
220 let (bytes, tail) = skip_newlines(&mut cur, 2, |_, _| {}).unwrap();
221 assert_eq!(bytes, 4);
222 let mut rest = String::from_utf8(tail).unwrap();
223 cur.read_to_string(&mut rest).unwrap();
224 assert_eq!(rest, "c\nd\n");
225 }
226
227 #[test]
228 fn prefix_capture_keeps_prefixes_only() {
229 let src = b"@prefix ex: <http://ex/> .\n# c\nex:a ex:b ex:c .\n";
230 let mut cap = PrefixCapture::new(&src[..]);
231 let mut out = Vec::new();
232 cap.read_to_end(&mut out).unwrap();
233 assert!(cap.prolog.contains("@prefix ex:"));
234 assert!(!cap.prolog.contains("ex:a ex:b"));
235 }
236}