1use serde::{Deserialize, Serialize};
4use sha2::{Digest, Sha256};
5use std::fs::File;
6use std::io::{self, BufReader, Read};
7use std::path::{Path, PathBuf};
8
9use super::job::append_window_hash;
10
11pub const WINDOW_BYTES: usize = 16 * 1024 * 1024;
12
13#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
14#[serde(rename_all = "snake_case")]
15pub enum IngestEncoding {
16 Identity,
17 Gzip,
18 Zstd,
19}
20
21#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
22#[serde(rename_all = "snake_case")]
23pub enum IngestRdfFormat {
24 Auto,
25 Turtle,
26 NTriples,
27 NQuads,
28 TriG,
29 RdfXml,
30 N3,
31 JsonLd,
32 CborLd,
33 YamlLd,
34 RdfJson,
35}
36
37impl IngestRdfFormat {
38 pub fn file_extension(self) -> &'static str {
40 match self {
41 Self::NTriples => "nt",
42 Self::NQuads => "nq",
43 Self::TriG => "trig",
44 Self::RdfXml => "rdf",
45 Self::N3 => "n3",
46 Self::JsonLd => "jsonld",
47 Self::CborLd => "cbor",
48 Self::YamlLd => "yamlld",
49 Self::RdfJson => "rj",
50 Self::Turtle | Self::Auto => "ttl",
51 }
52 }
53}
54
55#[derive(Clone, Debug, Serialize, Deserialize)]
56#[serde(rename_all = "snake_case")]
57pub enum IngestSourceKind {
58 File { path: String },
59 Url { url: String },
60}
61
62impl IngestSourceKind {
63 pub fn locator(&self) -> &str {
65 match self {
66 Self::File { path } => path,
67 Self::Url { url } => url,
68 }
69 }
70
71 pub fn is_seekable_identity(&self, encoding: IngestEncoding) -> bool {
73 matches!(self, Self::File { .. }) && encoding == IngestEncoding::Identity
74 }
75}
76
77#[derive(Default)]
78pub struct DigestOutcome {
79 pub full: std::sync::Mutex<Option<[u8; 32]>>,
80 pub bytes: std::sync::atomic::AtomicU64,
81 pub windows: std::sync::atomic::AtomicU64,
82}
83
84pub struct DigestingReader<R: Read> {
85 inner: R,
86 hasher: Sha256,
87 window: Sha256,
88 window_filled: usize,
89 uncompressed_bytes: u64,
90 windows_path: Option<PathBuf>,
91 window_count: u64,
92 outcome: Option<std::sync::Arc<DigestOutcome>>,
93 finalized: bool,
94}
95
96impl<R: Read> DigestingReader<R> {
97 pub fn new(inner: R, windows_path: Option<PathBuf>) -> Self {
99 Self::with_outcome(inner, windows_path, None)
100 }
101
102 pub fn with_outcome(
104 inner: R,
105 windows_path: Option<PathBuf>,
106 outcome: Option<std::sync::Arc<DigestOutcome>>,
107 ) -> Self {
108 Self {
109 inner,
110 hasher: Sha256::new(),
111 window: Sha256::new(),
112 window_filled: 0,
113 uncompressed_bytes: 0,
114 windows_path,
115 window_count: 0,
116 outcome,
117 finalized: false,
118 }
119 }
120
121 pub fn uncompressed_bytes(&self) -> u64 {
123 self.uncompressed_bytes
124 }
125
126 pub fn window_count(&self) -> u64 {
128 self.window_count
129 }
130
131 pub fn finish(mut self) -> io::Result<([u8; 32], u64, u64, Option<[u8; 32]>)> {
133 self.flush_tail()?;
134 let full: [u8; 32] = std::mem::replace(&mut self.hasher, Sha256::new())
135 .finalize()
136 .into();
137 self.publish_outcome(full);
138 self.finalized = true;
139 Ok((full, self.uncompressed_bytes, self.window_count, None))
140 }
141
142 fn publish_outcome(&self, full: [u8; 32]) {
143 if let Some(out) = &self.outcome {
144 *out.full.lock().unwrap_or_else(|e| e.into_inner()) = Some(full);
145 out.bytes.store(
146 self.uncompressed_bytes,
147 std::sync::atomic::Ordering::Relaxed,
148 );
149 out.windows
150 .store(self.window_count, std::sync::atomic::Ordering::Relaxed);
151 }
152 }
153
154 fn flush_tail(&mut self) -> io::Result<()> {
155 if self.window_filled > 0 {
156 let digest: [u8; 32] = std::mem::replace(&mut self.window, Sha256::new())
157 .finalize()
158 .into();
159 if let Some(path) = &self.windows_path {
160 append_window_hash(path, &digest)?;
161 }
162 self.window_count += 1;
163 self.window_filled = 0;
164 }
165 Ok(())
166 }
167}
168
169impl<R: Read> Drop for DigestingReader<R> {
170 fn drop(&mut self) {
171 if self.finalized {
172 return;
173 }
174 let _ = self.flush_tail();
175 let full: [u8; 32] = std::mem::replace(&mut self.hasher, Sha256::new())
176 .finalize()
177 .into();
178 self.publish_outcome(full);
179 }
180}
181
182impl<R: Read> Read for DigestingReader<R> {
183 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
184 let n = self.inner.read(buf)?;
185 if n == 0 {
186 return Ok(0);
187 }
188 let got = &buf[..n];
189 self.hasher.update(got);
190 self.uncompressed_bytes += n as u64;
191 let mut rest = got;
192 while !rest.is_empty() {
193 let need = WINDOW_BYTES - self.window_filled;
194 let take = rest.len().min(need);
195 self.window.update(&rest[..take]);
196 self.window_filled += take;
197 rest = &rest[take..];
198 if self.window_filled == WINDOW_BYTES {
199 let digest: [u8; 32] = self.window.finalize_reset().into();
200 if let Some(path) = &self.windows_path {
201 append_window_hash(path, &digest)?;
202 }
203 self.window_count += 1;
204 self.window_filled = 0;
205 }
206 }
207 Ok(n)
208 }
209}
210
211pub fn detect_encoding(name: &str, content_encoding: Option<&str>, magic: &[u8]) -> IngestEncoding {
213 if let Some(ce) = content_encoding {
214 let ce = ce.to_ascii_lowercase();
215 if ce.contains("gzip") {
216 return IngestEncoding::Gzip;
217 }
218 if ce.contains("zstd") || ce.contains("zst") {
219 return IngestEncoding::Zstd;
220 }
221 }
222 let lower = name.to_ascii_lowercase();
223 if lower.ends_with(".gz") || lower.ends_with(".tgz") {
224 return IngestEncoding::Gzip;
225 }
226 if lower.ends_with(".zst") || lower.ends_with(".zstd") {
227 return IngestEncoding::Zstd;
228 }
229 if magic.len() >= 2 && magic[0] == 0x1f && magic[1] == 0x8b {
230 return IngestEncoding::Gzip;
231 }
232 if magic.len() >= 4
233 && magic[0] == 0x28
234 && magic[1] == 0xb5
235 && magic[2] == 0x2f
236 && magic[3] == 0xfd
237 {
238 return IngestEncoding::Zstd;
239 }
240 IngestEncoding::Identity
241}
242
243pub fn infer_rdf_format(name: &str) -> IngestRdfFormat {
245 let lower = name.to_ascii_lowercase();
246 let stem = lower.trim_end_matches(".gz").trim_end_matches(".zst");
247 if stem.ends_with(".nt") || stem.ends_with(".ntriples") {
248 IngestRdfFormat::NTriples
249 } else if stem.ends_with(".nq") || stem.ends_with(".nquads") {
250 IngestRdfFormat::NQuads
251 } else if stem.ends_with(".trig") {
252 IngestRdfFormat::TriG
253 } else if stem.ends_with(".rdf") || stem.ends_with(".owl") || stem.ends_with(".xml") {
254 IngestRdfFormat::RdfXml
255 } else if stem.ends_with(".n3") {
256 IngestRdfFormat::N3
257 } else if stem.ends_with(".jsonld") || stem.ends_with(".json-ld") || stem.ends_with(".json") {
258 IngestRdfFormat::JsonLd
259 } else if stem.ends_with(".cbor") || stem.ends_with(".cborld") || stem.ends_with(".cbor-ld") {
260 IngestRdfFormat::CborLd
261 } else if stem.ends_with(".yamlld")
262 || stem.ends_with(".yaml-ld")
263 || stem.ends_with(".yml")
264 || stem.ends_with(".yaml")
265 {
266 IngestRdfFormat::YamlLd
267 } else if stem.ends_with(".rdfjson") || stem.ends_with(".rj") {
268 IngestRdfFormat::RdfJson
269 } else {
270 IngestRdfFormat::Turtle
271 }
272}
273
274pub struct OpenedSource {
275 pub reader: Box<dyn Read + Send>,
276 pub encoding: IngestEncoding,
277 pub format: IngestRdfFormat,
278 pub kind: IngestSourceKind,
279 pub content_length: Option<u64>,
280 pub etag: Option<String>,
281 pub resumable: bool,
282}
283
284pub fn open_ingest_source(
286 kind: &IngestSourceKind,
287 encoding_hint: Option<IngestEncoding>,
288 format_hint: IngestRdfFormat,
289) -> io::Result<OpenedSource> {
290 match kind {
291 IngestSourceKind::File { path } => open_file(Path::new(path), encoding_hint, format_hint),
292 IngestSourceKind::Url { url } => open_url(url, encoding_hint, format_hint),
293 }
294}
295
296fn open_file(
297 path: &Path,
298 encoding_hint: Option<IngestEncoding>,
299 format_hint: IngestRdfFormat,
300) -> io::Result<OpenedSource> {
301 let file = File::open(path)?;
302 let len = file.metadata().ok().map(|m| m.len());
303 let mut peek = BufReader::new(file);
304 let mut magic = [0u8; 4];
305 let n = peek.read(&mut magic)?;
306 let file = File::open(path)?;
308 let name = path.to_string_lossy();
309 let encoding = encoding_hint.unwrap_or_else(|| detect_encoding(&name, None, &magic[..n]));
310 if matches!(
311 encoding_hint,
312 None if name.to_ascii_lowercase().ends_with(".bz2")
313 ) || name.to_ascii_lowercase().ends_with(".bz2")
314 {
315 return Err(io::Error::new(
316 io::ErrorKind::InvalidInput,
317 "bzip2 sources are not streamed yet; use gzip, zstd, or uncompressed",
318 ));
319 }
320 let format = match format_hint {
321 IngestRdfFormat::Auto => infer_rdf_format(&name),
322 other => other,
323 };
324 let reader: Box<dyn Read + Send> = wrap_decoder(Box::new(file), encoding)?;
325 Ok(OpenedSource {
326 reader,
327 encoding,
328 format,
329 kind: IngestSourceKind::File {
330 path: path.to_string_lossy().into_owned(),
331 },
332 content_length: len,
333 etag: None,
334 resumable: encoding == IngestEncoding::Identity,
335 })
336}
337
338fn open_url(
339 url: &str,
340 encoding_hint: Option<IngestEncoding>,
341 format_hint: IngestRdfFormat,
342) -> io::Result<OpenedSource> {
343 let client = reqwest::blocking::Client::builder()
344 .user_agent(concat!("qualiaDB-ingest/", env!("CARGO_PKG_VERSION")))
345 .build()
346 .map_err(io_other)?;
347 let resp = client
348 .get(url)
349 .send()
350 .map_err(io_other)?
351 .error_for_status()
352 .map_err(io_other)?;
353 let etag = resp
354 .headers()
355 .get(reqwest::header::ETAG)
356 .and_then(|v| v.to_str().ok())
357 .map(str::to_string);
358 let content_encoding = resp
359 .headers()
360 .get(reqwest::header::CONTENT_ENCODING)
361 .and_then(|v| v.to_str().ok())
362 .map(str::to_string);
363 let content_length = resp.content_length();
364 let encoding =
365 encoding_hint.unwrap_or_else(|| detect_encoding(url, content_encoding.as_deref(), &[]));
366 if url.to_ascii_lowercase().ends_with(".bz2") {
367 return Err(io::Error::new(
368 io::ErrorKind::InvalidInput,
369 "bzip2 URL streams are not supported yet; use gzip, zstd, or uncompressed",
370 ));
371 }
372 let format = match format_hint {
373 IngestRdfFormat::Auto => infer_rdf_format(url),
374 other => other,
375 };
376 let reader: Box<dyn Read + Send> = wrap_decoder(Box::new(ResponseRead(resp)), encoding)?;
377 Ok(OpenedSource {
378 reader,
379 encoding,
380 format,
381 kind: IngestSourceKind::Url {
382 url: url.to_string(),
383 },
384 content_length,
385 etag,
386 resumable: false,
387 })
388}
389
390fn wrap_decoder(
391 inner: Box<dyn Read + Send>,
392 encoding: IngestEncoding,
393) -> io::Result<Box<dyn Read + Send>> {
394 match encoding {
395 IngestEncoding::Identity => Ok(inner),
396 IngestEncoding::Gzip => Ok(Box::new(flate2::read::GzDecoder::new(inner))),
397 IngestEncoding::Zstd => {
398 let dec = zstd::stream::read::Decoder::new(inner).map_err(io_other)?;
399 Ok(Box::new(dec))
400 }
401 }
402}
403
404struct ResponseRead(reqwest::blocking::Response);
406
407impl Read for ResponseRead {
408 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
409 Read::read(&mut self.0, buf)
410 }
411}
412
413fn io_other(e: impl std::fmt::Display) -> io::Error {
414 io::Error::other(e.to_string())
415}
416
417#[cfg(test)]
418mod tests {
419 use super::*;
420
421 #[test]
422 fn gzip_and_nt_inferred_from_name() {
423 assert_eq!(
424 detect_encoding("dump.ttl.gz", None, &[]),
425 IngestEncoding::Gzip
426 );
427 assert_eq!(
428 infer_rdf_format("https://ex.test/a.nt.gz"),
429 IngestRdfFormat::NTriples
430 );
431 assert_eq!(infer_rdf_format("graph.ttl"), IngestRdfFormat::Turtle);
432 assert_eq!(infer_rdf_format("doc.jsonld"), IngestRdfFormat::JsonLd);
433 assert_eq!(infer_rdf_format("doc.yamlld"), IngestRdfFormat::YamlLd);
434 assert_eq!(infer_rdf_format("doc.rj"), IngestRdfFormat::RdfJson);
435 assert_eq!(infer_rdf_format("schema.owl"), IngestRdfFormat::RdfXml);
436 assert_eq!(infer_rdf_format("g.nq"), IngestRdfFormat::NQuads);
437 }
438
439 #[test]
440 fn digesting_reader_hashes_and_windows() {
441 let data = vec![7u8; WINDOW_BYTES + 10];
442 let dir = tempfile::TempDir::new().unwrap();
443 let win = dir.path().join("windows.sha256");
444 let mut r = DigestingReader::new(std::io::Cursor::new(data.clone()), Some(win.clone()));
445 let mut sink = Vec::new();
446 r.read_to_end(&mut sink).unwrap();
447 assert_eq!(sink, data);
448 let (full, bytes, windows, _) = r.finish().unwrap();
449 assert_eq!(bytes, data.len() as u64);
450 assert_eq!(windows, 2);
451 let expect: [u8; 32] = Sha256::digest(&data).into();
452 assert_eq!(full, expect);
453 let lines = std::fs::read_to_string(win).unwrap();
454 assert_eq!(lines.lines().count(), 2);
455 }
456}