Skip to main content

qualia_core_db/query/ingest_job/
source.rs

1//! Seekable files, HTTP streams, gzip/zstd inflate, and SHA-256 of what Rio sees.
2
3use serde::{Deserialize, Serialize};
4use sha2::{Digest, Sha256};
5use std::fs::File;
6use std::io::{self, BufReader, Read};
7use std::path::{Path, PathBuf};
8
9use super::job::append_window_hash;
10
11pub const WINDOW_BYTES: usize = 16 * 1024 * 1024;
12
13#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
14#[serde(rename_all = "snake_case")]
15pub enum IngestEncoding {
16    Identity,
17    Gzip,
18    Zstd,
19}
20
21#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
22#[serde(rename_all = "snake_case")]
23pub enum IngestRdfFormat {
24    Auto,
25    Turtle,
26    NTriples,
27    NQuads,
28    TriG,
29    RdfXml,
30    N3,
31    JsonLd,
32    CborLd,
33    YamlLd,
34    RdfJson,
35}
36
37impl IngestRdfFormat {
38    /// Canonical file extension for this RDF format.
39    pub fn file_extension(self) -> &'static str {
40        match self {
41            Self::NTriples => "nt",
42            Self::NQuads => "nq",
43            Self::TriG => "trig",
44            Self::RdfXml => "rdf",
45            Self::N3 => "n3",
46            Self::JsonLd => "jsonld",
47            Self::CborLd => "cbor",
48            Self::YamlLd => "yamlld",
49            Self::RdfJson => "rj",
50            Self::Turtle | Self::Auto => "ttl",
51        }
52    }
53}
54
55#[derive(Clone, Debug, Serialize, Deserialize)]
56#[serde(rename_all = "snake_case")]
57pub enum IngestSourceKind {
58    File { path: String },
59    Url { url: String },
60}
61
62impl IngestSourceKind {
63    /// The source locator string (path or URL).
64    pub fn locator(&self) -> &str {
65        match self {
66            Self::File { path } => path,
67            Self::Url { url } => url,
68        }
69    }
70
71    /// `true` when the source is a seekable local file with identity (uncompressed) encoding.
72    pub fn is_seekable_identity(&self, encoding: IngestEncoding) -> bool {
73        matches!(self, Self::File { .. }) && encoding == IngestEncoding::Identity
74    }
75}
76
77#[derive(Default)]
78pub struct DigestOutcome {
79    pub full: std::sync::Mutex<Option<[u8; 32]>>,
80    pub bytes: std::sync::atomic::AtomicU64,
81    pub windows: std::sync::atomic::AtomicU64,
82}
83
84pub struct DigestingReader<R: Read> {
85    inner: R,
86    hasher: Sha256,
87    window: Sha256,
88    window_filled: usize,
89    uncompressed_bytes: u64,
90    windows_path: Option<PathBuf>,
91    window_count: u64,
92    outcome: Option<std::sync::Arc<DigestOutcome>>,
93    finalized: bool,
94}
95
96impl<R: Read> DigestingReader<R> {
97    /// Wraps `inner` recording bytes + 16 MiB window hashes to `windows_path`.
98    pub fn new(inner: R, windows_path: Option<PathBuf>) -> Self {
99        Self::with_outcome(inner, windows_path, None)
100    }
101
102    /// As `new`, plus an outcome sink that receives the final digest result.
103    pub fn with_outcome(
104        inner: R,
105        windows_path: Option<PathBuf>,
106        outcome: Option<std::sync::Arc<DigestOutcome>>,
107    ) -> Self {
108        Self {
109            inner,
110            hasher: Sha256::new(),
111            window: Sha256::new(),
112            window_filled: 0,
113            uncompressed_bytes: 0,
114            windows_path,
115            window_count: 0,
116            outcome,
117            finalized: false,
118        }
119    }
120
121    /// Bytes seen so far after decompression.
122    pub fn uncompressed_bytes(&self) -> u64 {
123        self.uncompressed_bytes
124    }
125
126    /// Complete 16 MiB windows hashed so far.
127    pub fn window_count(&self) -> u64 {
128        self.window_count
129    }
130
131    /// Finalizes: returns `(sha256, bytes, window_count, window_commitment)`.
132    pub fn finish(mut self) -> io::Result<([u8; 32], u64, u64, Option<[u8; 32]>)> {
133        self.flush_tail()?;
134        let full: [u8; 32] = std::mem::replace(&mut self.hasher, Sha256::new())
135            .finalize()
136            .into();
137        self.publish_outcome(full);
138        self.finalized = true;
139        Ok((full, self.uncompressed_bytes, self.window_count, None))
140    }
141
142    fn publish_outcome(&self, full: [u8; 32]) {
143        if let Some(out) = &self.outcome {
144            *out.full.lock().unwrap_or_else(|e| e.into_inner()) = Some(full);
145            out.bytes.store(
146                self.uncompressed_bytes,
147                std::sync::atomic::Ordering::Relaxed,
148            );
149            out.windows
150                .store(self.window_count, std::sync::atomic::Ordering::Relaxed);
151        }
152    }
153
154    fn flush_tail(&mut self) -> io::Result<()> {
155        if self.window_filled > 0 {
156            let digest: [u8; 32] = std::mem::replace(&mut self.window, Sha256::new())
157                .finalize()
158                .into();
159            if let Some(path) = &self.windows_path {
160                append_window_hash(path, &digest)?;
161            }
162            self.window_count += 1;
163            self.window_filled = 0;
164        }
165        Ok(())
166    }
167}
168
169impl<R: Read> Drop for DigestingReader<R> {
170    fn drop(&mut self) {
171        if self.finalized {
172            return;
173        }
174        let _ = self.flush_tail();
175        let full: [u8; 32] = std::mem::replace(&mut self.hasher, Sha256::new())
176            .finalize()
177            .into();
178        self.publish_outcome(full);
179    }
180}
181
182impl<R: Read> Read for DigestingReader<R> {
183    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
184        let n = self.inner.read(buf)?;
185        if n == 0 {
186            return Ok(0);
187        }
188        let got = &buf[..n];
189        self.hasher.update(got);
190        self.uncompressed_bytes += n as u64;
191        let mut rest = got;
192        while !rest.is_empty() {
193            let need = WINDOW_BYTES - self.window_filled;
194            let take = rest.len().min(need);
195            self.window.update(&rest[..take]);
196            self.window_filled += take;
197            rest = &rest[take..];
198            if self.window_filled == WINDOW_BYTES {
199                let digest: [u8; 32] = self.window.finalize_reset().into();
200                if let Some(path) = &self.windows_path {
201                    append_window_hash(path, &digest)?;
202                }
203                self.window_count += 1;
204                self.window_filled = 0;
205            }
206        }
207        Ok(n)
208    }
209}
210
211/// Detects ingest encoding from filename, `Content-Encoding`, and magic bytes.
212pub fn detect_encoding(name: &str, content_encoding: Option<&str>, magic: &[u8]) -> IngestEncoding {
213    if let Some(ce) = content_encoding {
214        let ce = ce.to_ascii_lowercase();
215        if ce.contains("gzip") {
216            return IngestEncoding::Gzip;
217        }
218        if ce.contains("zstd") || ce.contains("zst") {
219            return IngestEncoding::Zstd;
220        }
221    }
222    let lower = name.to_ascii_lowercase();
223    if lower.ends_with(".gz") || lower.ends_with(".tgz") {
224        return IngestEncoding::Gzip;
225    }
226    if lower.ends_with(".zst") || lower.ends_with(".zstd") {
227        return IngestEncoding::Zstd;
228    }
229    if magic.len() >= 2 && magic[0] == 0x1f && magic[1] == 0x8b {
230        return IngestEncoding::Gzip;
231    }
232    if magic.len() >= 4
233        && magic[0] == 0x28
234        && magic[1] == 0xb5
235        && magic[2] == 0x2f
236        && magic[3] == 0xfd
237    {
238        return IngestEncoding::Zstd;
239    }
240    IngestEncoding::Identity
241}
242
243/// Infers RDF format from the filename extension.
244pub fn infer_rdf_format(name: &str) -> IngestRdfFormat {
245    let lower = name.to_ascii_lowercase();
246    let stem = lower.trim_end_matches(".gz").trim_end_matches(".zst");
247    if stem.ends_with(".nt") || stem.ends_with(".ntriples") {
248        IngestRdfFormat::NTriples
249    } else if stem.ends_with(".nq") || stem.ends_with(".nquads") {
250        IngestRdfFormat::NQuads
251    } else if stem.ends_with(".trig") {
252        IngestRdfFormat::TriG
253    } else if stem.ends_with(".rdf") || stem.ends_with(".owl") || stem.ends_with(".xml") {
254        IngestRdfFormat::RdfXml
255    } else if stem.ends_with(".n3") {
256        IngestRdfFormat::N3
257    } else if stem.ends_with(".jsonld") || stem.ends_with(".json-ld") || stem.ends_with(".json") {
258        IngestRdfFormat::JsonLd
259    } else if stem.ends_with(".cbor") || stem.ends_with(".cborld") || stem.ends_with(".cbor-ld") {
260        IngestRdfFormat::CborLd
261    } else if stem.ends_with(".yamlld")
262        || stem.ends_with(".yaml-ld")
263        || stem.ends_with(".yml")
264        || stem.ends_with(".yaml")
265    {
266        IngestRdfFormat::YamlLd
267    } else if stem.ends_with(".rdfjson") || stem.ends_with(".rj") {
268        IngestRdfFormat::RdfJson
269    } else {
270        IngestRdfFormat::Turtle
271    }
272}
273
274pub struct OpenedSource {
275    pub reader: Box<dyn Read + Send>,
276    pub encoding: IngestEncoding,
277    pub format: IngestRdfFormat,
278    pub kind: IngestSourceKind,
279    pub content_length: Option<u64>,
280    pub etag: Option<String>,
281    pub resumable: bool,
282}
283
284/// Opens and wraps the source (decompression + digesting reader) per kind/encoding.
285pub fn open_ingest_source(
286    kind: &IngestSourceKind,
287    encoding_hint: Option<IngestEncoding>,
288    format_hint: IngestRdfFormat,
289) -> io::Result<OpenedSource> {
290    match kind {
291        IngestSourceKind::File { path } => open_file(Path::new(path), encoding_hint, format_hint),
292        IngestSourceKind::Url { url } => open_url(url, encoding_hint, format_hint),
293    }
294}
295
296fn open_file(
297    path: &Path,
298    encoding_hint: Option<IngestEncoding>,
299    format_hint: IngestRdfFormat,
300) -> io::Result<OpenedSource> {
301    let file = File::open(path)?;
302    let len = file.metadata().ok().map(|m| m.len());
303    let mut peek = BufReader::new(file);
304    let mut magic = [0u8; 4];
305    let n = peek.read(&mut magic)?;
306    // Rewind after peek.
307    let file = File::open(path)?;
308    let name = path.to_string_lossy();
309    let encoding = encoding_hint.unwrap_or_else(|| detect_encoding(&name, None, &magic[..n]));
310    if matches!(
311        encoding_hint,
312        None if name.to_ascii_lowercase().ends_with(".bz2")
313    ) || name.to_ascii_lowercase().ends_with(".bz2")
314    {
315        return Err(io::Error::new(
316            io::ErrorKind::InvalidInput,
317            "bzip2 sources are not streamed yet; use gzip, zstd, or uncompressed",
318        ));
319    }
320    let format = match format_hint {
321        IngestRdfFormat::Auto => infer_rdf_format(&name),
322        other => other,
323    };
324    let reader: Box<dyn Read + Send> = wrap_decoder(Box::new(file), encoding)?;
325    Ok(OpenedSource {
326        reader,
327        encoding,
328        format,
329        kind: IngestSourceKind::File {
330            path: path.to_string_lossy().into_owned(),
331        },
332        content_length: len,
333        etag: None,
334        resumable: encoding == IngestEncoding::Identity,
335    })
336}
337
338fn open_url(
339    url: &str,
340    encoding_hint: Option<IngestEncoding>,
341    format_hint: IngestRdfFormat,
342) -> io::Result<OpenedSource> {
343    let client = reqwest::blocking::Client::builder()
344        .user_agent(concat!("qualiaDB-ingest/", env!("CARGO_PKG_VERSION")))
345        .build()
346        .map_err(io_other)?;
347    let resp = client
348        .get(url)
349        .send()
350        .map_err(io_other)?
351        .error_for_status()
352        .map_err(io_other)?;
353    let etag = resp
354        .headers()
355        .get(reqwest::header::ETAG)
356        .and_then(|v| v.to_str().ok())
357        .map(str::to_string);
358    let content_encoding = resp
359        .headers()
360        .get(reqwest::header::CONTENT_ENCODING)
361        .and_then(|v| v.to_str().ok())
362        .map(str::to_string);
363    let content_length = resp.content_length();
364    let encoding =
365        encoding_hint.unwrap_or_else(|| detect_encoding(url, content_encoding.as_deref(), &[]));
366    if url.to_ascii_lowercase().ends_with(".bz2") {
367        return Err(io::Error::new(
368            io::ErrorKind::InvalidInput,
369            "bzip2 URL streams are not supported yet; use gzip, zstd, or uncompressed",
370        ));
371    }
372    let format = match format_hint {
373        IngestRdfFormat::Auto => infer_rdf_format(url),
374        other => other,
375    };
376    let reader: Box<dyn Read + Send> = wrap_decoder(Box::new(ResponseRead(resp)), encoding)?;
377    Ok(OpenedSource {
378        reader,
379        encoding,
380        format,
381        kind: IngestSourceKind::Url {
382            url: url.to_string(),
383        },
384        content_length,
385        etag,
386        resumable: false,
387    })
388}
389
390fn wrap_decoder(
391    inner: Box<dyn Read + Send>,
392    encoding: IngestEncoding,
393) -> io::Result<Box<dyn Read + Send>> {
394    match encoding {
395        IngestEncoding::Identity => Ok(inner),
396        IngestEncoding::Gzip => Ok(Box::new(flate2::read::GzDecoder::new(inner))),
397        IngestEncoding::Zstd => {
398            let dec = zstd::stream::read::Decoder::new(inner).map_err(io_other)?;
399            Ok(Box::new(dec))
400        }
401    }
402}
403
404/// reqwest 0.13 blocking Response is Read on the default blocking API.
405struct ResponseRead(reqwest::blocking::Response);
406
407impl Read for ResponseRead {
408    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
409        Read::read(&mut self.0, buf)
410    }
411}
412
413fn io_other(e: impl std::fmt::Display) -> io::Error {
414    io::Error::other(e.to_string())
415}
416
417#[cfg(test)]
418mod tests {
419    use super::*;
420
421    #[test]
422    fn gzip_and_nt_inferred_from_name() {
423        assert_eq!(
424            detect_encoding("dump.ttl.gz", None, &[]),
425            IngestEncoding::Gzip
426        );
427        assert_eq!(
428            infer_rdf_format("https://ex.test/a.nt.gz"),
429            IngestRdfFormat::NTriples
430        );
431        assert_eq!(infer_rdf_format("graph.ttl"), IngestRdfFormat::Turtle);
432        assert_eq!(infer_rdf_format("doc.jsonld"), IngestRdfFormat::JsonLd);
433        assert_eq!(infer_rdf_format("doc.yamlld"), IngestRdfFormat::YamlLd);
434        assert_eq!(infer_rdf_format("doc.rj"), IngestRdfFormat::RdfJson);
435        assert_eq!(infer_rdf_format("schema.owl"), IngestRdfFormat::RdfXml);
436        assert_eq!(infer_rdf_format("g.nq"), IngestRdfFormat::NQuads);
437    }
438
439    #[test]
440    fn digesting_reader_hashes_and_windows() {
441        let data = vec![7u8; WINDOW_BYTES + 10];
442        let dir = tempfile::TempDir::new().unwrap();
443        let win = dir.path().join("windows.sha256");
444        let mut r = DigestingReader::new(std::io::Cursor::new(data.clone()), Some(win.clone()));
445        let mut sink = Vec::new();
446        r.read_to_end(&mut sink).unwrap();
447        assert_eq!(sink, data);
448        let (full, bytes, windows, _) = r.finish().unwrap();
449        assert_eq!(bytes, data.len() as u64);
450        assert_eq!(windows, 2);
451        let expect: [u8; 32] = Sha256::digest(&data).into();
452        assert_eq!(full, expect);
453        let lines = std::fs::read_to_string(win).unwrap();
454        assert_eq!(lines.lines().count(), 2);
455    }
456}