Skip to main content

qualia_core_db/net/qdnf/replication/
stream.rs

1//! Caller-paged larger-than-RAM replica stream (E06.6).
2//!
3//! A file is read in fixed [`PAGE_BYTES`] windows. The window is the only
4//! payload buffer on this path: the dataset is never loaded as a whole, and
5//! tests must not allocate a 42 MiB (or file-sized) replica buffer to prove
6//! paging. Sequential SHA-384 covers every page in order.
7
8use std::fs::{File, OpenOptions};
9use std::io::{Read, Seek, SeekFrom, Write};
10use std::path::Path;
11
12use sha2::{Digest, Sha384};
13
14use crate::net::qdnf::errors::QdnfError;
15use crate::net::qdnf::types::StrongDigest;
16
17/// Replica page window. Not a dataset allocation and not a 64 KiB QNF chunk.
18pub const PAGE_BYTES: usize = 4096;
19
20/// Caller-owned 4 KiB window plus fill/peak counters.
21pub struct PageWindow {
22    buf: [u8; PAGE_BYTES],
23    filled: usize,
24    peak_filled: usize,
25    pages_seen: u32,
26}
27
28impl PageWindow {
29    /// Creates a default `PageWindow`.
30    pub const fn new() -> Self {
31        Self {
32            buf: [0u8; PAGE_BYTES],
33            filled: 0,
34            peak_filled: 0,
35            pages_seen: 0,
36        }
37    }
38
39    #[inline]
40    /// `PageWindow::capacity`.
41    pub const fn capacity(&self) -> usize {
42        PAGE_BYTES
43    }
44
45    #[inline]
46    /// `PageWindow::filled`.
47    pub fn filled(&self) -> usize {
48        self.filled
49    }
50
51    /// Peak bytes placed in this window. Never the file length.
52    #[inline]
53    pub fn peak_bytes(&self) -> usize {
54        self.peak_filled
55    }
56
57    #[inline]
58    /// `PageWindow::pages_seen`.
59    pub fn pages_seen(&self) -> u32 {
60        self.pages_seen
61    }
62
63    #[inline]
64    /// Slice view.
65    pub fn as_slice(&self) -> &[u8] {
66        &self.buf[..self.filled]
67    }
68
69    fn accept(&mut self, n: usize) {
70        self.filled = n;
71        if n > self.peak_filled {
72            self.peak_filled = n;
73        }
74        if n > 0 {
75            self.pages_seen = self.pages_seen.saturating_add(1);
76        }
77    }
78}
79
80impl Default for PageWindow {
81    fn default() -> Self {
82        Self::new()
83    }
84}
85
86/// Open replica file. Length is metadata, not a loaded buffer.
87pub struct PagedFile {
88    file: File,
89    len: u64,
90}
91
92impl PagedFile {
93    /// Opens the resource.
94    pub fn open(path: impl AsRef<Path>) -> Result<Self, QdnfError> {
95        let file = File::open(path.as_ref()).map_err(|_| QdnfError::Incomplete)?;
96        let len = file.metadata().map_err(|_| QdnfError::Incomplete)?.len();
97        Ok(Self { file, len })
98    }
99
100    #[inline]
101    /// `PagedFile::logical_len`.
102    pub fn logical_len(&self) -> u64 {
103        self.len
104    }
105
106    /// Copy one page at `offset` into `window`. Offset at/past EOF yields 0.
107    pub fn read_page(&mut self, offset: u64, window: &mut PageWindow) -> Result<usize, QdnfError> {
108        if offset >= self.len {
109            window.accept(0);
110            return Ok(0);
111        }
112        self.file
113            .seek(SeekFrom::Start(offset))
114            .map_err(|_| QdnfError::Incomplete)?;
115        let remain = (self.len - offset) as usize;
116        let want = if remain < PAGE_BYTES {
117            remain
118        } else {
119            PAGE_BYTES
120        };
121        let n = self
122            .file
123            .read(&mut window.buf[..want])
124            .map_err(|_| QdnfError::Incomplete)?;
125        window.accept(n);
126        Ok(n)
127    }
128}
129
130/// Write `page_count` patterned pages. Holds at most one page in RAM.
131pub fn write_patterned_pages(
132    path: impl AsRef<Path>,
133    page_count: u32,
134    seed: u8,
135) -> Result<StrongDigest, QdnfError> {
136    if page_count == 0 {
137        return Err(QdnfError::Malformed);
138    }
139    let mut file = OpenOptions::new()
140        .create(true)
141        .write(true)
142        .truncate(true)
143        .open(path.as_ref())
144        .map_err(|_| QdnfError::Incomplete)?;
145    let mut page = [0u8; PAGE_BYTES];
146    let mut hasher = Sha384::new();
147    let mut i = 0u32;
148    while i < page_count {
149        fill_page(&mut page, seed, i);
150        file.write_all(&page).map_err(|_| QdnfError::Incomplete)?;
151        hasher.update(&page);
152        i += 1;
153    }
154    file.sync_all().map_err(|_| QdnfError::Incomplete)?;
155    Ok(finish(hasher))
156}
157
158/// Sequential SHA-384 over every page. `window` is the only payload buffer.
159pub fn hash_file_pages(
160    file: &mut PagedFile,
161    window: &mut PageWindow,
162    page_digests: &mut [StrongDigest],
163) -> Result<(StrongDigest, u32), QdnfError> {
164    if file.logical_len() == 0 {
165        return Err(QdnfError::Range);
166    }
167    let mut hasher = Sha384::new();
168    let mut offset = 0u64;
169    let mut pages = 0u32;
170    while offset < file.logical_len() {
171        let n = file.read_page(offset, window)?;
172        if n == 0 {
173            return Err(QdnfError::Range);
174        }
175        hasher.update(&window.buf[..n]);
176        if (pages as usize) < page_digests.len() {
177            page_digests[pages as usize] = page_digest(&window.buf[..n]);
178        }
179        offset = offset.checked_add(n as u64).ok_or(QdnfError::Range)?;
180        pages = pages.checked_add(1).ok_or(QdnfError::Range)?;
181    }
182    if offset != file.logical_len() {
183        return Err(QdnfError::Range);
184    }
185    Ok((finish(hasher), pages))
186}
187
188/// Hash every page and require `expected_len` plus `expected` digest.
189pub fn verify_file_pages(
190    file: &mut PagedFile,
191    expected_len: u64,
192    expected: StrongDigest,
193    window: &mut PageWindow,
194    page_digests: &mut [StrongDigest],
195) -> Result<(StrongDigest, u32), QdnfError> {
196    if expected == StrongDigest::ZERO {
197        return Err(QdnfError::Malformed);
198    }
199    if expected_len == 0 || file.logical_len() != expected_len {
200        return Err(QdnfError::Range);
201    }
202    let (digest, pages) = hash_file_pages(file, window, page_digests)?;
203    if digest != expected {
204        return Err(QdnfError::Conflict);
205    }
206    Ok((digest, pages))
207}
208
209fn fill_page(page: &mut [u8; PAGE_BYTES], seed: u8, index: u32) {
210    let mut i = 0usize;
211    while i < PAGE_BYTES {
212        page[i] = seed.wrapping_add(index as u8).wrapping_add(i as u8);
213        i += 1;
214    }
215}
216
217fn page_digest(bytes: &[u8]) -> StrongDigest {
218    let mut hasher = Sha384::new();
219    hasher.update(bytes);
220    finish(hasher)
221}
222
223fn finish(hasher: Sha384) -> StrongDigest {
224    let out = hasher.finalize();
225    let mut d = StrongDigest::ZERO;
226    d.0.copy_from_slice(&out);
227    d
228}
229
230#[cfg(test)]
231mod tests {
232    use super::*;
233    use tempfile::TempDir;
234
235    const FILE_PAGES: u32 = 16;
236    const FILE_BYTES: u64 = (FILE_PAGES as u64) * (PAGE_BYTES as u64);
237
238    #[test]
239    fn sixty_four_kib_file_pages_at_four_kib() {
240        let dir = TempDir::new().unwrap();
241        let path = dir.path().join("replica.bin");
242        let expected = write_patterned_pages(&path, FILE_PAGES, 0xA5).unwrap();
243        let mut file = PagedFile::open(&path).unwrap();
244        assert_eq!(file.logical_len(), FILE_BYTES);
245        assert!(FILE_BYTES > PAGE_BYTES as u64);
246
247        let mut window = PageWindow::new();
248        let mut page_digests = [StrongDigest::ZERO; FILE_PAGES as usize];
249        let (digest, pages) = verify_file_pages(
250            &mut file,
251            FILE_BYTES,
252            expected,
253            &mut window,
254            &mut page_digests,
255        )
256        .unwrap();
257        assert_eq!(digest, expected);
258        assert_eq!(pages, FILE_PAGES);
259        assert_eq!(window.pages_seen(), FILE_PAGES);
260        assert_eq!(window.capacity(), PAGE_BYTES);
261        assert_eq!(window.peak_bytes(), PAGE_BYTES);
262        assert!(core::mem::size_of::<PageWindow>() >= PAGE_BYTES);
263        assert!(core::mem::size_of::<PageWindow>() < FILE_BYTES as usize);
264
265        let mut i = 0usize;
266        while i < FILE_PAGES as usize {
267            assert_ne!(page_digests[i], StrongDigest::ZERO);
268            if i > 0 {
269                assert_ne!(page_digests[i], page_digests[i - 1]);
270            }
271            i += 1;
272        }
273
274        let mut wrong = expected;
275        wrong.0[0] ^= 1;
276        let mut window2 = PageWindow::new();
277        let mut digests2 = [StrongDigest::ZERO; FILE_PAGES as usize];
278        assert_eq!(
279            verify_file_pages(&mut file, FILE_BYTES, wrong, &mut window2, &mut digests2),
280            Err(QdnfError::Conflict)
281        );
282        assert_eq!(window2.peak_bytes(), PAGE_BYTES);
283    }
284
285    #[test]
286    fn length_mismatch_is_range() {
287        let dir = TempDir::new().unwrap();
288        let path = dir.path().join("short.bin");
289        let expected = write_patterned_pages(&path, 2, 0x11).unwrap();
290        let mut file = PagedFile::open(&path).unwrap();
291        let mut window = PageWindow::new();
292        let mut digests = [StrongDigest::ZERO; 2];
293        assert_eq!(
294            verify_file_pages(&mut file, 4096, expected, &mut window, &mut digests),
295            Err(QdnfError::Range)
296        );
297    }
298}