Skip to main content

qualia_core_db/platform/
host.rs

1//! Host-side I/O management for calculus modality.
2//!
3//! This module provides platform-specific zero-copy streaming implementations
4//! that bypass the OS page cache for deterministic edge compute. It manages
5//! memory-mapped I/O, io_uring (Linux), and IOCP (Windows) with strict alignment
6//! requirements for DMA transfers.
7//!
8//! ## Architecture
9//!
10//! - **ZeroCopyStreamer trait**: Platform-agnostic interface for async I/O
11//! - **DmaBuffer**: Page-aligned (4096-byte) buffers for DMA transfers
12//! - **Memory pinning**: Prevents swap to ensure DMA stability
13//! - **Double-buffering**: One buffer active for reading, one inactive for DMA
14//!
15//! ## Windows FFI Firewall
16//!
17//! Windows-specific IOCP implementation is isolated in directml_bridge.rs to avoid
18//! DirectX API version conflicts. This module uses type-erased FFI functions to
19//! communicate with the Windows-specific code.
20
21use std::fs::File;
22use std::io;
23use std::path::Path;
24
25// ─── Constants ─────────────────────────────────────────────────────────────────
26
27/// OS page size for DMA alignment (4096 bytes on most systems)
28pub const PAGE_SIZE: usize = 4096;
29
30/// Default buffer size (65536 bytes = 16 pages = 8192 f64 values)
31pub const DEFAULT_BUFFER_SIZE: usize = 65536;
32
33// ─── Errors ─────────────────────────────────────────────────────────────────────
34
35#[derive(Debug)]
36pub enum IoError {
37    MisalignedOffset { offset: u64, required: u64 },
38    MisalignedBufferSize { size: usize, required: usize },
39    FileOpenError(io::Error),
40    IoError(io::Error),
41    LockError(String),
42    InvalidState(String),
43}
44
45impl From<io::Error> for IoError {
46    fn from(err: io::Error) -> Self {
47        IoError::IoError(err)
48    }
49}
50
51// ─── DMA Buffer (Page-Aligned) ─────────────────────────────────────────────────
52
53/// DMA buffer aligned to OS page boundaries (4096 bytes).
54///
55/// Required for O_DIRECT (Linux) and FILE_FLAG_NO_BUFFERING (Windows) to ensure
56/// the NVMe controller can DMA directly into the buffer without kernel intervention.
57#[repr(C, align(4096))]
58pub struct DmaBuffer<const N: usize> {
59    data: [u8; N],
60}
61
62impl<const N: usize> DmaBuffer<N> {
63    /// Creates a new zero-initialized DMA buffer.
64    ///
65    /// # Panics
66    ///
67    /// Panics if N is not a multiple of PAGE_SIZE (4096).
68    pub fn new() -> Self {
69        assert!(
70            N % PAGE_SIZE == 0,
71            "DMA buffer size must be a multiple of PAGE_SIZE (4096), got {}",
72            N
73        );
74
75        Self { data: [0u8; N] }
76    }
77
78    /// Returns the buffer as a byte slice.
79    pub fn as_slice(&self) -> &[u8] {
80        &self.data
81    }
82
83    /// Returns the buffer as a mutable byte slice.
84    pub fn as_mut_slice(&mut self) -> &mut [u8] {
85        &mut self.data
86    }
87
88    /// Returns the buffer length in bytes.
89    pub fn len(&self) -> usize {
90        N
91    }
92
93    /// Returns true if the buffer is empty.
94    pub fn is_empty(&self) -> bool {
95        N == 0
96    }
97
98    /// Returns the number of f64 values this buffer can hold.
99    pub fn f64_capacity(&self) -> usize {
100        N / 8
101    }
102}
103
104impl<const N: usize> Default for DmaBuffer<N> {
105    fn default() -> Self {
106        Self::new()
107    }
108}
109
110// ─── ZeroCopyStreamer Trait ─────────────────────────────────────────────────────
111
112/// Platform-agnostic zero-copy streaming interface.
113///
114/// This trait abstracts over Linux io_uring and Windows IOCP, providing a
115/// consistent API for asynchronous DMA transfers. The host guarantees that
116/// one buffer is always safely mutable by the OS while the core reads from the other.
117pub trait ZeroCopyStreamer: Send {
118    /// Issues an asynchronous hardware read into the INACTIVE buffer.
119    ///
120    /// # Errors
121    ///
122    /// Returns an error if the offset is not 4096-byte aligned. Both O_DIRECT
123    /// and FILE_FLAG_NO_BUFFERING require strict sector alignment.
124    fn async_read_chunk(&mut self, offset: u64) -> Result<(), IoError>;
125
126    /// Non-blocking poll for completion.
127    ///
128    /// Returns `Some(&[u8])` only if the hardware has completed the DMA transfer
129    /// into the inactive buffer. Upon returning `Some`, internally swaps
130    /// active/inactive pointers.
131    ///
132    /// Returns `None` if the transfer is still in progress.
133    fn poll_completion(&mut self) -> Option<&[u8]>;
134
135    /// Returns the currently active buffer for SIMD chunking.
136    ///
137    /// This buffer is guaranteed to be stable and readable by the core.
138    fn get_active_buffer(&self) -> &[u8];
139
140    /// Returns the buffer size in bytes.
141    fn buffer_size(&self) -> usize;
142}
143
144// ─── Windows IOCP Implementation (FFI Firewall) ───────────────────────────────────
145
146/// Windows IOCP-based zero-copy streamer using FFI firewall.
147///
148/// This implementation delegates all Windows-specific operations to directml_bridge.rs
149/// via type-erased FFI functions to avoid DirectX API version conflicts.
150#[cfg(target_os = "windows")]
151pub struct IocpGridManager {
152    handle: *mut crate::directml_bridge::IocpHandle,
153    buffer_a: DmaBuffer<DEFAULT_BUFFER_SIZE>,
154    buffer_b: DmaBuffer<DEFAULT_BUFFER_SIZE>,
155    active_buffer: BufferId,
156    pending_read: bool,
157}
158
159#[cfg(target_os = "windows")]
160#[derive(Clone, Copy, PartialEq, Eq)]
161enum BufferId {
162    A,
163    B,
164}
165
166#[cfg(target_os = "windows")]
167unsafe impl Send for IocpGridManager {}
168
169#[cfg(target_os = "windows")]
170impl IocpGridManager {
171    /// Creates a new IOCP grid manager via FFI.
172    pub fn new(file_path: &Path) -> Result<Self, IoError> {
173        let path_str = file_path
174            .to_str()
175            .ok_or_else(|| IoError::InvalidState("Invalid UTF-8 path".to_string()))?;
176
177        unsafe {
178            let mut handle = std::ptr::null_mut();
179            let status = crate::directml_bridge::iocp_create_ffi(
180                path_str.as_ptr(),
181                path_str.len(),
182                &mut handle,
183            );
184
185            if status != crate::directml_bridge::DmlStatus::Success {
186                return Err(IoError::IoError(io::Error::new(
187                    io::ErrorKind::Other,
188                    status.message(),
189                )));
190            }
191
192            Ok(Self {
193                handle,
194                buffer_a: DmaBuffer::new(),
195                buffer_b: DmaBuffer::new(),
196                active_buffer: BufferId::A,
197                pending_read: false,
198            })
199        }
200    }
201
202    fn get_inactive_buffer_mut(&mut self) -> &mut [u8] {
203        match self.active_buffer {
204            BufferId::A => self.buffer_b.as_mut_slice(),
205            BufferId::B => self.buffer_a.as_mut_slice(),
206        }
207    }
208
209    fn get_inactive_buffer(&self) -> &[u8] {
210        match self.active_buffer {
211            BufferId::A => self.buffer_b.as_slice(),
212            BufferId::B => self.buffer_a.as_slice(),
213        }
214    }
215
216    fn swap_buffers(&mut self) {
217        self.active_buffer = match self.active_buffer {
218            BufferId::A => BufferId::B,
219            BufferId::B => BufferId::A,
220        };
221    }
222}
223
224#[cfg(target_os = "windows")]
225impl ZeroCopyStreamer for IocpGridManager {
226    fn async_read_chunk(&mut self, offset: u64) -> Result<(), IoError> {
227        // Validate 4096-byte alignment
228        if offset % PAGE_SIZE as u64 != 0 {
229            return Err(IoError::MisalignedOffset {
230                offset,
231                required: PAGE_SIZE as u64,
232            });
233        }
234
235        if self.pending_read {
236            return Err(IoError::InvalidState(
237                "Read already in progress. Call poll_completion first.".to_string(),
238            ));
239        }
240
241        // Prepare the inactive DMA buffer before the async transfer posts.
242        self.get_inactive_buffer_mut().fill(0);
243        let _dma_len = self.get_inactive_buffer().len();
244
245        unsafe {
246            let status = crate::directml_bridge::iocp_async_read_ffi(self.handle, offset);
247            if status != crate::directml_bridge::DmlStatus::Success {
248                return Err(IoError::IoError(io::Error::new(
249                    io::ErrorKind::Other,
250                    status.message(),
251                )));
252            }
253        }
254
255        self.pending_read = true;
256        Ok(())
257    }
258
259    fn poll_completion(&mut self) -> Option<&[u8]> {
260        if !self.pending_read {
261            return None;
262        }
263
264        unsafe {
265            let mut buffer_ptr = std::ptr::null();
266            let mut size = 0usize;
267
268            if crate::directml_bridge::iocp_poll_completion_ffi(
269                self.handle,
270                &mut buffer_ptr,
271                &mut size,
272            ) {
273                if !buffer_ptr.is_null() && size > 0 {
274                    let inactive = self.get_inactive_buffer_mut();
275                    let copy_len = size.min(inactive.len());
276                    std::ptr::copy_nonoverlapping(buffer_ptr, inactive.as_mut_ptr(), copy_len);
277                }
278                self.pending_read = false;
279                self.swap_buffers();
280                Some(self.get_active_buffer())
281            } else {
282                None
283            }
284        }
285    }
286
287    fn get_active_buffer(&self) -> &[u8] {
288        match self.active_buffer {
289            BufferId::A => self.buffer_a.as_slice(),
290            BufferId::B => self.buffer_b.as_slice(),
291        }
292    }
293
294    fn buffer_size(&self) -> usize {
295        DEFAULT_BUFFER_SIZE
296    }
297}
298
299#[cfg(target_os = "windows")]
300impl Drop for IocpGridManager {
301    fn drop(&mut self) {
302        unsafe {
303            crate::directml_bridge::iocp_destroy_ffi(self.handle);
304        }
305    }
306}
307
308// ─── Linux io_uring Implementation ─────────────────────────────────────────────
309
310#[cfg(target_os = "linux")]
311use libc::{c_void, mlock, O_DIRECT};
312#[cfg(target_os = "linux")]
313use std::os::unix::fs::OpenOptionsExt;
314#[cfg(target_os = "linux")]
315use std::os::unix::io::AsRawFd;
316
317/// Linux io_uring-based zero-copy streamer.
318///
319/// Uses O_DIRECT to bypass the kernel page cache and achieve deterministic
320/// DMA transfers from NVMe to RAM.
321#[cfg(target_os = "linux")]
322pub struct IoUringGridManager {
323    ring: io_uring::IoUring,
324    file: File,
325    buffer_a: DmaBuffer<DEFAULT_BUFFER_SIZE>,
326    buffer_b: DmaBuffer<DEFAULT_BUFFER_SIZE>,
327    active_buffer: BufferId,
328    pending_submission: bool,
329}
330
331#[cfg(target_os = "linux")]
332#[derive(Clone, Copy, PartialEq, Eq)]
333enum BufferId {
334    A,
335    B,
336}
337
338#[cfg(target_os = "linux")]
339impl IoUringGridManager {
340    /// Creates a new io_uring grid manager.
341    ///
342    /// Opens the file with O_DIRECT, creates an io_uring instance, and pins
343    /// both buffers in physical RAM.
344    pub fn new(file_path: &Path) -> Result<Self, IoError> {
345        let file = File::options()
346            .read(true)
347            .custom_flags(O_DIRECT)
348            .open(file_path)
349            .map_err(IoError::FileOpenError)?;
350
351        let ring = io_uring::IoUring::new(8).map_err(IoError::IoError)?;
352
353        let mut manager = Self {
354            ring,
355            file,
356            buffer_a: DmaBuffer::new(),
357            buffer_b: DmaBuffer::new(),
358            active_buffer: BufferId::A,
359            pending_submission: false,
360        };
361
362        // Pin buffers in physical RAM to prevent swap
363        manager.pin_buffers()?;
364
365        Ok(manager)
366    }
367
368    /// Pins both buffers in physical RAM using mlock.
369    fn pin_buffers(&mut self) -> Result<(), IoError> {
370        unsafe {
371            let result_a = mlock(
372                self.buffer_a.as_slice().as_ptr() as *const c_void,
373                self.buffer_a.len(),
374            );
375
376            let result_b = mlock(
377                self.buffer_b.as_slice().as_ptr() as *const c_void,
378                self.buffer_b.len(),
379            );
380
381            if result_a == 0 && result_b == 0 {
382                Ok(())
383            } else {
384                Err(IoError::LockError(
385                    "Failed to pin DMA buffers in physical RAM".to_string(),
386                ))
387            }
388        }
389    }
390
391    fn get_inactive_buffer_mut(&mut self) -> &mut [u8] {
392        match self.active_buffer {
393            BufferId::A => self.buffer_b.as_mut_slice(),
394            BufferId::B => self.buffer_a.as_mut_slice(),
395        }
396    }
397
398    fn get_inactive_buffer(&self) -> &[u8] {
399        match self.active_buffer {
400            BufferId::A => self.buffer_b.as_slice(),
401            BufferId::B => self.buffer_a.as_slice(),
402        }
403    }
404
405    fn swap_buffers(&mut self) {
406        self.active_buffer = match self.active_buffer {
407            BufferId::A => BufferId::B,
408            BufferId::B => BufferId::A,
409        };
410    }
411}
412
413#[cfg(target_os = "linux")]
414impl ZeroCopyStreamer for IoUringGridManager {
415    fn async_read_chunk(&mut self, offset: u64) -> Result<(), IoError> {
416        // Validate 4096-byte alignment
417        if offset % PAGE_SIZE as u64 != 0 {
418            return Err(IoError::MisalignedOffset {
419                offset,
420                required: PAGE_SIZE as u64,
421            });
422        }
423
424        if self.pending_submission {
425            return Err(IoError::InvalidState(
426                "Read already submitted. Call poll_completion first.".to_string(),
427            ));
428        }
429
430        let read_op = io_uring::opcode::Read::new(
431            io_uring::types::Fd(self.file.as_raw_fd()),
432            self.get_inactive_buffer_mut().as_mut_ptr(),
433            self.get_inactive_buffer().len() as u32,
434        )
435        .offset(offset)
436        .build();
437
438        unsafe {
439            self.ring.submission().push(&read_op).map_err(|e| {
440                IoError::IoError(io::Error::new(io::ErrorKind::Other, e.to_string()))
441            })?;
442        }
443
444        self.pending_submission = true;
445        Ok(())
446    }
447
448    fn poll_completion(&mut self) -> Option<&[u8]> {
449        if !self.pending_submission {
450            return None;
451        }
452
453        match self.ring.submit_and_wait(1) {
454            Ok(_) => {
455                let cqe_result = self.ring.completion().next().map(|cqe| cqe.result());
456                if let Some(res) = cqe_result {
457                    self.pending_submission = false;
458
459                    if res >= 0 {
460                        self.swap_buffers();
461                        Some(self.get_active_buffer())
462                    } else {
463                        // I/O error - reset state
464                        None
465                    }
466                } else {
467                    None
468                }
469            }
470            Err(_) => None,
471        }
472    }
473
474    fn get_active_buffer(&self) -> &[u8] {
475        match self.active_buffer {
476            BufferId::A => self.buffer_a.as_slice(),
477            BufferId::B => self.buffer_b.as_slice(),
478        }
479    }
480
481    fn buffer_size(&self) -> usize {
482        DEFAULT_BUFFER_SIZE
483    }
484}
485
486#[cfg(target_os = "linux")]
487impl Drop for IoUringGridManager {
488    fn drop(&mut self) {
489        unsafe {
490            // Unlock buffers
491            let _ = libc::munlock(
492                self.buffer_a.as_slice().as_ptr() as *const c_void,
493                self.buffer_a.len(),
494            );
495            let _ = libc::munlock(
496                self.buffer_b.as_slice().as_ptr() as *const c_void,
497                self.buffer_b.len(),
498            );
499        }
500    }
501}
502
503// ─── Fallback mmap Implementation (Non-Deterministic) ───────────────────────────
504
505/// Simple mmap-based grid manager for non-real-time use cases.
506///
507/// This implementation does not bypass the page cache and may incur page faults.
508/// Use only for desktop applications where occasional stalls are acceptable.
509pub struct MmapGridManager {
510    mmap: memmap2::Mmap,
511}
512
513impl MmapGridManager {
514    /// Creates a new mmap grid manager.
515    ///
516    /// Uses madvise to hint sequential read-ahead pattern.
517    pub fn new(file_path: &Path) -> Result<Self, IoError> {
518        let file = File::open(file_path).map_err(IoError::FileOpenError)?;
519        let mmap = unsafe { memmap2::Mmap::map(&file) }.map_err(IoError::IoError)?;
520
521        // Issue madvise for aggressive read-ahead
522        #[cfg(target_os = "linux")]
523        unsafe {
524            libc::madvise(
525                mmap.as_ptr() as *mut libc::c_void,
526                mmap.len(),
527                libc::MADV_SEQUENTIAL | libc::MADV_WILLNEED,
528            );
529        }
530
531        Ok(Self { mmap })
532    }
533
534    /// Returns the entire memory-mapped slice.
535    pub fn get_slice(&self) -> &[u8] {
536        &self.mmap
537    }
538}
539
540// ─── Tests ─────────────────────────────────────────────────────────────────────
541
542#[cfg(test)]
543mod tests {
544    use super::*;
545    use std::io::Write;
546
547    #[test]
548    fn test_dma_buffer_alignment() {
549        let buffer = DmaBuffer::<4096>::new();
550
551        // Verify pointer is 4096-byte aligned
552        assert_eq!(buffer.as_slice().as_ptr() as usize % 4096, 0);
553    }
554
555    #[test]
556    #[should_panic(expected = "DMA buffer size must be a multiple of PAGE_SIZE")]
557    fn test_dma_buffer_misaligned_size() {
558        let _buffer = DmaBuffer::<4095>::new();
559    }
560
561    #[test]
562    fn test_dma_buffer_f64_capacity() {
563        let buffer = DmaBuffer::<65536>::new();
564        assert_eq!(buffer.f64_capacity(), 8192); // 65536 / 8
565    }
566
567    #[cfg(target_os = "windows")]
568    #[test]
569    fn test_iocp_offset_validation() {
570        // Create a temporary test file
571        let temp_dir = std::env::temp_dir();
572        let file_path = temp_dir.join("test_iocp.dat");
573
574        let mut file = File::create(&file_path).unwrap();
575        // Write at least 2 pages of data with distinctive marker
576        let mut test_data = vec![0u8; 8192];
577        test_data[0..4].copy_from_slice(b"IOCP");
578        file.write_all(&test_data).unwrap();
579        file.sync_all().unwrap();
580
581        let mut streamer =
582            IocpGridManager::new(&file_path).expect("IocpGridManager creation should succeed");
583
584        // Test misaligned offset fails
585        let misaligned = streamer.async_read_chunk(100);
586        assert!(misaligned.is_err(), "Misaligned offset must return error");
587
588        // Test aligned read succeeds
589        assert!(
590            streamer.async_read_chunk(0).is_ok(),
591            "Aligned read at 0 must succeed"
592        );
593        let completed = streamer.poll_completion();
594        assert!(completed.is_some(), "poll_completion should return buffer");
595        let buf = completed.unwrap();
596        assert_eq!(&buf[0..4], b"IOCP", "Read buffer must match written data");
597
598        // Cleanup
599        std::fs::remove_file(&file_path).unwrap();
600    }
601
602    #[cfg(target_os = "linux")]
603    #[test]
604    fn test_iouring_offset_validation() {
605        // Create a temporary test file
606        let temp_dir = std::env::temp_dir();
607        let file_path = temp_dir.join("test_iouring.dat");
608
609        let mut file = File::create(&file_path).unwrap();
610        file.write_all(&vec![0u8; 8192]).unwrap();
611        file.sync_all().unwrap();
612
613        let mut manager = IoUringGridManager::new(&file_path).unwrap();
614
615        // Test valid offset (4096-byte aligned)
616        assert!(manager.async_read_chunk(4096).is_ok());
617
618        // Test invalid offset (not aligned)
619        let result = manager.async_read_chunk(4095);
620        assert!(matches!(result, Err(IoError::MisalignedOffset { .. })));
621
622        // Cleanup
623        std::fs::remove_file(&file_path).unwrap();
624    }
625}