1use std::fs::File;
22use std::io;
23use std::path::Path;
24
25pub const PAGE_SIZE: usize = 4096;
29
30pub const DEFAULT_BUFFER_SIZE: usize = 65536;
32
33#[derive(Debug)]
36pub enum IoError {
37 MisalignedOffset { offset: u64, required: u64 },
38 MisalignedBufferSize { size: usize, required: usize },
39 FileOpenError(io::Error),
40 IoError(io::Error),
41 LockError(String),
42 InvalidState(String),
43}
44
45impl From<io::Error> for IoError {
46 fn from(err: io::Error) -> Self {
47 IoError::IoError(err)
48 }
49}
50
51#[repr(C, align(4096))]
58pub struct DmaBuffer<const N: usize> {
59 data: [u8; N],
60}
61
62impl<const N: usize> DmaBuffer<N> {
63 pub fn new() -> Self {
69 assert!(
70 N % PAGE_SIZE == 0,
71 "DMA buffer size must be a multiple of PAGE_SIZE (4096), got {}",
72 N
73 );
74
75 Self { data: [0u8; N] }
76 }
77
78 pub fn as_slice(&self) -> &[u8] {
80 &self.data
81 }
82
83 pub fn as_mut_slice(&mut self) -> &mut [u8] {
85 &mut self.data
86 }
87
88 pub fn len(&self) -> usize {
90 N
91 }
92
93 pub fn is_empty(&self) -> bool {
95 N == 0
96 }
97
98 pub fn f64_capacity(&self) -> usize {
100 N / 8
101 }
102}
103
104impl<const N: usize> Default for DmaBuffer<N> {
105 fn default() -> Self {
106 Self::new()
107 }
108}
109
110pub trait ZeroCopyStreamer: Send {
118 fn async_read_chunk(&mut self, offset: u64) -> Result<(), IoError>;
125
126 fn poll_completion(&mut self) -> Option<&[u8]>;
134
135 fn get_active_buffer(&self) -> &[u8];
139
140 fn buffer_size(&self) -> usize;
142}
143
144#[cfg(target_os = "windows")]
151pub struct IocpGridManager {
152 handle: *mut crate::directml_bridge::IocpHandle,
153 buffer_a: DmaBuffer<DEFAULT_BUFFER_SIZE>,
154 buffer_b: DmaBuffer<DEFAULT_BUFFER_SIZE>,
155 active_buffer: BufferId,
156 pending_read: bool,
157}
158
159#[cfg(target_os = "windows")]
160#[derive(Clone, Copy, PartialEq, Eq)]
161enum BufferId {
162 A,
163 B,
164}
165
166#[cfg(target_os = "windows")]
167unsafe impl Send for IocpGridManager {}
168
169#[cfg(target_os = "windows")]
170impl IocpGridManager {
171 pub fn new(file_path: &Path) -> Result<Self, IoError> {
173 let path_str = file_path
174 .to_str()
175 .ok_or_else(|| IoError::InvalidState("Invalid UTF-8 path".to_string()))?;
176
177 unsafe {
178 let mut handle = std::ptr::null_mut();
179 let status = crate::directml_bridge::iocp_create_ffi(
180 path_str.as_ptr(),
181 path_str.len(),
182 &mut handle,
183 );
184
185 if status != crate::directml_bridge::DmlStatus::Success {
186 return Err(IoError::IoError(io::Error::new(
187 io::ErrorKind::Other,
188 status.message(),
189 )));
190 }
191
192 Ok(Self {
193 handle,
194 buffer_a: DmaBuffer::new(),
195 buffer_b: DmaBuffer::new(),
196 active_buffer: BufferId::A,
197 pending_read: false,
198 })
199 }
200 }
201
202 fn get_inactive_buffer_mut(&mut self) -> &mut [u8] {
203 match self.active_buffer {
204 BufferId::A => self.buffer_b.as_mut_slice(),
205 BufferId::B => self.buffer_a.as_mut_slice(),
206 }
207 }
208
209 fn get_inactive_buffer(&self) -> &[u8] {
210 match self.active_buffer {
211 BufferId::A => self.buffer_b.as_slice(),
212 BufferId::B => self.buffer_a.as_slice(),
213 }
214 }
215
216 fn swap_buffers(&mut self) {
217 self.active_buffer = match self.active_buffer {
218 BufferId::A => BufferId::B,
219 BufferId::B => BufferId::A,
220 };
221 }
222}
223
224#[cfg(target_os = "windows")]
225impl ZeroCopyStreamer for IocpGridManager {
226 fn async_read_chunk(&mut self, offset: u64) -> Result<(), IoError> {
227 if offset % PAGE_SIZE as u64 != 0 {
229 return Err(IoError::MisalignedOffset {
230 offset,
231 required: PAGE_SIZE as u64,
232 });
233 }
234
235 if self.pending_read {
236 return Err(IoError::InvalidState(
237 "Read already in progress. Call poll_completion first.".to_string(),
238 ));
239 }
240
241 self.get_inactive_buffer_mut().fill(0);
243 let _dma_len = self.get_inactive_buffer().len();
244
245 unsafe {
246 let status = crate::directml_bridge::iocp_async_read_ffi(self.handle, offset);
247 if status != crate::directml_bridge::DmlStatus::Success {
248 return Err(IoError::IoError(io::Error::new(
249 io::ErrorKind::Other,
250 status.message(),
251 )));
252 }
253 }
254
255 self.pending_read = true;
256 Ok(())
257 }
258
259 fn poll_completion(&mut self) -> Option<&[u8]> {
260 if !self.pending_read {
261 return None;
262 }
263
264 unsafe {
265 let mut buffer_ptr = std::ptr::null();
266 let mut size = 0usize;
267
268 if crate::directml_bridge::iocp_poll_completion_ffi(
269 self.handle,
270 &mut buffer_ptr,
271 &mut size,
272 ) {
273 if !buffer_ptr.is_null() && size > 0 {
274 let inactive = self.get_inactive_buffer_mut();
275 let copy_len = size.min(inactive.len());
276 std::ptr::copy_nonoverlapping(buffer_ptr, inactive.as_mut_ptr(), copy_len);
277 }
278 self.pending_read = false;
279 self.swap_buffers();
280 Some(self.get_active_buffer())
281 } else {
282 None
283 }
284 }
285 }
286
287 fn get_active_buffer(&self) -> &[u8] {
288 match self.active_buffer {
289 BufferId::A => self.buffer_a.as_slice(),
290 BufferId::B => self.buffer_b.as_slice(),
291 }
292 }
293
294 fn buffer_size(&self) -> usize {
295 DEFAULT_BUFFER_SIZE
296 }
297}
298
299#[cfg(target_os = "windows")]
300impl Drop for IocpGridManager {
301 fn drop(&mut self) {
302 unsafe {
303 crate::directml_bridge::iocp_destroy_ffi(self.handle);
304 }
305 }
306}
307
308#[cfg(target_os = "linux")]
311use libc::{c_void, mlock, O_DIRECT};
312#[cfg(target_os = "linux")]
313use std::os::unix::fs::OpenOptionsExt;
314#[cfg(target_os = "linux")]
315use std::os::unix::io::AsRawFd;
316
317#[cfg(target_os = "linux")]
322pub struct IoUringGridManager {
323 ring: io_uring::IoUring,
324 file: File,
325 buffer_a: DmaBuffer<DEFAULT_BUFFER_SIZE>,
326 buffer_b: DmaBuffer<DEFAULT_BUFFER_SIZE>,
327 active_buffer: BufferId,
328 pending_submission: bool,
329}
330
331#[cfg(target_os = "linux")]
332#[derive(Clone, Copy, PartialEq, Eq)]
333enum BufferId {
334 A,
335 B,
336}
337
338#[cfg(target_os = "linux")]
339impl IoUringGridManager {
340 pub fn new(file_path: &Path) -> Result<Self, IoError> {
345 let file = File::options()
346 .read(true)
347 .custom_flags(O_DIRECT)
348 .open(file_path)
349 .map_err(IoError::FileOpenError)?;
350
351 let ring = io_uring::IoUring::new(8).map_err(IoError::IoError)?;
352
353 let mut manager = Self {
354 ring,
355 file,
356 buffer_a: DmaBuffer::new(),
357 buffer_b: DmaBuffer::new(),
358 active_buffer: BufferId::A,
359 pending_submission: false,
360 };
361
362 manager.pin_buffers()?;
364
365 Ok(manager)
366 }
367
368 fn pin_buffers(&mut self) -> Result<(), IoError> {
370 unsafe {
371 let result_a = mlock(
372 self.buffer_a.as_slice().as_ptr() as *const c_void,
373 self.buffer_a.len(),
374 );
375
376 let result_b = mlock(
377 self.buffer_b.as_slice().as_ptr() as *const c_void,
378 self.buffer_b.len(),
379 );
380
381 if result_a == 0 && result_b == 0 {
382 Ok(())
383 } else {
384 Err(IoError::LockError(
385 "Failed to pin DMA buffers in physical RAM".to_string(),
386 ))
387 }
388 }
389 }
390
391 fn get_inactive_buffer_mut(&mut self) -> &mut [u8] {
392 match self.active_buffer {
393 BufferId::A => self.buffer_b.as_mut_slice(),
394 BufferId::B => self.buffer_a.as_mut_slice(),
395 }
396 }
397
398 fn get_inactive_buffer(&self) -> &[u8] {
399 match self.active_buffer {
400 BufferId::A => self.buffer_b.as_slice(),
401 BufferId::B => self.buffer_a.as_slice(),
402 }
403 }
404
405 fn swap_buffers(&mut self) {
406 self.active_buffer = match self.active_buffer {
407 BufferId::A => BufferId::B,
408 BufferId::B => BufferId::A,
409 };
410 }
411}
412
413#[cfg(target_os = "linux")]
414impl ZeroCopyStreamer for IoUringGridManager {
415 fn async_read_chunk(&mut self, offset: u64) -> Result<(), IoError> {
416 if offset % PAGE_SIZE as u64 != 0 {
418 return Err(IoError::MisalignedOffset {
419 offset,
420 required: PAGE_SIZE as u64,
421 });
422 }
423
424 if self.pending_submission {
425 return Err(IoError::InvalidState(
426 "Read already submitted. Call poll_completion first.".to_string(),
427 ));
428 }
429
430 let read_op = io_uring::opcode::Read::new(
431 io_uring::types::Fd(self.file.as_raw_fd()),
432 self.get_inactive_buffer_mut().as_mut_ptr(),
433 self.get_inactive_buffer().len() as u32,
434 )
435 .offset(offset)
436 .build();
437
438 unsafe {
439 self.ring.submission().push(&read_op).map_err(|e| {
440 IoError::IoError(io::Error::new(io::ErrorKind::Other, e.to_string()))
441 })?;
442 }
443
444 self.pending_submission = true;
445 Ok(())
446 }
447
448 fn poll_completion(&mut self) -> Option<&[u8]> {
449 if !self.pending_submission {
450 return None;
451 }
452
453 match self.ring.submit_and_wait(1) {
454 Ok(_) => {
455 let cqe_result = self.ring.completion().next().map(|cqe| cqe.result());
456 if let Some(res) = cqe_result {
457 self.pending_submission = false;
458
459 if res >= 0 {
460 self.swap_buffers();
461 Some(self.get_active_buffer())
462 } else {
463 None
465 }
466 } else {
467 None
468 }
469 }
470 Err(_) => None,
471 }
472 }
473
474 fn get_active_buffer(&self) -> &[u8] {
475 match self.active_buffer {
476 BufferId::A => self.buffer_a.as_slice(),
477 BufferId::B => self.buffer_b.as_slice(),
478 }
479 }
480
481 fn buffer_size(&self) -> usize {
482 DEFAULT_BUFFER_SIZE
483 }
484}
485
486#[cfg(target_os = "linux")]
487impl Drop for IoUringGridManager {
488 fn drop(&mut self) {
489 unsafe {
490 let _ = libc::munlock(
492 self.buffer_a.as_slice().as_ptr() as *const c_void,
493 self.buffer_a.len(),
494 );
495 let _ = libc::munlock(
496 self.buffer_b.as_slice().as_ptr() as *const c_void,
497 self.buffer_b.len(),
498 );
499 }
500 }
501}
502
503pub struct MmapGridManager {
510 mmap: memmap2::Mmap,
511}
512
513impl MmapGridManager {
514 pub fn new(file_path: &Path) -> Result<Self, IoError> {
518 let file = File::open(file_path).map_err(IoError::FileOpenError)?;
519 let mmap = unsafe { memmap2::Mmap::map(&file) }.map_err(IoError::IoError)?;
520
521 #[cfg(target_os = "linux")]
523 unsafe {
524 libc::madvise(
525 mmap.as_ptr() as *mut libc::c_void,
526 mmap.len(),
527 libc::MADV_SEQUENTIAL | libc::MADV_WILLNEED,
528 );
529 }
530
531 Ok(Self { mmap })
532 }
533
534 pub fn get_slice(&self) -> &[u8] {
536 &self.mmap
537 }
538}
539
540#[cfg(test)]
543mod tests {
544 use super::*;
545 use std::io::Write;
546
547 #[test]
548 fn test_dma_buffer_alignment() {
549 let buffer = DmaBuffer::<4096>::new();
550
551 assert_eq!(buffer.as_slice().as_ptr() as usize % 4096, 0);
553 }
554
555 #[test]
556 #[should_panic(expected = "DMA buffer size must be a multiple of PAGE_SIZE")]
557 fn test_dma_buffer_misaligned_size() {
558 let _buffer = DmaBuffer::<4095>::new();
559 }
560
561 #[test]
562 fn test_dma_buffer_f64_capacity() {
563 let buffer = DmaBuffer::<65536>::new();
564 assert_eq!(buffer.f64_capacity(), 8192); }
566
567 #[cfg(target_os = "windows")]
568 #[test]
569 fn test_iocp_offset_validation() {
570 let temp_dir = std::env::temp_dir();
572 let file_path = temp_dir.join("test_iocp.dat");
573
574 let mut file = File::create(&file_path).unwrap();
575 let mut test_data = vec![0u8; 8192];
577 test_data[0..4].copy_from_slice(b"IOCP");
578 file.write_all(&test_data).unwrap();
579 file.sync_all().unwrap();
580
581 let mut streamer =
582 IocpGridManager::new(&file_path).expect("IocpGridManager creation should succeed");
583
584 let misaligned = streamer.async_read_chunk(100);
586 assert!(misaligned.is_err(), "Misaligned offset must return error");
587
588 assert!(
590 streamer.async_read_chunk(0).is_ok(),
591 "Aligned read at 0 must succeed"
592 );
593 let completed = streamer.poll_completion();
594 assert!(completed.is_some(), "poll_completion should return buffer");
595 let buf = completed.unwrap();
596 assert_eq!(&buf[0..4], b"IOCP", "Read buffer must match written data");
597
598 std::fs::remove_file(&file_path).unwrap();
600 }
601
602 #[cfg(target_os = "linux")]
603 #[test]
604 fn test_iouring_offset_validation() {
605 let temp_dir = std::env::temp_dir();
607 let file_path = temp_dir.join("test_iouring.dat");
608
609 let mut file = File::create(&file_path).unwrap();
610 file.write_all(&vec![0u8; 8192]).unwrap();
611 file.sync_all().unwrap();
612
613 let mut manager = IoUringGridManager::new(&file_path).unwrap();
614
615 assert!(manager.async_read_chunk(4096).is_ok());
617
618 let result = manager.async_read_chunk(4095);
620 assert!(matches!(result, Err(IoError::MisalignedOffset { .. })));
621
622 std::fs::remove_file(&file_path).unwrap();
624 }
625}