Skip to main content

qualia_core_db/medical/
dicom_ingest.rs

1//! Core 3 DICOM split-ingestion pipeline.
2//!
3//! Pixel payloads stream to a memory-mapped blob store; semantic metadata becomes
4//! 48-byte Quins appended to the conduct WAL. Core 1 never blocks on I/O — jobs are
5//! handed to a dedicated Swarm worker via a lock-free channel.
6
7use crate::dicom::{decode_blob_pointer, encode_blob_pointer, pack_volume_metadata, DicomMetadata};
8use crate::q_hash;
9use crate::NQuin;
10use crossbeam_channel::{bounded, Receiver, Sender};
11use memmap2::Mmap;
12use std::cell::UnsafeCell;
13use std::fs::{File, OpenOptions};
14use std::io::{Seek, SeekFrom, Write};
15use std::path::{Path, PathBuf};
16use std::sync::atomic::{AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering};
17use std::sync::OnceLock;
18use std::thread;
19
20pub const BLOB_FILE_NAME: &str = "dicom_pixels.blob";
21pub const MAX_SERIES_RECORDS: usize = 64;
22pub const JOB_PENDING: u8 = 0;
23pub const JOB_COMPLETE: u8 = 1;
24pub const JOB_FAILED: u8 = 2;
25
26/// Published series record — single-writer (Core 3), lock-free readers (Core 1).
27#[derive(Debug, Clone, Copy, Default)]
28pub struct DicomSeriesRecord {
29    pub patient_did_hash: u64,
30    pub series_hash: u64,
31    pub blob_offset: u64,
32    pub blob_length: u32,
33    pub rows: u16,
34    pub cols: u16,
35    pub organ_hash: u64,
36    pub pixel_pointer_quin_parity: u64,
37}
38
39struct IngestJob {
40    job_id: u64,
41    source_path: PathBuf,
42    patient_did_hash: u64,
43    storage_root: PathBuf,
44}
45
46struct JobSlot {
47    status: AtomicU8,
48    series_hash: AtomicU64,
49    blob_offset: AtomicU64,
50    blob_length: AtomicU32,
51}
52
53impl JobSlot {
54    const fn new() -> Self {
55        Self {
56            status: AtomicU8::new(JOB_PENDING),
57            series_hash: AtomicU64::new(0),
58            blob_offset: AtomicU64::new(0),
59            blob_length: AtomicU32::new(0),
60        }
61    }
62}
63
64static IO_TX: OnceLock<Sender<IngestJob>> = OnceLock::new();
65static NEXT_JOB_ID: AtomicU64 = AtomicU64::new(1);
66static JOB_SLOTS: [JobSlot; MAX_SERIES_RECORDS] = [const { JobSlot::new() }; MAX_SERIES_RECORDS];
67const EMPTY_SERIES_RECORD: DicomSeriesRecord = DicomSeriesRecord {
68    patient_did_hash: 0,
69    series_hash: 0,
70    blob_offset: 0,
71    blob_length: 0,
72    rows: 0,
73    cols: 0,
74    organ_hash: 0,
75    pixel_pointer_quin_parity: 0,
76};
77struct SyncSeriesRegistry([UnsafeCell<DicomSeriesRecord>; MAX_SERIES_RECORDS]);
78// SAFETY: Core 3 is the sole writer; readers bound indices via SERIES_COUNT Acquire.
79unsafe impl Sync for SyncSeriesRegistry {}
80
81static SERIES_RECORDS: SyncSeriesRegistry =
82    SyncSeriesRegistry([const { UnsafeCell::new(EMPTY_SERIES_RECORD) }; MAX_SERIES_RECORDS]);
83static SERIES_COUNT: AtomicUsize = AtomicUsize::new(0);
84
85#[derive(Debug, PartialEq, Eq)]
86pub enum DicomIngestError {
87    WorkerUnavailable,
88    Io(String),
89    Parse(String),
90    RegistryFull,
91    JobNotFound,
92    BlobRead(String),
93}
94
95impl std::fmt::Display for DicomIngestError {
96    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
97        match self {
98            Self::WorkerUnavailable => write!(f, "Core 3 DICOM worker not initialized"),
99            Self::Io(msg) => write!(f, "DICOM ingest IO: {msg}"),
100            Self::Parse(msg) => write!(f, "DICOM ingest parse: {msg}"),
101            Self::RegistryFull => write!(f, "DICOM series registry full"),
102            Self::JobNotFound => write!(f, "DICOM ingest job not found"),
103            Self::BlobRead(msg) => write!(f, "DICOM blob read: {msg}"),
104        }
105    }
106}
107
108impl std::error::Error for DicomIngestError {}
109
110/// Append-only pixel blob store (Core 3 writer).
111pub struct DicomBlobStore {
112    file: File,
113    write_offset: u64,
114}
115
116impl DicomBlobStore {
117    /// Opens/creates the pixel blob store at `path`.
118    pub fn open(path: &Path) -> Result<Self, std::io::Error> {
119        let file = OpenOptions::new()
120            .read(true)
121            .write(true)
122            .create(true)
123            .open(path)?;
124        let len = file.metadata()?.len();
125        Ok(Self {
126            file,
127            write_offset: len,
128        })
129    }
130
131    /// Appends pixel bytes; returns the byte offset.
132    pub fn append_pixels(&mut self, pixels: &[u8]) -> Result<u64, std::io::Error> {
133        let offset = self.write_offset;
134        self.file.seek(SeekFrom::Start(offset))?;
135        self.file.write_all(pixels)?;
136        self.file.sync_data()?;
137        self.write_offset = offset.saturating_add(pixels.len() as u64);
138        Ok(offset)
139    }
140
141    /// Memory-maps the store at `path`.
142    pub fn path_mmap(&self) -> Result<Mmap, std::io::Error> {
143        unsafe { memmap2::MmapOptions::new().map(&self.file) }
144    }
145}
146
147/// Zero-copy slice reader over the blob file (Core 1 / FRB boundary).
148pub struct DicomBlobReader {
149    mmap: Mmap,
150}
151
152impl DicomBlobReader {
153    /// Opens a DICOM blob reader at `path`.
154    pub fn open(path: &Path) -> Result<Self, DicomIngestError> {
155        let file = File::open(path).map_err(|e| DicomIngestError::Io(e.to_string()))?;
156        let mmap = unsafe { memmap2::MmapOptions::new().map(&file) }
157            .map_err(|e| DicomIngestError::Io(e.to_string()))?;
158        Ok(Self { mmap })
159    }
160
161    #[inline]
162    /// `DicomBlobReader::slice`.
163    pub fn slice(&self, offset: usize, length: usize) -> Result<&[u8], DicomIngestError> {
164        let end = offset
165            .checked_add(length)
166            .ok_or_else(|| DicomIngestError::BlobRead("overflow".into()))?;
167        if end > self.mmap.len() {
168            return Err(DicomIngestError::BlobRead("range past blob end".into()));
169        }
170        Ok(&self.mmap[offset..end])
171    }
172}
173
174fn imaging_dir(storage_root: &Path) -> PathBuf {
175    storage_root.join("Imaging")
176}
177
178pub(super) fn blob_path(storage_root: &Path) -> PathBuf {
179    imaging_dir(storage_root).join(BLOB_FILE_NAME)
180}
181
182pub(super) fn wal_path(storage_root: &Path) -> PathBuf {
183    storage_root.join("qualia_global.wal")
184}
185
186pub(super) fn hash_series_uid(uid: &str) -> u64 {
187    if uid.is_empty() {
188        return 0;
189    }
190    q_hash(uid)
191}
192
193pub(super) fn infer_organ_hash(meta: &DicomMetadata) -> u64 {
194    if let Some(organ) =
195        crate::dicom::infer_organ_from_metadata(meta, &crate::dicom::default_organ_matchers())
196    {
197        q_hash(&organ)
198    } else {
199        0
200    }
201}
202
203pub(super) fn compile_semantic_quins(
204    meta: &DicomMetadata,
205    patient_did_hash: u64,
206    series_hash: u64,
207    blob_offset: u64,
208    blob_length: u32,
209    out: &mut [NQuin; 6],
210) -> usize {
211    let study_ctx = q_hash("q42:imagingStudy");
212    let modality_hash = q_hash(&meta.modality);
213    let pred_has_series = q_hash("q42:hasDicomSeries");
214    let pred_modality = q_hash("q42:hasModality");
215    let pred_pixel_ptr = q_hash("q42:pixelBlobPointer");
216    let pred_body_part = q_hash("q42:bodyPartExamined");
217
218    let mut count = 0usize;
219
220    let mut q0 = NQuin::default();
221    q0.subject = patient_did_hash;
222    q0.predicate = pred_has_series;
223    q0.object = series_hash;
224    q0.context = study_ctx;
225    q0.metadata = pack_volume_metadata(meta.rows, meta.columns, blob_length);
226    q0.parity = q0.subject ^ q0.predicate ^ q0.object ^ q0.context;
227    out[count] = q0;
228    count += 1;
229
230    let mut q1 = NQuin::default();
231    q1.subject = series_hash;
232    q1.predicate = pred_modality;
233    q1.object = modality_hash;
234    q1.context = study_ctx;
235    q1.parity = q1.subject ^ q1.predicate ^ q1.object ^ q1.context;
236    out[count] = q1;
237    count += 1;
238
239    if !meta.body_part_examined.is_empty() {
240        let mut q2 = NQuin::default();
241        q2.subject = series_hash;
242        q2.predicate = pred_body_part;
243        q2.object = q_hash(&meta.body_part_examined);
244        q2.context = study_ctx;
245        q2.parity = q2.subject ^ q2.predicate ^ q2.object ^ q2.context;
246        out[count] = q2;
247        count += 1;
248    }
249
250    let mut q3 = NQuin::default();
251    q3.subject = series_hash;
252    q3.predicate = pred_pixel_ptr;
253    q3.object = encode_blob_pointer(blob_offset);
254    q3.context = study_ctx;
255    q3.metadata = pack_volume_metadata(meta.rows, meta.columns, blob_length);
256    q3.parity = q3.subject ^ q3.predicate ^ q3.object ^ q3.context;
257    out[count] = q3;
258    count += 1;
259
260    let _ = decode_blob_pointer(q3.object);
261    count
262}
263
264pub(super) fn publish_series_record(record: DicomSeriesRecord) -> Result<usize, DicomIngestError> {
265    let idx = SERIES_COUNT.load(Ordering::Acquire);
266    if idx >= MAX_SERIES_RECORDS {
267        return Err(DicomIngestError::RegistryFull);
268    }
269    // SAFETY: single Core-3 writer; readers use Acquire on SERIES_COUNT before indexing.
270    unsafe {
271        *SERIES_RECORDS.0[idx].get() = record;
272    }
273    SERIES_COUNT.store(idx + 1, Ordering::Release);
274    Ok(idx)
275}
276
277fn run_split_ingest(job: IngestJob) {
278    let slot_idx = ((job.job_id as usize) - 1) % MAX_SERIES_RECORDS;
279    let slot = &JOB_SLOTS[slot_idx];
280
281    let result = super::dicom_source::split_ingest_sync_with_source(
282        &job.storage_root,
283        &job.source_path,
284        job.patient_did_hash,
285        super::dicom_source::DicomSourceBudget::default(),
286    )
287    .map(|ingested| ingested.record);
288
289    match result {
290        Ok(record) => {
291            slot.series_hash
292                .store(record.series_hash, Ordering::Release);
293            slot.blob_offset
294                .store(record.blob_offset, Ordering::Release);
295            slot.blob_length
296                .store(record.blob_length, Ordering::Release);
297            slot.status.store(JOB_COMPLETE, Ordering::Release);
298        }
299        Err(_) => {
300            slot.status.store(JOB_FAILED, Ordering::Release);
301        }
302    }
303}
304
305/// Pin the Core 3 DICOM Swarm worker (idempotent).
306pub fn init_core3_dicom_worker(storage_root: PathBuf) {
307    if IO_TX.get().is_some() {
308        return;
309    }
310
311    let (tx, rx): (Sender<IngestJob>, Receiver<IngestJob>) = bounded(32);
312    let _ = IO_TX.set(tx);
313
314    thread::Builder::new()
315        .name("qualia-core3-dicom".into())
316        .spawn(move || {
317            while let Ok(job) = rx.recv() {
318                run_split_ingest(job);
319            }
320        })
321        .expect("spawn Core 3 DICOM worker");
322
323    let _ = storage_root;
324}
325
326fn job_sender() -> Result<&'static Sender<IngestJob>, DicomIngestError> {
327    IO_TX.get().ok_or(DicomIngestError::WorkerUnavailable)
328}
329
330/// Submit a `.dcm` path to Core 3; returns a lock-free job id immediately.
331pub fn submit_dicom_ingest(
332    storage_root: &Path,
333    source_path: &Path,
334    patient_did_hash: u64,
335) -> Result<u64, DicomIngestError> {
336    init_core3_dicom_worker(storage_root.to_path_buf());
337    let job_id = NEXT_JOB_ID.fetch_add(1, Ordering::Relaxed);
338    let slot_idx = ((job_id as usize) - 1) % MAX_SERIES_RECORDS;
339    JOB_SLOTS[slot_idx]
340        .status
341        .store(JOB_PENDING, Ordering::Release);
342
343    job_sender()?
344        .send(IngestJob {
345            job_id,
346            source_path: source_path.to_path_buf(),
347            patient_did_hash,
348            storage_root: storage_root.to_path_buf(),
349        })
350        .map_err(|e| DicomIngestError::Io(e.to_string()))?;
351
352    Ok(job_id)
353}
354
355/// Poll job completion without blocking Core 1.
356pub fn dicom_ingest_status(job_id: u64) -> u8 {
357    let slot_idx = ((job_id as usize) - 1) % MAX_SERIES_RECORDS;
358    JOB_SLOTS[slot_idx].status.load(Ordering::Acquire)
359}
360
361/// Synchronous split-ingest for tests and CLI (runs on caller thread).
362/// Original Part-10 bytes are retained under the default 256 MiB source budget.
363/// Use `medical::dicom_source::split_ingest_sync_with_source` for explicit admission
364/// and the additive source result; `source_for_record` resolves legacy API records.
365pub fn split_ingest_sync(
366    storage_root: &Path,
367    source_path: &Path,
368    patient_did_hash: u64,
369) -> Result<DicomSeriesRecord, DicomIngestError> {
370    super::dicom_source::split_ingest_sync_with_source(
371        storage_root,
372        source_path,
373        patient_did_hash,
374        super::dicom_source::DicomSourceBudget::default(),
375    )
376    .map(|ingested| ingested.record)
377}
378
379#[cfg(test)]
380pub(super) static INGEST_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
381
382#[cfg(test)]
383/// `DicomBlobReader::reset_ingest_registry_for_tests`.
384pub fn reset_ingest_registry_for_tests() {
385    SERIES_COUNT.store(0, Ordering::Release);
386}
387
388#[allow(dead_code)]
389fn split_ingest_sync_via_worker(
390    storage_root: &Path,
391    source_path: &Path,
392    patient_did_hash: u64,
393) -> Result<DicomSeriesRecord, DicomIngestError> {
394    init_core3_dicom_worker(storage_root.to_path_buf());
395    let job_id = NEXT_JOB_ID.fetch_add(1, Ordering::Relaxed);
396    run_split_ingest(IngestJob {
397        job_id,
398        source_path: source_path.to_path_buf(),
399        patient_did_hash,
400        storage_root: storage_root.to_path_buf(),
401    });
402    if dicom_ingest_status(job_id) != JOB_COMPLETE {
403        return Err(DicomIngestError::Parse("split ingest worker failed".into()));
404    }
405    let idx = SERIES_COUNT.load(Ordering::Acquire);
406    if idx == 0 {
407        return Err(DicomIngestError::JobNotFound);
408    }
409    Ok(unsafe { *SERIES_RECORDS.0[idx - 1].get() })
410}
411
412/// Copies up to `out.len()` series records into `out`; returns count copied.
413pub fn series_records_snapshot(out: &mut [DicomSeriesRecord]) -> usize {
414    let count = SERIES_COUNT.load(Ordering::Acquire).min(MAX_SERIES_RECORDS);
415    for (i, slot) in out.iter_mut().enumerate().take(count) {
416        // SAFETY: reader synchronizes via Acquire load on SERIES_COUNT.
417        *slot = unsafe { *SERIES_RECORDS.0[i].get() };
418    }
419    count
420}
421
422/// Finds the series record for `(patient_did_hash, series_hash)`.
423pub fn find_series_record(patient_did_hash: u64, series_hash: u64) -> Option<DicomSeriesRecord> {
424    let count = SERIES_COUNT.load(Ordering::Acquire);
425    for i in 0..count.min(MAX_SERIES_RECORDS) {
426        let record = unsafe { *SERIES_RECORDS.0[i].get() };
427        if record.patient_did_hash == patient_did_hash && record.series_hash == series_hash {
428            return Some(record);
429        }
430    }
431    None
432}
433
434/// Read a pixel payload slice from the blob store (single copy at FRB boundary).
435pub fn read_volume_bytes(
436    storage_root: &Path,
437    record: &DicomSeriesRecord,
438) -> Result<Vec<u8>, DicomIngestError> {
439    let reader = DicomBlobReader::open(&blob_path(storage_root))?;
440    Ok(reader
441        .slice(record.blob_offset as usize, record.blob_length as usize)?
442        .to_vec())
443}
444
445#[cfg(test)]
446#[path = "dicom_ingest_tests.rs"]
447mod tests;