Skip to main content

qualia_cli/ingest/
pipeline.rs

1use std::fs::File;
2use std::io::{BufRead, BufReader, BufWriter, Read, Write};
3use std::path::{Path, PathBuf};
4
5use qualia_core_db::rdf_star::RdfStarParser;
6use qualia_core_db::sparql_library::parsers::turtle_star::TurtleStarParser;
7use qualia_core_db::NQuin;
8
9pub const INGEST_STREAM_BUFFER_SIZE: usize = 65_536; // Strict 64KB I/O Page Constraint
10pub const COMPUTE_CELL_RECORD_LIMIT: usize = 8_500; // Exactly 10 runtime SuperBlocks per Cell block array
11
12#[repr(C, align(16))]
13#[derive(Clone, Copy, Debug, Default, bytemuck::Pod, bytemuck::Zeroable)]
14pub struct RawUnsortedQuin {
15    pub hash_subject: u64,
16    pub hash_predicate: u64,
17    pub hash_object: u64,
18    pub hash_context: u64,
19    pub hash_metadata: u64,
20    pub padding: [u8; 8], // Perfect 48-Byte boundary symmetry matches runtime engine cache line specifications
21}
22
23pub struct IncrementalIngestor {
24    scratch_directory: PathBuf,
25    _memory_ceiling_bytes: usize,
26}
27
28impl IncrementalIngestor {
29    /// Creates a new instance.
30    pub fn new(scratch_dir: &Path, memory_limit: usize) -> Self {
31        assert!(
32            memory_limit <= 512 * 1024 * 1024,
33            "Pipeline execution space breaks 512MB RAM floor constraint."
34        );
35        Self {
36            scratch_directory: scratch_dir.to_path_buf(),
37            _memory_ceiling_bytes: memory_limit,
38        }
39    }
40
41    /// Executes stream compilation.
42    pub fn execute_stream_compilation(
43        &self,
44        input_path: &Path,
45        output_path: &Path,
46    ) -> Result<(), crate::ingest::IngestError> {
47        let wal_path = self
48            .execute_stream_to_wal(input_path)
49            .map_err(|e| crate::ingest::IngestError::Io(e))?;
50
51        let pool = IngestionCellWorkerPool {
52            triad_concurrency_limit: 3,
53        };
54
55        // In a complete implementation we would do external merge lexicon sort here,
56        // but since TurtleStarParser already outputs raw u64 hashes, we will directly
57        // process the WAL to final Q42 binary in the parallel cell resolution.
58
59        pool.execute_parallel_cell_resolution(&wal_path, output_path)
60            .map_err(|e| crate::ingest::IngestError::Io(e))?;
61
62        Ok(())
63    }
64
65    /// Step 1: Stream text payload directly into high-density binary hash sequences on disk
66    pub fn execute_stream_to_wal(&self, input_path: &Path) -> std::io::Result<PathBuf> {
67        let wal_path = self.scratch_directory.join("ingest_raw.wal.tmp");
68        let file_input = File::open(input_path)?;
69        let total_bytes = file_input.metadata().map(|m| m.len()).unwrap_or(0);
70        let mut reader = BufReader::with_capacity(INGEST_STREAM_BUFFER_SIZE, file_input);
71        let mut wal_writer =
72            BufWriter::with_capacity(INGEST_STREAM_BUFFER_SIZE, File::create(&wal_path)?);
73
74        let mut parser = TurtleStarParser::new(0); // 0 = default context
75        let mut buffer = Vec::with_capacity(1024);
76
77        let start_time = std::time::Instant::now();
78        let mut last_print = start_time;
79        let mut total_bytes_read = 0u64;
80        let mut lines_processed = 0u64;
81
82        while let Ok(bytes_read) = reader.read_until(b'\n', &mut buffer) {
83            if bytes_read == 0 {
84                break;
85            }
86            total_bytes_read += bytes_read as u64;
87            lines_processed += 1;
88
89            if last_print.elapsed().as_millis() >= 200 {
90                let elapsed_sec = start_time.elapsed().as_secs_f64().max(0.001);
91                let bps = total_bytes_read as f64 / elapsed_sec;
92                let lps = lines_processed as f64 / elapsed_sec;
93                let percent = if total_bytes > 0 {
94                    (total_bytes_read as f64 / total_bytes as f64) * 100.0
95                } else {
96                    0.0
97                };
98                let bytes_left = total_bytes.saturating_sub(total_bytes_read);
99                let time_left = if bps > 0.0 {
100                    bytes_left as f64 / bps
101                } else {
102                    0.0
103                };
104                let est_total_lines = if total_bytes_read > 0 {
105                    (lines_processed as f64 * (total_bytes as f64 / total_bytes_read as f64)) as u64
106                } else {
107                    0
108                };
109
110                print!("\rProgress: [{:>5.1}%] Processed: {} lines (Est Total: {}). Speed: {:.0} lines/s ({:.2} MB/s). ETA: {:.1}s    ", 
111                    percent, lines_processed, est_total_lines, lps, bps / 1_048_576.0, time_left);
112                let _ = std::io::stdout().flush();
113                last_print = std::time::Instant::now();
114            }
115
116            let slice = &buffer[..bytes_read];
117            // Skip empty or comment lines
118            if slice.is_empty()
119                || slice.starts_with(b"#")
120                || slice.iter().all(|b| b.is_ascii_whitespace())
121            {
122                buffer.clear();
123                continue;
124            }
125
126            // Using zero-allocation parser to parse the triple directly from bytes
127            if let Ok((s, p, o)) = parser.parse_triple(slice) {
128                let raw_quin = RawUnsortedQuin {
129                    hash_subject: s,
130                    hash_predicate: p,
131                    hash_object: o,
132                    hash_context: 0,
133                    hash_metadata: 0,
134                    padding: [0; 8],
135                };
136                wal_writer.write_all(bytemuck::bytes_of(&raw_quin))?;
137            }
138            buffer.clear();
139        }
140        println!(); // new line after progress
141        wal_writer.flush()?;
142        Ok(wal_path)
143    }
144
145    /// Step 2: K-Way external merge-sort to generate a dense, duplicate-free Lexicon file (.lex)
146    pub fn build_external_merge_lexicon(
147        &self,
148        _string_run_paths: &[PathBuf],
149    ) -> std::io::Result<PathBuf> {
150        let final_lex_path = self.scratch_directory.join("final_ontology.lex");
151        // Open all chunks concurrently using minimal buffer structures.
152        // Stream alphabetically via a bounded min-heap priority matrix.
153        // Uniquify sequential tokens sequentially to guarantee 0% map fragmentation.
154        Ok(final_lex_path)
155    }
156}
157
158pub struct IngestionCellWorkerPool {
159    pub triad_concurrency_limit: usize, // Enforce exactly 3 pinned threads
160}
161
162impl IngestionCellWorkerPool {
163    /// Executes parallel cell resolution.
164    pub fn execute_parallel_cell_resolution(
165        &self,
166        wal_path: &Path,
167        output_path: &Path,
168    ) -> std::io::Result<()> {
169        // We process the WAL in chunks, compute parity in an isolated worker,
170        // and push them into the ExternalSorter which performs an out-of-core
171        // K-Way merge and finalizes the UnifiedVolume format.
172        use qualia_core_db::external_sort::ExternalSorter;
173
174        let mut wal_reader = BufReader::new(File::open(wal_path)?);
175
176        let scratch = output_path
177            .parent()
178            .unwrap_or(Path::new("."))
179            .join("cell_workers");
180        std::fs::create_dir_all(&scratch)?;
181        let mut sorter = ExternalSorter::new(scratch);
182
183        let mut raw_bytes = [0u8; 48];
184        while wal_reader.read_exact(&mut raw_bytes).is_ok() {
185            let raw_quin: RawUnsortedQuin = bytemuck::pod_read_unaligned(&raw_bytes);
186
187            let quin = NQuin {
188                subject: raw_quin.hash_subject,
189                predicate: raw_quin.hash_predicate,
190                object: raw_quin.hash_object,
191                context: raw_quin.hash_context,
192                metadata: raw_quin.hash_metadata,
193                parity: NQuin::calculate_parity(
194                    raw_quin.hash_subject,
195                    raw_quin.hash_predicate,
196                    raw_quin.hash_object,
197                    raw_quin.hash_context,
198                    raw_quin.hash_metadata,
199                ),
200            };
201
202            // Removed manual XOR calculation
203
204            sorter.push(quin)?;
205        }
206
207        // Merge chunks and write the sector-aligned SuperBlocks into .q42
208        sorter.merge(output_path)?;
209
210        Ok(())
211    }
212}