qualia_cli/ingest/
pipeline.rs1use std::fs::File;
2use std::io::{BufRead, BufReader, BufWriter, Read, Write};
3use std::path::{Path, PathBuf};
4
5use qualia_core_db::rdf_star::RdfStarParser;
6use qualia_core_db::sparql_library::parsers::turtle_star::TurtleStarParser;
7use qualia_core_db::NQuin;
8
9pub const INGEST_STREAM_BUFFER_SIZE: usize = 65_536; pub const COMPUTE_CELL_RECORD_LIMIT: usize = 8_500; #[repr(C, align(16))]
13#[derive(Clone, Copy, Debug, Default, bytemuck::Pod, bytemuck::Zeroable)]
14pub struct RawUnsortedQuin {
15 pub hash_subject: u64,
16 pub hash_predicate: u64,
17 pub hash_object: u64,
18 pub hash_context: u64,
19 pub hash_metadata: u64,
20 pub padding: [u8; 8], }
22
23pub struct IncrementalIngestor {
24 scratch_directory: PathBuf,
25 _memory_ceiling_bytes: usize,
26}
27
28impl IncrementalIngestor {
29 pub fn new(scratch_dir: &Path, memory_limit: usize) -> Self {
31 assert!(
32 memory_limit <= 512 * 1024 * 1024,
33 "Pipeline execution space breaks 512MB RAM floor constraint."
34 );
35 Self {
36 scratch_directory: scratch_dir.to_path_buf(),
37 _memory_ceiling_bytes: memory_limit,
38 }
39 }
40
41 pub fn execute_stream_compilation(
43 &self,
44 input_path: &Path,
45 output_path: &Path,
46 ) -> Result<(), crate::ingest::IngestError> {
47 let wal_path = self
48 .execute_stream_to_wal(input_path)
49 .map_err(|e| crate::ingest::IngestError::Io(e))?;
50
51 let pool = IngestionCellWorkerPool {
52 triad_concurrency_limit: 3,
53 };
54
55 pool.execute_parallel_cell_resolution(&wal_path, output_path)
60 .map_err(|e| crate::ingest::IngestError::Io(e))?;
61
62 Ok(())
63 }
64
65 pub fn execute_stream_to_wal(&self, input_path: &Path) -> std::io::Result<PathBuf> {
67 let wal_path = self.scratch_directory.join("ingest_raw.wal.tmp");
68 let file_input = File::open(input_path)?;
69 let total_bytes = file_input.metadata().map(|m| m.len()).unwrap_or(0);
70 let mut reader = BufReader::with_capacity(INGEST_STREAM_BUFFER_SIZE, file_input);
71 let mut wal_writer =
72 BufWriter::with_capacity(INGEST_STREAM_BUFFER_SIZE, File::create(&wal_path)?);
73
74 let mut parser = TurtleStarParser::new(0); let mut buffer = Vec::with_capacity(1024);
76
77 let start_time = std::time::Instant::now();
78 let mut last_print = start_time;
79 let mut total_bytes_read = 0u64;
80 let mut lines_processed = 0u64;
81
82 while let Ok(bytes_read) = reader.read_until(b'\n', &mut buffer) {
83 if bytes_read == 0 {
84 break;
85 }
86 total_bytes_read += bytes_read as u64;
87 lines_processed += 1;
88
89 if last_print.elapsed().as_millis() >= 200 {
90 let elapsed_sec = start_time.elapsed().as_secs_f64().max(0.001);
91 let bps = total_bytes_read as f64 / elapsed_sec;
92 let lps = lines_processed as f64 / elapsed_sec;
93 let percent = if total_bytes > 0 {
94 (total_bytes_read as f64 / total_bytes as f64) * 100.0
95 } else {
96 0.0
97 };
98 let bytes_left = total_bytes.saturating_sub(total_bytes_read);
99 let time_left = if bps > 0.0 {
100 bytes_left as f64 / bps
101 } else {
102 0.0
103 };
104 let est_total_lines = if total_bytes_read > 0 {
105 (lines_processed as f64 * (total_bytes as f64 / total_bytes_read as f64)) as u64
106 } else {
107 0
108 };
109
110 print!("\rProgress: [{:>5.1}%] Processed: {} lines (Est Total: {}). Speed: {:.0} lines/s ({:.2} MB/s). ETA: {:.1}s ",
111 percent, lines_processed, est_total_lines, lps, bps / 1_048_576.0, time_left);
112 let _ = std::io::stdout().flush();
113 last_print = std::time::Instant::now();
114 }
115
116 let slice = &buffer[..bytes_read];
117 if slice.is_empty()
119 || slice.starts_with(b"#")
120 || slice.iter().all(|b| b.is_ascii_whitespace())
121 {
122 buffer.clear();
123 continue;
124 }
125
126 if let Ok((s, p, o)) = parser.parse_triple(slice) {
128 let raw_quin = RawUnsortedQuin {
129 hash_subject: s,
130 hash_predicate: p,
131 hash_object: o,
132 hash_context: 0,
133 hash_metadata: 0,
134 padding: [0; 8],
135 };
136 wal_writer.write_all(bytemuck::bytes_of(&raw_quin))?;
137 }
138 buffer.clear();
139 }
140 println!(); wal_writer.flush()?;
142 Ok(wal_path)
143 }
144
145 pub fn build_external_merge_lexicon(
147 &self,
148 _string_run_paths: &[PathBuf],
149 ) -> std::io::Result<PathBuf> {
150 let final_lex_path = self.scratch_directory.join("final_ontology.lex");
151 Ok(final_lex_path)
155 }
156}
157
158pub struct IngestionCellWorkerPool {
159 pub triad_concurrency_limit: usize, }
161
162impl IngestionCellWorkerPool {
163 pub fn execute_parallel_cell_resolution(
165 &self,
166 wal_path: &Path,
167 output_path: &Path,
168 ) -> std::io::Result<()> {
169 use qualia_core_db::external_sort::ExternalSorter;
173
174 let mut wal_reader = BufReader::new(File::open(wal_path)?);
175
176 let scratch = output_path
177 .parent()
178 .unwrap_or(Path::new("."))
179 .join("cell_workers");
180 std::fs::create_dir_all(&scratch)?;
181 let mut sorter = ExternalSorter::new(scratch);
182
183 let mut raw_bytes = [0u8; 48];
184 while wal_reader.read_exact(&mut raw_bytes).is_ok() {
185 let raw_quin: RawUnsortedQuin = bytemuck::pod_read_unaligned(&raw_bytes);
186
187 let quin = NQuin {
188 subject: raw_quin.hash_subject,
189 predicate: raw_quin.hash_predicate,
190 object: raw_quin.hash_object,
191 context: raw_quin.hash_context,
192 metadata: raw_quin.hash_metadata,
193 parity: NQuin::calculate_parity(
194 raw_quin.hash_subject,
195 raw_quin.hash_predicate,
196 raw_quin.hash_object,
197 raw_quin.hash_context,
198 raw_quin.hash_metadata,
199 ),
200 };
201
202 sorter.push(quin)?;
205 }
206
207 sorter.merge(output_path)?;
209
210 Ok(())
211 }
212}