1use std::collections::HashMap;
8use std::io::{Read, Write};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex};
12
13use qualia_core_db::{
14 ingest,
15 resource_catalog::{OntologyResource, ResourceCatalog},
16 wal::WriteAheadLog,
17};
18use serde::Serialize;
19use sha2::{Digest, Sha256};
20
21use crate::state::ProgressPayload;
22
23pub struct ImportProgressCtx {
25 pub id: String,
26 pub handles: Arc<Mutex<HashMap<String, Arc<AtomicBool>>>>,
27 pub active_downloads: Arc<Mutex<HashMap<String, ProgressPayload>>>,
28 pub download_events: tokio::sync::broadcast::Sender<ProgressPayload>,
29}
30
31impl ImportProgressCtx {
32 pub fn emit(&self, payload: ProgressPayload) {
34 let _ = self.download_events.send(payload.clone());
35 self.active_downloads
36 .lock()
37 .unwrap()
38 .insert(self.id.clone(), payload);
39 }
40
41 pub fn clear(&self) {
43 self.handles.lock().unwrap().remove(&self.id);
44 self.active_downloads.lock().unwrap().remove(&self.id);
45 }
46}
47
48#[derive(Debug)]
49pub enum ImportError {
50 NotFound(String),
51 NoDownloadUrl(String),
52 Download(String),
53 Ingest(String),
54 Wal(String),
55 Io(std::io::Error),
56 Json(serde_json::Error),
57}
58
59impl std::fmt::Display for ImportError {
60 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61 match self {
62 ImportError::NotFound(id) => write!(f, "Ontology not found in catalog: {id}"),
63 ImportError::NoDownloadUrl(id) => write!(f, "No download URL for ontology: {id}"),
64 ImportError::Download(e) => write!(f, "Download failed: {e}"),
65 ImportError::Ingest(e) => write!(f, "Ingest failed: {e}"),
66 ImportError::Wal(e) => write!(f, "WAL write failed: {e}"),
67 ImportError::Io(e) => write!(f, "IO error: {e}"),
68 ImportError::Json(e) => write!(f, "JSON error: {e}"),
69 }
70 }
71}
72
73impl From<std::io::Error> for ImportError {
74 fn from(e: std::io::Error) -> Self {
75 ImportError::Io(e)
76 }
77}
78
79impl From<serde_json::Error> for ImportError {
80 fn from(e: serde_json::Error) -> Self {
81 ImportError::Json(e)
82 }
83}
84
85#[derive(Debug, Clone, Serialize)]
86pub struct OntologyImportResult {
87 pub ontology_id: String,
88 pub source_path: String,
89 pub q42_path: String,
90 pub wal_path: String,
91 pub quin_count: u64,
92 pub catalog_quins: usize,
93 pub sha256: String,
94 pub imported_at: u64,
95 pub source_removed: bool,
97}
98
99#[derive(Serialize)]
100struct OntologyMetaSidecar {
101 ontology_id: String,
102 quin_count: u64,
103 sha256: String,
104 imported_at: u64,
105 source_path: String,
106 q42_path: String,
107}
108
109pub fn index_dir(storage_root: &Path) -> PathBuf {
111 storage_root.join("Index")
112}
113
114fn ontology_source_filename(ont: &OntologyResource) -> String {
115 ont.download
116 .local_filename()
117 .unwrap_or_else(|| format!("{}.{}", ont.id, ont.format))
118}
119
120fn unix_now() -> u64 {
121 std::time::SystemTime::now()
122 .duration_since(std::time::UNIX_EPOCH)
123 .unwrap_or_default()
124 .as_secs()
125}
126
127fn sha256_file(path: &Path) -> Result<String, std::io::Error> {
128 let mut file = std::fs::File::open(path)?;
129 let mut hasher = Sha256::new();
130 let mut buf = [0u8; 65_536];
131 loop {
132 let n = file.read(&mut buf)?;
133 if n == 0 {
134 break;
135 }
136 hasher.update(&buf[..n]);
137 }
138 Ok(hex::encode(hasher.finalize()))
139}
140
141fn write_meta_sidecar(
142 ontology_id: &str,
143 q42_path: &Path,
144 source_path: &Path,
145 quin_count: u64,
146 sha256: &str,
147 imported_at: u64,
148) -> Result<(), ImportError> {
149 let meta_path = q42_path.with_extension("q42.meta.json");
150 let meta = OntologyMetaSidecar {
151 ontology_id: ontology_id.to_string(),
152 quin_count,
153 sha256: sha256.to_string(),
154 imported_at,
155 source_path: source_path.to_string_lossy().into_owned(),
156 q42_path: q42_path.to_string_lossy().into_owned(),
157 };
158 let json = serde_json::to_string_pretty(&meta)?;
159 std::fs::write(&meta_path, json)?;
160 Ok(())
161}
162
163pub async fn stream_download(url: &str, dest: &Path) -> Result<(), String> {
165 stream_download_with_progress(url, dest, None)
166 .await
167 .map_err(|e| e.to_string())
168}
169
170pub async fn stream_download_with_progress(
172 url: &str,
173 dest: &Path,
174 progress: Option<&ImportProgressCtx>,
175) -> Result<(), ImportError> {
176 let client = reqwest::Client::new();
177 let response = client
178 .get(url)
179 .header(
180 "User-Agent",
181 concat!("qualiaDB-client/", env!("CARGO_PKG_VERSION")),
182 )
183 .send()
184 .await
185 .map_err(|e| ImportError::Download(e.to_string()))?
186 .error_for_status()
187 .map_err(|e| ImportError::Download(e.to_string()))?;
188
189 let total_bytes = response.content_length().unwrap_or(0);
190 if let Some(ctx) = progress {
191 ctx.emit(ProgressPayload {
192 id: ctx.id.clone(),
193 progress: 0.0,
194 downloaded_bytes: 0,
195 total_bytes,
196 speed_kbps: 0.0,
197 status: "downloading".to_string(),
198 });
199 }
200
201 if let Some(parent) = dest.parent() {
202 std::fs::create_dir_all(parent)?;
203 }
204
205 let mut file = std::fs::File::create(dest).map_err(|e| {
206 ImportError::Io(std::io::Error::new(
207 e.kind(),
208 format!("Cannot create {}: {}", dest.display(), e),
209 ))
210 })?;
211
212 let mut stream = response.bytes_stream();
213 use futures_util::StreamExt;
214 let mut downloaded: u64 = 0;
215 let mut last_report = std::time::Instant::now();
216 let mut last_downloaded: u64 = 0;
217
218 while let Some(chunk) = stream.next().await {
219 if let Some(ctx) = progress {
220 if let Some(flag) = ctx.handles.lock().unwrap().get(&ctx.id) {
221 if flag.load(Ordering::Relaxed) {
222 let _ = std::fs::remove_file(dest);
223 ctx.emit(ProgressPayload {
224 id: ctx.id.clone(),
225 progress: 0.0,
226 downloaded_bytes: downloaded,
227 total_bytes,
228 speed_kbps: 0.0,
229 status: "cancelled".to_string(),
230 });
231 ctx.clear();
232 return Err(ImportError::Download("Cancelled".to_string()));
233 }
234 }
235 }
236
237 let chunk = chunk.map_err(|e| ImportError::Download(e.to_string()))?;
238 file.write_all(&chunk)?;
239 downloaded += chunk.len() as u64;
240
241 if let Some(ctx) = progress {
242 let now = std::time::Instant::now();
243 if now.duration_since(last_report).as_millis() >= 200 {
244 let elapsed = now.duration_since(last_report).as_secs_f64().max(0.001);
245 let speed_kbps = ((downloaded - last_downloaded) as f64 / 1024.0) / elapsed;
246 let progress_pct = if total_bytes > 0 {
247 (downloaded as f64 / total_bytes as f64) * 100.0
248 } else {
249 0.0
250 };
251 ctx.emit(ProgressPayload {
252 id: ctx.id.clone(),
253 progress: progress_pct,
254 downloaded_bytes: downloaded,
255 total_bytes,
256 speed_kbps,
257 status: "downloading".to_string(),
258 });
259 last_report = now;
260 last_downloaded = downloaded;
261 }
262 }
263 }
264
265 Ok(())
266}
267
268pub fn ingest_local_rdf(
270 source_path: &Path,
271 ontology_id: &str,
272 storage_root: &Path,
273 ont: Option<&OntologyResource>,
274) -> Result<u64, ImportError> {
275 ingest_local_rdf_with_progress(source_path, ontology_id, storage_root, ont, None)
276}
277
278pub fn ingest_local_rdf_with_progress(
281 source_path: &Path,
282 ontology_id: &str,
283 storage_root: &Path,
284 ont: Option<&OntologyResource>,
285 progress: Option<&ImportProgressCtx>,
286) -> Result<u64, ImportError> {
287 let index = index_dir(storage_root);
288 std::fs::create_dir_all(&index)?;
289
290 let q42_path = index.join(format!("{ontology_id}.q42"));
291 let in_str = source_path.to_string_lossy().into_owned();
292 let out_str = q42_path.to_string_lossy().into_owned();
293
294 let report = match progress {
295 Some(ctx) => {
296 let id = ctx.id.clone();
297 let events = ctx.download_events.clone();
298 let active = ctx.active_downloads.clone();
299 ingest::IngestReport::new(1, move |snap| {
300 let payload = ProgressPayload {
301 id: id.clone(),
302 progress: snap.pct,
303 downloaded_bytes: snap.bytes_read,
304 total_bytes: snap.source_bytes,
305 speed_kbps: snap.mb_per_sec * 1024.0,
306 status: format!("ingest:{}", snap.format_text(1)),
307 };
308 let _ = events.send(payload.clone());
309 if let Ok(mut map) = active.lock() {
310 map.insert(id.clone(), payload);
311 }
312 })
313 }
314 None => ingest::IngestReport::silent(),
315 };
316
317 let quin_count = ingest::streaming_import_rdf_with_report(
318 &in_str,
319 &out_str,
320 ingest::IngestMode::Complete,
321 None,
322 report,
323 )
324 .map_err(|e| ImportError::Ingest(e.to_string()))?;
325
326 let imported_at = unix_now();
327 let sha256 = sha256_file(&q42_path)?;
328
329 if let Some(ont) = ont {
330 append_ontology_wal(storage_root, ont, &q42_path, imported_at)?;
331 }
332
333 write_meta_sidecar(
334 ontology_id,
335 &q42_path,
336 source_path,
337 quin_count,
338 &sha256,
339 imported_at,
340 )?;
341
342 Ok(quin_count)
343}
344
345fn append_ontology_wal(
346 storage_root: &Path,
347 ont: &OntologyResource,
348 q42_path: &Path,
349 timestamp: u64,
350) -> Result<usize, ImportError> {
351 let wal_path = index_dir(storage_root).join("ontologies.wal");
352 let mut wal = WriteAheadLog::open(&wal_path)
353 .map_err(|e| ImportError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
354
355 let prov = ont.provenance_quin(timestamp, &q42_path.to_string_lossy());
356 wal.append_mutation(&prov)
357 .map_err(|e| ImportError::Wal(e.to_string()))?;
358
359 let catalog_quins = ont.to_quins();
360 for q in &catalog_quins {
361 wal.append_mutation(q)
362 .map_err(|e| ImportError::Wal(e.to_string()))?;
363 }
364
365 Ok(catalog_quins.len())
366}
367
368pub async fn import_catalog_ontology(
370 catalog: &ResourceCatalog,
371 id: &str,
372 storage_root: &Path,
373) -> Result<OntologyImportResult, ImportError> {
374 import_catalog_ontology_with_options(catalog, id, storage_root, None, true).await
375}
376
377pub async fn import_catalog_ontology_with_options(
379 catalog: &ResourceCatalog,
380 id: &str,
381 storage_root: &Path,
382 progress: Option<&ImportProgressCtx>,
383 delete_source_after_ingest: bool,
384) -> Result<OntologyImportResult, ImportError> {
385 let ont = catalog
386 .find_ontology(id)
387 .ok_or_else(|| ImportError::NotFound(id.to_string()))?;
388
389 if let Some(source_path) = crate::bundled_ontologies::resolve_bundled_ontology_source(id) {
390 if let Some(ctx) = progress {
391 ctx.emit(ProgressPayload {
392 id: ctx.id.clone(),
393 progress: 100.0,
394 downloaded_bytes: 0,
395 total_bytes: 0,
396 speed_kbps: 0.0,
397 status: "processing".to_string(),
398 });
399 }
400
401 let index = index_dir(storage_root);
402 std::fs::create_dir_all(&index)?;
403 let quin_count =
404 ingest_local_rdf_with_progress(&source_path, id, storage_root, Some(ont), progress)?;
405 let q42_path = index.join(format!("{id}.q42"));
406 let wal_path = index.join("ontologies.wal");
407 let catalog_quins = ont.to_quins().len();
408 let sha256 = sha256_file(&q42_path)?;
409 let imported_at = unix_now();
410
411 if let Some(ctx) = progress {
412 ctx.emit(ProgressPayload {
413 id: ctx.id.clone(),
414 progress: 100.0,
415 downloaded_bytes: 0,
416 total_bytes: 0,
417 speed_kbps: 0.0,
418 status: "complete".to_string(),
419 });
420 ctx.clear();
421 }
422
423 return Ok(OntologyImportResult {
424 ontology_id: id.to_string(),
425 source_path: source_path.to_string_lossy().into_owned(),
426 q42_path: q42_path.to_string_lossy().into_owned(),
427 wal_path: wal_path.to_string_lossy().into_owned(),
428 quin_count,
429 catalog_quins,
430 sha256,
431 imported_at,
432 source_removed: false,
433 });
434 }
435
436 let url = ont
437 .download
438 .resolved_url()
439 .ok_or_else(|| ImportError::NoDownloadUrl(id.to_string()))?;
440
441 let index = index_dir(storage_root);
442 std::fs::create_dir_all(&index)?;
443
444 let filename = ontology_source_filename(ont);
445 let source_path = index.join(&filename);
446
447 stream_download_with_progress(&url, &source_path, progress).await?;
448
449 if let Some(ctx) = progress {
450 ctx.emit(ProgressPayload {
451 id: ctx.id.clone(),
452 progress: 100.0,
453 downloaded_bytes: std::fs::metadata(&source_path)
454 .map(|m| m.len())
455 .unwrap_or(0),
456 total_bytes: std::fs::metadata(&source_path)
457 .map(|m| m.len())
458 .unwrap_or(0),
459 speed_kbps: 0.0,
460 status: "processing".to_string(),
461 });
462 }
463
464 let quin_count =
465 ingest_local_rdf_with_progress(&source_path, id, storage_root, Some(ont), progress)?;
466 let q42_path = index.join(format!("{id}.q42"));
467 let wal_path = index.join("ontologies.wal");
468 let catalog_quins = ont.to_quins().len();
469 let sha256 = sha256_file(&q42_path)?;
470 let imported_at = unix_now();
471 let source_path_str = source_path.to_string_lossy().into_owned();
472
473 let mut source_removed = false;
474 if delete_source_after_ingest && source_path.is_file() {
475 if std::fs::remove_file(&source_path).is_ok() {
476 source_removed = true;
477 }
478 }
479
480 if let Some(ctx) = progress {
481 ctx.emit(ProgressPayload {
482 id: ctx.id.clone(),
483 progress: 100.0,
484 downloaded_bytes: 0,
485 total_bytes: 0,
486 speed_kbps: 0.0,
487 status: "complete".to_string(),
488 });
489 ctx.clear();
490 }
491
492 Ok(OntologyImportResult {
493 ontology_id: id.to_string(),
494 source_path: source_path_str,
495 q42_path: q42_path.to_string_lossy().into_owned(),
496 wal_path: wal_path.to_string_lossy().into_owned(),
497 quin_count,
498 catalog_quins,
499 sha256,
500 imported_at,
501 source_removed,
502 })
503}