Skip to main content

qualia_client_core/
resource_import.rs

1//! Catalog ontology download → `.q42` import pipeline (in-process, no subprocess).
2//!
3//! Mirrors `qualia-cli resources import-ontology` but writes under `{storage}/Index/`.
4//! After a successful compile the raw RDF/OWL source file is removed by default so only
5//! `{ontology_id}.q42` (+ `.q42.meta.json` sidecar) remain under `Index/`.
6
7use std::collections::HashMap;
8use std::io::{Read, Write};
9use std::path::{Path, PathBuf};
10use std::sync::atomic::{AtomicBool, Ordering};
11use std::sync::{Arc, Mutex};
12
13use qualia_core_db::{
14    ingest,
15    resource_catalog::{OntologyResource, ResourceCatalog},
16    wal::WriteAheadLog,
17};
18use serde::Serialize;
19use sha2::{Digest, Sha256};
20
21use crate::state::ProgressPayload;
22
23/// Optional download/import progress sink (shared with LLM Hub via `get_active_downloads`).
24pub struct ImportProgressCtx {
25    pub id: String,
26    pub handles: Arc<Mutex<HashMap<String, Arc<AtomicBool>>>>,
27    pub active_downloads: Arc<Mutex<HashMap<String, ProgressPayload>>>,
28    pub download_events: tokio::sync::broadcast::Sender<ProgressPayload>,
29}
30
31impl ImportProgressCtx {
32    /// Emits an event.
33    pub fn emit(&self, payload: ProgressPayload) {
34        let _ = self.download_events.send(payload.clone());
35        self.active_downloads
36            .lock()
37            .unwrap()
38            .insert(self.id.clone(), payload);
39    }
40
41    /// Clears.
42    pub fn clear(&self) {
43        self.handles.lock().unwrap().remove(&self.id);
44        self.active_downloads.lock().unwrap().remove(&self.id);
45    }
46}
47
48#[derive(Debug)]
49pub enum ImportError {
50    NotFound(String),
51    NoDownloadUrl(String),
52    Download(String),
53    Ingest(String),
54    Wal(String),
55    Io(std::io::Error),
56    Json(serde_json::Error),
57}
58
59impl std::fmt::Display for ImportError {
60    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61        match self {
62            ImportError::NotFound(id) => write!(f, "Ontology not found in catalog: {id}"),
63            ImportError::NoDownloadUrl(id) => write!(f, "No download URL for ontology: {id}"),
64            ImportError::Download(e) => write!(f, "Download failed: {e}"),
65            ImportError::Ingest(e) => write!(f, "Ingest failed: {e}"),
66            ImportError::Wal(e) => write!(f, "WAL write failed: {e}"),
67            ImportError::Io(e) => write!(f, "IO error: {e}"),
68            ImportError::Json(e) => write!(f, "JSON error: {e}"),
69        }
70    }
71}
72
73impl From<std::io::Error> for ImportError {
74    fn from(e: std::io::Error) -> Self {
75        ImportError::Io(e)
76    }
77}
78
79impl From<serde_json::Error> for ImportError {
80    fn from(e: serde_json::Error) -> Self {
81        ImportError::Json(e)
82    }
83}
84
85#[derive(Debug, Clone, Serialize)]
86pub struct OntologyImportResult {
87    pub ontology_id: String,
88    pub source_path: String,
89    pub q42_path: String,
90    pub wal_path: String,
91    pub quin_count: u64,
92    pub catalog_quins: usize,
93    pub sha256: String,
94    pub imported_at: u64,
95    /// True when the downloaded RDF/OWL file was deleted after `.q42` compile.
96    pub source_removed: bool,
97}
98
99#[derive(Serialize)]
100struct OntologyMetaSidecar {
101    ontology_id: String,
102    quin_count: u64,
103    sha256: String,
104    imported_at: u64,
105    source_path: String,
106    q42_path: String,
107}
108
109/// Index directory.
110pub fn index_dir(storage_root: &Path) -> PathBuf {
111    storage_root.join("Index")
112}
113
114fn ontology_source_filename(ont: &OntologyResource) -> String {
115    ont.download
116        .local_filename()
117        .unwrap_or_else(|| format!("{}.{}", ont.id, ont.format))
118}
119
120fn unix_now() -> u64 {
121    std::time::SystemTime::now()
122        .duration_since(std::time::UNIX_EPOCH)
123        .unwrap_or_default()
124        .as_secs()
125}
126
127fn sha256_file(path: &Path) -> Result<String, std::io::Error> {
128    let mut file = std::fs::File::open(path)?;
129    let mut hasher = Sha256::new();
130    let mut buf = [0u8; 65_536];
131    loop {
132        let n = file.read(&mut buf)?;
133        if n == 0 {
134            break;
135        }
136        hasher.update(&buf[..n]);
137    }
138    Ok(hex::encode(hasher.finalize()))
139}
140
141fn write_meta_sidecar(
142    ontology_id: &str,
143    q42_path: &Path,
144    source_path: &Path,
145    quin_count: u64,
146    sha256: &str,
147    imported_at: u64,
148) -> Result<(), ImportError> {
149    let meta_path = q42_path.with_extension("q42.meta.json");
150    let meta = OntologyMetaSidecar {
151        ontology_id: ontology_id.to_string(),
152        quin_count,
153        sha256: sha256.to_string(),
154        imported_at,
155        source_path: source_path.to_string_lossy().into_owned(),
156        q42_path: q42_path.to_string_lossy().into_owned(),
157    };
158    let json = serde_json::to_string_pretty(&meta)?;
159    std::fs::write(&meta_path, json)?;
160    Ok(())
161}
162
163/// Streams a download.
164pub async fn stream_download(url: &str, dest: &Path) -> Result<(), String> {
165    stream_download_with_progress(url, dest, None)
166        .await
167        .map_err(|e| e.to_string())
168}
169
170/// Streams a download with progress events.
171pub async fn stream_download_with_progress(
172    url: &str,
173    dest: &Path,
174    progress: Option<&ImportProgressCtx>,
175) -> Result<(), ImportError> {
176    let client = reqwest::Client::new();
177    let response = client
178        .get(url)
179        .header(
180            "User-Agent",
181            concat!("qualiaDB-client/", env!("CARGO_PKG_VERSION")),
182        )
183        .send()
184        .await
185        .map_err(|e| ImportError::Download(e.to_string()))?
186        .error_for_status()
187        .map_err(|e| ImportError::Download(e.to_string()))?;
188
189    let total_bytes = response.content_length().unwrap_or(0);
190    if let Some(ctx) = progress {
191        ctx.emit(ProgressPayload {
192            id: ctx.id.clone(),
193            progress: 0.0,
194            downloaded_bytes: 0,
195            total_bytes,
196            speed_kbps: 0.0,
197            status: "downloading".to_string(),
198        });
199    }
200
201    if let Some(parent) = dest.parent() {
202        std::fs::create_dir_all(parent)?;
203    }
204
205    let mut file = std::fs::File::create(dest).map_err(|e| {
206        ImportError::Io(std::io::Error::new(
207            e.kind(),
208            format!("Cannot create {}: {}", dest.display(), e),
209        ))
210    })?;
211
212    let mut stream = response.bytes_stream();
213    use futures_util::StreamExt;
214    let mut downloaded: u64 = 0;
215    let mut last_report = std::time::Instant::now();
216    let mut last_downloaded: u64 = 0;
217
218    while let Some(chunk) = stream.next().await {
219        if let Some(ctx) = progress {
220            if let Some(flag) = ctx.handles.lock().unwrap().get(&ctx.id) {
221                if flag.load(Ordering::Relaxed) {
222                    let _ = std::fs::remove_file(dest);
223                    ctx.emit(ProgressPayload {
224                        id: ctx.id.clone(),
225                        progress: 0.0,
226                        downloaded_bytes: downloaded,
227                        total_bytes,
228                        speed_kbps: 0.0,
229                        status: "cancelled".to_string(),
230                    });
231                    ctx.clear();
232                    return Err(ImportError::Download("Cancelled".to_string()));
233                }
234            }
235        }
236
237        let chunk = chunk.map_err(|e| ImportError::Download(e.to_string()))?;
238        file.write_all(&chunk)?;
239        downloaded += chunk.len() as u64;
240
241        if let Some(ctx) = progress {
242            let now = std::time::Instant::now();
243            if now.duration_since(last_report).as_millis() >= 200 {
244                let elapsed = now.duration_since(last_report).as_secs_f64().max(0.001);
245                let speed_kbps = ((downloaded - last_downloaded) as f64 / 1024.0) / elapsed;
246                let progress_pct = if total_bytes > 0 {
247                    (downloaded as f64 / total_bytes as f64) * 100.0
248                } else {
249                    0.0
250                };
251                ctx.emit(ProgressPayload {
252                    id: ctx.id.clone(),
253                    progress: progress_pct,
254                    downloaded_bytes: downloaded,
255                    total_bytes,
256                    speed_kbps,
257                    status: "downloading".to_string(),
258                });
259                last_report = now;
260                last_downloaded = downloaded;
261            }
262        }
263    }
264
265    Ok(())
266}
267
268/// Ingest a local RDF file into `{storage}/Index/{ontology_id}.q42`.
269pub fn ingest_local_rdf(
270    source_path: &Path,
271    ontology_id: &str,
272    storage_root: &Path,
273    ont: Option<&OntologyResource>,
274) -> Result<u64, ImportError> {
275    ingest_local_rdf_with_progress(source_path, ontology_id, storage_root, ont, None)
276}
277
278/// Same as [`ingest_local_rdf`], emitting phase / RSS / rate snapshots onto the
279/// existing download-progress bus so desktop UIs can show ingest work.
280pub fn ingest_local_rdf_with_progress(
281    source_path: &Path,
282    ontology_id: &str,
283    storage_root: &Path,
284    ont: Option<&OntologyResource>,
285    progress: Option<&ImportProgressCtx>,
286) -> Result<u64, ImportError> {
287    let index = index_dir(storage_root);
288    std::fs::create_dir_all(&index)?;
289
290    let q42_path = index.join(format!("{ontology_id}.q42"));
291    let in_str = source_path.to_string_lossy().into_owned();
292    let out_str = q42_path.to_string_lossy().into_owned();
293
294    let report = match progress {
295        Some(ctx) => {
296            let id = ctx.id.clone();
297            let events = ctx.download_events.clone();
298            let active = ctx.active_downloads.clone();
299            ingest::IngestReport::new(1, move |snap| {
300                let payload = ProgressPayload {
301                    id: id.clone(),
302                    progress: snap.pct,
303                    downloaded_bytes: snap.bytes_read,
304                    total_bytes: snap.source_bytes,
305                    speed_kbps: snap.mb_per_sec * 1024.0,
306                    status: format!("ingest:{}", snap.format_text(1)),
307                };
308                let _ = events.send(payload.clone());
309                if let Ok(mut map) = active.lock() {
310                    map.insert(id.clone(), payload);
311                }
312            })
313        }
314        None => ingest::IngestReport::silent(),
315    };
316
317    let quin_count = ingest::streaming_import_rdf_with_report(
318        &in_str,
319        &out_str,
320        ingest::IngestMode::Complete,
321        None,
322        report,
323    )
324    .map_err(|e| ImportError::Ingest(e.to_string()))?;
325
326    let imported_at = unix_now();
327    let sha256 = sha256_file(&q42_path)?;
328
329    if let Some(ont) = ont {
330        append_ontology_wal(storage_root, ont, &q42_path, imported_at)?;
331    }
332
333    write_meta_sidecar(
334        ontology_id,
335        &q42_path,
336        source_path,
337        quin_count,
338        &sha256,
339        imported_at,
340    )?;
341
342    Ok(quin_count)
343}
344
345fn append_ontology_wal(
346    storage_root: &Path,
347    ont: &OntologyResource,
348    q42_path: &Path,
349    timestamp: u64,
350) -> Result<usize, ImportError> {
351    let wal_path = index_dir(storage_root).join("ontologies.wal");
352    let mut wal = WriteAheadLog::open(&wal_path)
353        .map_err(|e| ImportError::Wal(format!("Cannot open {}: {}", wal_path.display(), e)))?;
354
355    let prov = ont.provenance_quin(timestamp, &q42_path.to_string_lossy());
356    wal.append_mutation(&prov)
357        .map_err(|e| ImportError::Wal(e.to_string()))?;
358
359    let catalog_quins = ont.to_quins();
360    for q in &catalog_quins {
361        wal.append_mutation(q)
362            .map_err(|e| ImportError::Wal(e.to_string()))?;
363    }
364
365    Ok(catalog_quins.len())
366}
367
368/// Download a catalog ontology and compile it to `.q42` under `{storage}/Index/`.
369pub async fn import_catalog_ontology(
370    catalog: &ResourceCatalog,
371    id: &str,
372    storage_root: &Path,
373) -> Result<OntologyImportResult, ImportError> {
374    import_catalog_ontology_with_options(catalog, id, storage_root, None, true).await
375}
376
377/// Download + compile with optional progress reporting and post-ingest source cleanup.
378pub async fn import_catalog_ontology_with_options(
379    catalog: &ResourceCatalog,
380    id: &str,
381    storage_root: &Path,
382    progress: Option<&ImportProgressCtx>,
383    delete_source_after_ingest: bool,
384) -> Result<OntologyImportResult, ImportError> {
385    let ont = catalog
386        .find_ontology(id)
387        .ok_or_else(|| ImportError::NotFound(id.to_string()))?;
388
389    if let Some(source_path) = crate::bundled_ontologies::resolve_bundled_ontology_source(id) {
390        if let Some(ctx) = progress {
391            ctx.emit(ProgressPayload {
392                id: ctx.id.clone(),
393                progress: 100.0,
394                downloaded_bytes: 0,
395                total_bytes: 0,
396                speed_kbps: 0.0,
397                status: "processing".to_string(),
398            });
399        }
400
401        let index = index_dir(storage_root);
402        std::fs::create_dir_all(&index)?;
403        let quin_count =
404            ingest_local_rdf_with_progress(&source_path, id, storage_root, Some(ont), progress)?;
405        let q42_path = index.join(format!("{id}.q42"));
406        let wal_path = index.join("ontologies.wal");
407        let catalog_quins = ont.to_quins().len();
408        let sha256 = sha256_file(&q42_path)?;
409        let imported_at = unix_now();
410
411        if let Some(ctx) = progress {
412            ctx.emit(ProgressPayload {
413                id: ctx.id.clone(),
414                progress: 100.0,
415                downloaded_bytes: 0,
416                total_bytes: 0,
417                speed_kbps: 0.0,
418                status: "complete".to_string(),
419            });
420            ctx.clear();
421        }
422
423        return Ok(OntologyImportResult {
424            ontology_id: id.to_string(),
425            source_path: source_path.to_string_lossy().into_owned(),
426            q42_path: q42_path.to_string_lossy().into_owned(),
427            wal_path: wal_path.to_string_lossy().into_owned(),
428            quin_count,
429            catalog_quins,
430            sha256,
431            imported_at,
432            source_removed: false,
433        });
434    }
435
436    let url = ont
437        .download
438        .resolved_url()
439        .ok_or_else(|| ImportError::NoDownloadUrl(id.to_string()))?;
440
441    let index = index_dir(storage_root);
442    std::fs::create_dir_all(&index)?;
443
444    let filename = ontology_source_filename(ont);
445    let source_path = index.join(&filename);
446
447    stream_download_with_progress(&url, &source_path, progress).await?;
448
449    if let Some(ctx) = progress {
450        ctx.emit(ProgressPayload {
451            id: ctx.id.clone(),
452            progress: 100.0,
453            downloaded_bytes: std::fs::metadata(&source_path)
454                .map(|m| m.len())
455                .unwrap_or(0),
456            total_bytes: std::fs::metadata(&source_path)
457                .map(|m| m.len())
458                .unwrap_or(0),
459            speed_kbps: 0.0,
460            status: "processing".to_string(),
461        });
462    }
463
464    let quin_count =
465        ingest_local_rdf_with_progress(&source_path, id, storage_root, Some(ont), progress)?;
466    let q42_path = index.join(format!("{id}.q42"));
467    let wal_path = index.join("ontologies.wal");
468    let catalog_quins = ont.to_quins().len();
469    let sha256 = sha256_file(&q42_path)?;
470    let imported_at = unix_now();
471    let source_path_str = source_path.to_string_lossy().into_owned();
472
473    let mut source_removed = false;
474    if delete_source_after_ingest && source_path.is_file() {
475        if std::fs::remove_file(&source_path).is_ok() {
476            source_removed = true;
477        }
478    }
479
480    if let Some(ctx) = progress {
481        ctx.emit(ProgressPayload {
482            id: ctx.id.clone(),
483            progress: 100.0,
484            downloaded_bytes: 0,
485            total_bytes: 0,
486            speed_kbps: 0.0,
487            status: "complete".to_string(),
488        });
489        ctx.clear();
490    }
491
492    Ok(OntologyImportResult {
493        ontology_id: id.to_string(),
494        source_path: source_path_str,
495        q42_path: q42_path.to_string_lossy().into_owned(),
496        wal_path: wal_path.to_string_lossy().into_owned(),
497        quin_count,
498        catalog_quins,
499        sha256,
500        imported_at,
501        source_removed,
502    })
503}