Skip to main content

qualia_core_db/query/
ingest_report.rs

1//! Progress and process snapshots for RDF → Q42 ingest.
2//!
3//! Library callers stay silent unless they attach a sink. The CLI and desktop
4//! UIs attach text, JSON-lines, or a `ProgressPayload` mapper. Cold path only.
5
6use serde::Serialize;
7use std::io::{self, Read, Write};
8use std::sync::atomic::{AtomicU64, Ordering};
9use std::sync::{Arc, Mutex};
10use std::time::Instant;
11
12const TICK_EVERY_TRIPLES: u64 = 2_000_000;
13const TICK_EVERY_MS: u128 = 2_000;
14
15#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, serde::Deserialize)]
16#[serde(rename_all = "snake_case")]
17pub enum IngestPhase {
18    Starting,
19    /// Re-reading already-accepted triples (skip-N). Not new work.
20    Skipping,
21    Parsing,
22    Sorting,
23    Compacting,
24    Publishing,
25    Lexicon,
26    Complete,
27    Failed,
28}
29
30impl IngestPhase {
31    /// Stable string tag for the phase.
32    pub fn as_str(self) -> &'static str {
33        match self {
34            Self::Starting => "starting",
35            Self::Skipping => "skip",
36            Self::Parsing => "parse",
37            Self::Sorting => "sort",
38            Self::Compacting => "compact",
39            Self::Publishing => "publish",
40            Self::Lexicon => "lexicon",
41            Self::Complete => "complete",
42            Self::Failed => "failed",
43        }
44    }
45}
46
47#[derive(Clone, Debug, Serialize, serde::Deserialize)]
48pub struct IngestSnapshot {
49    pub phase: IngestPhase,
50    pub triples: u64,
51    pub source_bytes: u64,
52    pub bytes_read: u64,
53    pub interned_terms: u64,
54    pub lex_runs: u64,
55    pub quin_chunks: u64,
56    pub workers: u32,
57    pub rss_bytes: Option<u64>,
58    pub elapsed_ms: u64,
59    pub triples_per_sec: f64,
60    pub mb_per_sec: f64,
61    pub pct: f64,
62    pub eta_secs: Option<f64>,
63    pub message: String,
64    #[serde(skip_serializing_if = "Option::is_none")]
65    pub detail: Option<String>,
66    /// Wall ms spent in parse (set at parse-finished).
67    pub parse_ms: u64,
68    /// Wall ms spent in GPU/CPU chunk sorts (sum).
69    pub sort_ms: u64,
70    /// Wall ms spent merging/publishing the volume.
71    pub publish_ms: u64,
72    /// Last chunk sort path (`gpu` / `cpu`), if any.
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub last_sort_path: Option<String>,
75    /// Accepted triples already on disk (skip-N target). 0 if this is a fresh ingest.
76    pub skip_target: u64,
77}
78
79impl IngestSnapshot {
80    /// Render the snapshot as human-readable text at `verbose` level.
81    pub fn format_text(&self, verbose: u8) -> String {
82        let rss = self
83            .rss_bytes
84            .map(format_bytes)
85            .unwrap_or_else(|| "n/a".into());
86        let eta = self.eta_secs.map(format_secs).unwrap_or_else(|| "—".into());
87        let triples_col = if self.phase == IngestPhase::Skipping && self.skip_target > 0 {
88            format!(
89                "{} / {} replayed",
90                format_count(self.triples),
91                format_count(self.skip_target)
92            )
93        } else {
94            format!("{} triples", format_count(self.triples))
95        };
96        let mut line = format!(
97            "ingest {:>5.1}%  {:<8}  {}  {:.0}/s  {:.1} MB/s  rss {}  eta {}",
98            self.pct,
99            self.phase.as_str(),
100            triples_col,
101            self.triples_per_sec,
102            self.mb_per_sec,
103            rss,
104            eta
105        );
106        if verbose >= 1 {
107            line.push_str(&format!(
108                "  | workers {}  interned {}  lex_runs {}  quin_chunks {}  read {}  parse {}ms sort {}ms publish {}ms",
109                self.workers,
110                format_count(self.interned_terms),
111                self.lex_runs,
112                self.quin_chunks,
113                format_bytes(self.bytes_read),
114                self.parse_ms,
115                self.sort_ms,
116                self.publish_ms
117            ));
118            if let Some(p) = &self.last_sort_path {
119                line.push_str("  accel ");
120                line.push_str(p);
121            }
122        }
123        if verbose >= 2 {
124            if let Some(detail) = &self.detail {
125                line.push_str("  | ");
126                line.push_str(detail);
127            } else if !self.message.is_empty() {
128                line.push_str("  | ");
129                line.push_str(&self.message);
130            }
131        }
132        line
133    }
134
135    /// Render the snapshot as a JSON object.
136    pub fn format_json(&self) -> String {
137        serde_json::to_string(self).unwrap_or_else(|_| "{\"phase\":\"failed\"}".into())
138    }
139}
140
141#[derive(Default)]
142struct Counters {
143    triples: AtomicU64,
144    bytes_read: AtomicU64,
145    source_bytes: AtomicU64,
146    interned: AtomicU64,
147    lex_runs: AtomicU64,
148    quin_chunks: AtomicU64,
149    parse_ms: AtomicU64,
150    sort_ms: AtomicU64,
151    publish_ms: AtomicU64,
152    last_sort_path: Mutex<Option<String>>,
153    skip_target: AtomicU64,
154}
155
156/// Cloneable handle shared by the parser, hasher collector, and merge.
157#[derive(Clone)]
158pub struct IngestReport {
159    sink: Arc<dyn Fn(&IngestSnapshot) + Send + Sync>,
160    verbose: u8,
161    workers: u32,
162    started: Instant,
163    last_emit: Arc<Mutex<Instant>>,
164    last_triples: Arc<AtomicU64>,
165    counters: Arc<Counters>,
166    progress_path: Arc<Mutex<Option<std::path::PathBuf>>>,
167}
168
169impl IngestReport {
170    /// Reporter that drops all output.
171    pub fn silent() -> Self {
172        Self::new(0, |_| {})
173    }
174
175    /// Reporter at `verbose` level invoking `sink` on each snapshot.
176    pub fn new(verbose: u8, sink: impl Fn(&IngestSnapshot) + Send + Sync + 'static) -> Self {
177        let now = Instant::now();
178        Self {
179            sink: Arc::new(sink),
180            verbose,
181            workers: 0,
182            started: now,
183            last_emit: Arc::new(Mutex::new(now)),
184            last_triples: Arc::new(AtomicU64::new(0)),
185            counters: Arc::new(Counters::default()),
186            progress_path: Arc::new(Mutex::new(None)),
187        }
188    }
189
190    /// Human-readable text progress to stderr.
191    pub fn text_stderr(verbose: u8) -> Self {
192        Self::new(verbose, move |snap| {
193            let _ = writeln!(io::stderr(), "{}", snap.format_text(verbose));
194        })
195    }
196
197    /// Machine-readable JSON progress to stdout.
198    pub fn json_stdout(verbose: u8) -> Self {
199        Self::new(verbose, move |snap| {
200            let _ = writeln!(io::stdout(), "{}", snap.format_json());
201        })
202    }
203
204    /// Record the configured worker count.
205    pub fn set_workers(&mut self, workers: u32) {
206        self.workers = workers;
207    }
208
209    /// Record total source bytes.
210    pub fn set_source_bytes(&self, n: u64) {
211        self.counters.source_bytes.store(n, Ordering::Relaxed);
212    }
213
214    /// Add `n` to bytes read.
215    pub fn add_bytes_read(&self, n: u64) {
216        self.counters.bytes_read.fetch_add(n, Ordering::Relaxed);
217    }
218
219    /// Record triples produced.
220    pub fn set_triples(&self, n: u64) {
221        self.counters.triples.store(n, Ordering::Relaxed);
222    }
223
224    /// Record interned-term count.
225    pub fn set_interned(&self, n: u64) {
226        self.counters.interned.store(n, Ordering::Relaxed);
227    }
228
229    /// Increment the lex-run counter.
230    pub fn add_lex_run(&self) {
231        self.counters.lex_runs.fetch_add(1, Ordering::Relaxed);
232    }
233
234    /// Increment the quin-chunk counter.
235    pub fn add_quin_chunk(&self) {
236        self.counters.quin_chunks.fetch_add(1, Ordering::Relaxed);
237    }
238
239    /// Add `ms` to cumulative sort time.
240    pub fn add_sort_ms(&self, ms: u64) {
241        self.counters.sort_ms.fetch_add(ms, Ordering::Relaxed);
242    }
243
244    /// Record parse time.
245    pub fn set_parse_ms(&self, ms: u64) {
246        self.counters.parse_ms.store(ms, Ordering::Relaxed);
247    }
248
249    /// Record publish time.
250    pub fn set_publish_ms(&self, ms: u64) {
251        self.counters.publish_ms.store(ms, Ordering::Relaxed);
252    }
253
254    /// Record the skip target.
255    pub fn set_skip_target(&self, n: u64) {
256        self.counters.skip_target.store(n, Ordering::Relaxed);
257    }
258
259    /// Set the quin-chunk counter.
260    pub fn set_quin_chunks(&self, n: u64) {
261        self.counters.quin_chunks.store(n, Ordering::Relaxed);
262    }
263
264    /// Set the lex-run counter.
265    pub fn set_lex_runs(&self, n: u64) {
266        self.counters.lex_runs.store(n, Ordering::Relaxed);
267    }
268
269    /// Quin-chunk counter.
270    pub fn quin_chunks(&self) -> u64 {
271        self.counters.quin_chunks.load(Ordering::Relaxed)
272    }
273
274    /// Lex-run counter.
275    pub fn lex_runs(&self) -> u64 {
276        self.counters.lex_runs.load(Ordering::Relaxed)
277    }
278
279    /// Durable snapshot for a job directory (`progress.json`). Does not change
280    /// `checkpoint.triples` — that is accepted work only.
281    pub fn attach_progress_file(&self, path: std::path::PathBuf) {
282        if let Ok(mut g) = self.progress_path.lock() {
283            *g = Some(path);
284        }
285    }
286
287    /// Verbosity level 0–2.
288    pub fn verbose(&self) -> u8 {
289        self.verbose
290    }
291
292    /// Shared callback for free-form note lines.
293    pub fn note_sink(&self) -> Arc<dyn Fn(&str) + Send + Sync> {
294        let me = self.clone();
295        Arc::new(move |msg: &str| {
296            if msg.contains("Lexicon spill: wrote") {
297                me.add_lex_run();
298            }
299            if msg.contains("quin chunk") {
300                me.add_quin_chunk();
301                if let Some(ms) = parse_trailing_ms(msg) {
302                    me.add_sort_ms(ms);
303                }
304                if let Some(p) = parse_sort_path(msg) {
305                    if let Ok(mut g) = me.counters.last_sort_path.lock() {
306                        *g = Some(p);
307                    }
308                }
309            }
310            let phase = if msg.contains("compact") {
311                IngestPhase::Compacting
312            } else if msg.to_ascii_lowercase().contains("lexicon") {
313                IngestPhase::Lexicon
314            } else {
315                IngestPhase::Publishing
316            };
317            me.emit(phase, msg, Some(msg.to_string()));
318        })
319    }
320
321    /// Emit a phase event: update counters from `detail`, invoke the sink when verbosity permits, write the progress file.
322    pub fn emit(&self, phase: IngestPhase, message: impl Into<String>, detail: Option<String>) {
323        let snap = self.snapshot(phase, message.into(), detail);
324        (self.sink)(&snap);
325        self.write_progress_file(&snap);
326        if let Ok(mut last) = self.last_emit.lock() {
327            *last = Instant::now();
328        }
329        self.last_triples.store(snap.triples, Ordering::Relaxed);
330    }
331
332    /// Persist `snap` to the progress file (when configured).
333    fn write_progress_file(&self, snap: &IngestSnapshot) {
334        let path = match self.progress_path.lock() {
335            Ok(g) => g.clone(),
336            Err(_) => return,
337        };
338        let Some(path) = path else {
339            return;
340        };
341        let Ok(body) = serde_json::to_vec(snap) else {
342            return;
343        };
344        let tmp = path.with_extension("json.tmp");
345        if std::fs::write(&tmp, body).is_ok() {
346            let _ = std::fs::rename(&tmp, path);
347        }
348    }
349
350    /// Throttled parse tick: every 2M triples or 2 seconds.
351    pub fn maybe_tick(&self, phase: IngestPhase, triples: u64) {
352        self.set_triples(triples);
353        let last_n = self.last_triples.load(Ordering::Relaxed);
354        let due_count = triples.saturating_sub(last_n) >= TICK_EVERY_TRIPLES;
355        let due_time = self
356            .last_emit
357            .lock()
358            .map(|t| t.elapsed().as_millis() >= TICK_EVERY_MS)
359            .unwrap_or(true);
360        if due_count || due_time {
361            self.emit(phase, format!("{} triples", format_count(triples)), None);
362        }
363    }
364
365    /// Skip-N replay tick. `replayed` is how many already-accepted triples this
366    /// pass has walked; `target` is `checkpoint.triples`. Does not mean new work.
367    pub fn maybe_tick_skip(&self, replayed: u64, target: u64) {
368        self.set_skip_target(target);
369        self.set_triples(replayed);
370        let last_n = self.last_triples.load(Ordering::Relaxed);
371        let due_count = replayed.saturating_sub(last_n) >= TICK_EVERY_TRIPLES;
372        let due_time = self
373            .last_emit
374            .lock()
375            .map(|t| t.elapsed().as_millis() >= TICK_EVERY_MS)
376            .unwrap_or(true);
377        if due_count || due_time {
378            self.emit(
379                IngestPhase::Skipping,
380                format!("skip {} / {}", format_count(replayed), format_count(target)),
381                None,
382            );
383        }
384    }
385
386    /// Build an `IngestSnapshot` for `phase`/`message` from the live
387    /// counters, computing rates, percent-complete, and ETA.
388    pub fn snapshot(
389        &self,
390        phase: IngestPhase,
391        message: String,
392        detail: Option<String>,
393    ) -> IngestSnapshot {
394        let triples = self.counters.triples.load(Ordering::Relaxed);
395        let source_bytes = self.counters.source_bytes.load(Ordering::Relaxed);
396        let bytes_read = self.counters.bytes_read.load(Ordering::Relaxed);
397        let elapsed = self.started.elapsed();
398        let secs = elapsed.as_secs_f64().max(0.001);
399        let triples_per_sec = triples as f64 / secs;
400        let mb_per_sec = (bytes_read as f64 / secs) / 1_048_576.0;
401        let pct = if source_bytes > 0 {
402            ((bytes_read as f64 / source_bytes as f64) * 100.0).min(100.0)
403        } else {
404            0.0
405        };
406        let eta_secs = if bytes_read > 0 && source_bytes > bytes_read && mb_per_sec > 0.0 {
407            let remain = (source_bytes - bytes_read) as f64 / 1_048_576.0;
408            Some(remain / mb_per_sec)
409        } else {
410            None
411        };
412        IngestSnapshot {
413            phase,
414            triples,
415            source_bytes,
416            bytes_read,
417            interned_terms: self.counters.interned.load(Ordering::Relaxed),
418            lex_runs: self.counters.lex_runs.load(Ordering::Relaxed),
419            quin_chunks: self.counters.quin_chunks.load(Ordering::Relaxed),
420            parse_ms: self.counters.parse_ms.load(Ordering::Relaxed),
421            sort_ms: self.counters.sort_ms.load(Ordering::Relaxed),
422            publish_ms: self.counters.publish_ms.load(Ordering::Relaxed),
423            last_sort_path: self
424                .counters
425                .last_sort_path
426                .lock()
427                .ok()
428                .and_then(|g| g.clone()),
429            workers: self.workers,
430            rss_bytes: sample_rss_bytes(),
431            elapsed_ms: elapsed.as_millis() as u64,
432            triples_per_sec,
433            mb_per_sec,
434            pct,
435            eta_secs,
436            message,
437            detail,
438            skip_target: self.counters.skip_target.load(Ordering::Relaxed),
439        }
440    }
441}
442
443/// Counts bytes as Rio reads the source, so `%` / ETA are honest.
444pub struct CountingReader<R: Read> {
445    inner: R,
446    report: IngestReport,
447}
448
449impl<R: Read> CountingReader<R> {
450    /// Construct a `CountingReader`.
451    pub fn new(inner: R, report: IngestReport) -> Self {
452        Self { inner, report }
453    }
454}
455
456impl<R: Read> Read for CountingReader<R> {
457    /// `Read` passthrough that accumulates bytes into the report.
458    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
459        let n = self.inner.read(buf)?;
460        if n > 0 {
461            self.report.add_bytes_read(n as u64);
462        }
463        Ok(n)
464    }
465}
466
467/// Human-readable byte count (KiB/MiB/GiB).
468pub fn format_bytes(n: u64) -> String {
469    const KIB: f64 = 1024.0;
470    const MIB: f64 = 1024.0 * 1024.0;
471    const GIB: f64 = 1024.0 * 1024.0 * 1024.0;
472    let x = n as f64;
473    if x >= GIB {
474        format!("{:.2} GiB", x / GIB)
475    } else if x >= MIB {
476        format!("{:.1} MiB", x / MIB)
477    } else if x >= KIB {
478        format!("{:.0} KiB", x / KIB)
479    } else {
480        format!("{n} B")
481    }
482}
483
484/// Human-readable integer with thousands separators.
485pub fn format_count(n: u64) -> String {
486    if n >= 1_000_000_000 {
487        format!("{:.2}B", n as f64 / 1_000_000_000.0)
488    } else if n >= 1_000_000 {
489        format!("{:.2}M", n as f64 / 1_000_000.0)
490    } else if n >= 1_000 {
491        format!("{:.1}k", n as f64 / 1_000.0)
492    } else {
493        n.to_string()
494    }
495}
496
497/// Extract a trailing `Nms` duration from a log message.
498fn parse_trailing_ms(msg: &str) -> Option<u64> {
499    let ms = msg.rsplit_once(' ')?;
500    let num = ms.1.strip_suffix("ms")?;
501    num.parse().ok()
502}
503
504/// Extract a sort-path token from a log message.
505fn parse_sort_path(msg: &str) -> Option<String> {
506    let rest = msg.split("path=").nth(1)?;
507    let token = rest.split_whitespace().next()?;
508    Some(token.to_string())
509}
510
511/// Human-readable duration in s/ms/µs.
512fn format_secs(secs: f64) -> String {
513    if !secs.is_finite() || secs < 0.0 {
514        return "—".into();
515    }
516    let s = secs.round() as u64;
517    let h = s / 3600;
518    let m = (s % 3600) / 60;
519    let sec = s % 60;
520    if h > 0 {
521        format!("{h}h{m:02}m")
522    } else if m > 0 {
523        format!("{m}m{sec:02}s")
524    } else {
525        format!("{sec}s")
526    }
527}
528
529/// Resident-set-size in bytes, `None` when unavailable.
530fn sample_rss_bytes() -> Option<u64> {
531    use sysinfo::{Pid, ProcessesToUpdate, System};
532    let mut sys = System::new();
533    let pid = Pid::from_u32(std::process::id());
534    sys.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
535    sys.process(pid).map(|p| p.memory())
536}
537
538#[cfg(test)]
539mod tests {
540    use super::*;
541
542    #[test]
543    fn text_line_includes_phase_and_percent() {
544        let snap = IngestSnapshot {
545            phase: IngestPhase::Parsing,
546            triples: 2_000_000,
547            source_bytes: 100,
548            bytes_read: 25,
549            interned_terms: 10,
550            lex_runs: 1,
551            quin_chunks: 2,
552            workers: 6,
553            rss_bytes: Some(64 * 1024 * 1024),
554            elapsed_ms: 1000,
555            triples_per_sec: 2_000_000.0,
556            mb_per_sec: 10.0,
557            pct: 25.0,
558            eta_secs: Some(90.0),
559            message: "2.00M triples".into(),
560            detail: Some("lexrun_0.tmp".into()),
561            parse_ms: 10,
562            sort_ms: 20,
563            publish_ms: 30,
564            last_sort_path: Some("cpu".into()),
565            skip_target: 0,
566        };
567        let line = snap.format_text(0);
568        assert!(line.contains("25.0%"), "{line}");
569        assert!(line.contains("parse"), "{line}");
570        assert!(line.contains("2.00M"), "{line}");
571        let verbose = snap.format_text(2);
572        assert!(verbose.contains("lexrun_0.tmp"), "{verbose}");
573        assert!(verbose.contains("workers 6"), "{verbose}");
574    }
575
576    #[test]
577    fn skip_tick_does_not_look_like_zero_parse() {
578        let seen = Arc::new(Mutex::new(Vec::new()));
579        let slot = seen.clone();
580        let report = IngestReport::new(1, move |s| slot.lock().unwrap().push(s.clone()));
581        report.set_source_bytes(100);
582        report.add_bytes_read(40);
583        report.maybe_tick_skip(2_000_000, 1_808_000_000);
584        let snaps = seen.lock().unwrap();
585        assert_eq!(snaps.len(), 1);
586        assert_eq!(snaps[0].phase, IngestPhase::Skipping);
587        assert_eq!(snaps[0].skip_target, 1_808_000_000);
588        assert_eq!(snaps[0].triples, 2_000_000);
589        let line = snaps[0].format_text(0);
590        assert!(line.contains("skip"), "{line}");
591        assert!(line.contains("replayed"), "{line}");
592    }
593
594    #[test]
595    fn progress_file_is_written_on_emit() {
596        let dir = tempfile::TempDir::new().unwrap();
597        let path = dir.path().join("progress.json");
598        let report = IngestReport::silent();
599        report.attach_progress_file(path.clone());
600        report.set_source_bytes(10);
601        report.emit(IngestPhase::Skipping, "skip 1 / 2", None);
602        let body = std::fs::read_to_string(&path).unwrap();
603        assert!(body.contains("\"skipping\""), "{body}");
604    }
605
606    #[test]
607    fn report_ticks_and_records_triples() {
608        let seen = Arc::new(Mutex::new(Vec::new()));
609        let slot = seen.clone();
610        let report = IngestReport::new(1, move |s| slot.lock().unwrap().push(s.clone()));
611        report.set_source_bytes(1000);
612        report.add_bytes_read(250);
613        report.maybe_tick(IngestPhase::Parsing, 2_000_000);
614        let snaps = seen.lock().unwrap();
615        assert_eq!(snaps.len(), 1);
616        assert_eq!(snaps[0].triples, 2_000_000);
617        assert_eq!(snaps[0].phase, IngestPhase::Parsing);
618    }
619}