1use serde::Serialize;
7use std::io::{self, Read, Write};
8use std::sync::atomic::{AtomicU64, Ordering};
9use std::sync::{Arc, Mutex};
10use std::time::Instant;
11
12const TICK_EVERY_TRIPLES: u64 = 2_000_000;
13const TICK_EVERY_MS: u128 = 2_000;
14
15#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, serde::Deserialize)]
16#[serde(rename_all = "snake_case")]
17pub enum IngestPhase {
18 Starting,
19 Skipping,
21 Parsing,
22 Sorting,
23 Compacting,
24 Publishing,
25 Lexicon,
26 Complete,
27 Failed,
28}
29
30impl IngestPhase {
31 pub fn as_str(self) -> &'static str {
33 match self {
34 Self::Starting => "starting",
35 Self::Skipping => "skip",
36 Self::Parsing => "parse",
37 Self::Sorting => "sort",
38 Self::Compacting => "compact",
39 Self::Publishing => "publish",
40 Self::Lexicon => "lexicon",
41 Self::Complete => "complete",
42 Self::Failed => "failed",
43 }
44 }
45}
46
47#[derive(Clone, Debug, Serialize, serde::Deserialize)]
48pub struct IngestSnapshot {
49 pub phase: IngestPhase,
50 pub triples: u64,
51 pub source_bytes: u64,
52 pub bytes_read: u64,
53 pub interned_terms: u64,
54 pub lex_runs: u64,
55 pub quin_chunks: u64,
56 pub workers: u32,
57 pub rss_bytes: Option<u64>,
58 pub elapsed_ms: u64,
59 pub triples_per_sec: f64,
60 pub mb_per_sec: f64,
61 pub pct: f64,
62 pub eta_secs: Option<f64>,
63 pub message: String,
64 #[serde(skip_serializing_if = "Option::is_none")]
65 pub detail: Option<String>,
66 pub parse_ms: u64,
68 pub sort_ms: u64,
70 pub publish_ms: u64,
72 #[serde(skip_serializing_if = "Option::is_none")]
74 pub last_sort_path: Option<String>,
75 pub skip_target: u64,
77}
78
79impl IngestSnapshot {
80 pub fn format_text(&self, verbose: u8) -> String {
82 let rss = self
83 .rss_bytes
84 .map(format_bytes)
85 .unwrap_or_else(|| "n/a".into());
86 let eta = self.eta_secs.map(format_secs).unwrap_or_else(|| "—".into());
87 let triples_col = if self.phase == IngestPhase::Skipping && self.skip_target > 0 {
88 format!(
89 "{} / {} replayed",
90 format_count(self.triples),
91 format_count(self.skip_target)
92 )
93 } else {
94 format!("{} triples", format_count(self.triples))
95 };
96 let mut line = format!(
97 "ingest {:>5.1}% {:<8} {} {:.0}/s {:.1} MB/s rss {} eta {}",
98 self.pct,
99 self.phase.as_str(),
100 triples_col,
101 self.triples_per_sec,
102 self.mb_per_sec,
103 rss,
104 eta
105 );
106 if verbose >= 1 {
107 line.push_str(&format!(
108 " | workers {} interned {} lex_runs {} quin_chunks {} read {} parse {}ms sort {}ms publish {}ms",
109 self.workers,
110 format_count(self.interned_terms),
111 self.lex_runs,
112 self.quin_chunks,
113 format_bytes(self.bytes_read),
114 self.parse_ms,
115 self.sort_ms,
116 self.publish_ms
117 ));
118 if let Some(p) = &self.last_sort_path {
119 line.push_str(" accel ");
120 line.push_str(p);
121 }
122 }
123 if verbose >= 2 {
124 if let Some(detail) = &self.detail {
125 line.push_str(" | ");
126 line.push_str(detail);
127 } else if !self.message.is_empty() {
128 line.push_str(" | ");
129 line.push_str(&self.message);
130 }
131 }
132 line
133 }
134
135 pub fn format_json(&self) -> String {
137 serde_json::to_string(self).unwrap_or_else(|_| "{\"phase\":\"failed\"}".into())
138 }
139}
140
141#[derive(Default)]
142struct Counters {
143 triples: AtomicU64,
144 bytes_read: AtomicU64,
145 source_bytes: AtomicU64,
146 interned: AtomicU64,
147 lex_runs: AtomicU64,
148 quin_chunks: AtomicU64,
149 parse_ms: AtomicU64,
150 sort_ms: AtomicU64,
151 publish_ms: AtomicU64,
152 last_sort_path: Mutex<Option<String>>,
153 skip_target: AtomicU64,
154}
155
156#[derive(Clone)]
158pub struct IngestReport {
159 sink: Arc<dyn Fn(&IngestSnapshot) + Send + Sync>,
160 verbose: u8,
161 workers: u32,
162 started: Instant,
163 last_emit: Arc<Mutex<Instant>>,
164 last_triples: Arc<AtomicU64>,
165 counters: Arc<Counters>,
166 progress_path: Arc<Mutex<Option<std::path::PathBuf>>>,
167}
168
169impl IngestReport {
170 pub fn silent() -> Self {
172 Self::new(0, |_| {})
173 }
174
175 pub fn new(verbose: u8, sink: impl Fn(&IngestSnapshot) + Send + Sync + 'static) -> Self {
177 let now = Instant::now();
178 Self {
179 sink: Arc::new(sink),
180 verbose,
181 workers: 0,
182 started: now,
183 last_emit: Arc::new(Mutex::new(now)),
184 last_triples: Arc::new(AtomicU64::new(0)),
185 counters: Arc::new(Counters::default()),
186 progress_path: Arc::new(Mutex::new(None)),
187 }
188 }
189
190 pub fn text_stderr(verbose: u8) -> Self {
192 Self::new(verbose, move |snap| {
193 let _ = writeln!(io::stderr(), "{}", snap.format_text(verbose));
194 })
195 }
196
197 pub fn json_stdout(verbose: u8) -> Self {
199 Self::new(verbose, move |snap| {
200 let _ = writeln!(io::stdout(), "{}", snap.format_json());
201 })
202 }
203
204 pub fn set_workers(&mut self, workers: u32) {
206 self.workers = workers;
207 }
208
209 pub fn set_source_bytes(&self, n: u64) {
211 self.counters.source_bytes.store(n, Ordering::Relaxed);
212 }
213
214 pub fn add_bytes_read(&self, n: u64) {
216 self.counters.bytes_read.fetch_add(n, Ordering::Relaxed);
217 }
218
219 pub fn set_triples(&self, n: u64) {
221 self.counters.triples.store(n, Ordering::Relaxed);
222 }
223
224 pub fn set_interned(&self, n: u64) {
226 self.counters.interned.store(n, Ordering::Relaxed);
227 }
228
229 pub fn add_lex_run(&self) {
231 self.counters.lex_runs.fetch_add(1, Ordering::Relaxed);
232 }
233
234 pub fn add_quin_chunk(&self) {
236 self.counters.quin_chunks.fetch_add(1, Ordering::Relaxed);
237 }
238
239 pub fn add_sort_ms(&self, ms: u64) {
241 self.counters.sort_ms.fetch_add(ms, Ordering::Relaxed);
242 }
243
244 pub fn set_parse_ms(&self, ms: u64) {
246 self.counters.parse_ms.store(ms, Ordering::Relaxed);
247 }
248
249 pub fn set_publish_ms(&self, ms: u64) {
251 self.counters.publish_ms.store(ms, Ordering::Relaxed);
252 }
253
254 pub fn set_skip_target(&self, n: u64) {
256 self.counters.skip_target.store(n, Ordering::Relaxed);
257 }
258
259 pub fn set_quin_chunks(&self, n: u64) {
261 self.counters.quin_chunks.store(n, Ordering::Relaxed);
262 }
263
264 pub fn set_lex_runs(&self, n: u64) {
266 self.counters.lex_runs.store(n, Ordering::Relaxed);
267 }
268
269 pub fn quin_chunks(&self) -> u64 {
271 self.counters.quin_chunks.load(Ordering::Relaxed)
272 }
273
274 pub fn lex_runs(&self) -> u64 {
276 self.counters.lex_runs.load(Ordering::Relaxed)
277 }
278
279 pub fn attach_progress_file(&self, path: std::path::PathBuf) {
282 if let Ok(mut g) = self.progress_path.lock() {
283 *g = Some(path);
284 }
285 }
286
287 pub fn verbose(&self) -> u8 {
289 self.verbose
290 }
291
292 pub fn note_sink(&self) -> Arc<dyn Fn(&str) + Send + Sync> {
294 let me = self.clone();
295 Arc::new(move |msg: &str| {
296 if msg.contains("Lexicon spill: wrote") {
297 me.add_lex_run();
298 }
299 if msg.contains("quin chunk") {
300 me.add_quin_chunk();
301 if let Some(ms) = parse_trailing_ms(msg) {
302 me.add_sort_ms(ms);
303 }
304 if let Some(p) = parse_sort_path(msg) {
305 if let Ok(mut g) = me.counters.last_sort_path.lock() {
306 *g = Some(p);
307 }
308 }
309 }
310 let phase = if msg.contains("compact") {
311 IngestPhase::Compacting
312 } else if msg.to_ascii_lowercase().contains("lexicon") {
313 IngestPhase::Lexicon
314 } else {
315 IngestPhase::Publishing
316 };
317 me.emit(phase, msg, Some(msg.to_string()));
318 })
319 }
320
321 pub fn emit(&self, phase: IngestPhase, message: impl Into<String>, detail: Option<String>) {
323 let snap = self.snapshot(phase, message.into(), detail);
324 (self.sink)(&snap);
325 self.write_progress_file(&snap);
326 if let Ok(mut last) = self.last_emit.lock() {
327 *last = Instant::now();
328 }
329 self.last_triples.store(snap.triples, Ordering::Relaxed);
330 }
331
332 fn write_progress_file(&self, snap: &IngestSnapshot) {
334 let path = match self.progress_path.lock() {
335 Ok(g) => g.clone(),
336 Err(_) => return,
337 };
338 let Some(path) = path else {
339 return;
340 };
341 let Ok(body) = serde_json::to_vec(snap) else {
342 return;
343 };
344 let tmp = path.with_extension("json.tmp");
345 if std::fs::write(&tmp, body).is_ok() {
346 let _ = std::fs::rename(&tmp, path);
347 }
348 }
349
350 pub fn maybe_tick(&self, phase: IngestPhase, triples: u64) {
352 self.set_triples(triples);
353 let last_n = self.last_triples.load(Ordering::Relaxed);
354 let due_count = triples.saturating_sub(last_n) >= TICK_EVERY_TRIPLES;
355 let due_time = self
356 .last_emit
357 .lock()
358 .map(|t| t.elapsed().as_millis() >= TICK_EVERY_MS)
359 .unwrap_or(true);
360 if due_count || due_time {
361 self.emit(phase, format!("{} triples", format_count(triples)), None);
362 }
363 }
364
365 pub fn maybe_tick_skip(&self, replayed: u64, target: u64) {
368 self.set_skip_target(target);
369 self.set_triples(replayed);
370 let last_n = self.last_triples.load(Ordering::Relaxed);
371 let due_count = replayed.saturating_sub(last_n) >= TICK_EVERY_TRIPLES;
372 let due_time = self
373 .last_emit
374 .lock()
375 .map(|t| t.elapsed().as_millis() >= TICK_EVERY_MS)
376 .unwrap_or(true);
377 if due_count || due_time {
378 self.emit(
379 IngestPhase::Skipping,
380 format!("skip {} / {}", format_count(replayed), format_count(target)),
381 None,
382 );
383 }
384 }
385
386 pub fn snapshot(
389 &self,
390 phase: IngestPhase,
391 message: String,
392 detail: Option<String>,
393 ) -> IngestSnapshot {
394 let triples = self.counters.triples.load(Ordering::Relaxed);
395 let source_bytes = self.counters.source_bytes.load(Ordering::Relaxed);
396 let bytes_read = self.counters.bytes_read.load(Ordering::Relaxed);
397 let elapsed = self.started.elapsed();
398 let secs = elapsed.as_secs_f64().max(0.001);
399 let triples_per_sec = triples as f64 / secs;
400 let mb_per_sec = (bytes_read as f64 / secs) / 1_048_576.0;
401 let pct = if source_bytes > 0 {
402 ((bytes_read as f64 / source_bytes as f64) * 100.0).min(100.0)
403 } else {
404 0.0
405 };
406 let eta_secs = if bytes_read > 0 && source_bytes > bytes_read && mb_per_sec > 0.0 {
407 let remain = (source_bytes - bytes_read) as f64 / 1_048_576.0;
408 Some(remain / mb_per_sec)
409 } else {
410 None
411 };
412 IngestSnapshot {
413 phase,
414 triples,
415 source_bytes,
416 bytes_read,
417 interned_terms: self.counters.interned.load(Ordering::Relaxed),
418 lex_runs: self.counters.lex_runs.load(Ordering::Relaxed),
419 quin_chunks: self.counters.quin_chunks.load(Ordering::Relaxed),
420 parse_ms: self.counters.parse_ms.load(Ordering::Relaxed),
421 sort_ms: self.counters.sort_ms.load(Ordering::Relaxed),
422 publish_ms: self.counters.publish_ms.load(Ordering::Relaxed),
423 last_sort_path: self
424 .counters
425 .last_sort_path
426 .lock()
427 .ok()
428 .and_then(|g| g.clone()),
429 workers: self.workers,
430 rss_bytes: sample_rss_bytes(),
431 elapsed_ms: elapsed.as_millis() as u64,
432 triples_per_sec,
433 mb_per_sec,
434 pct,
435 eta_secs,
436 message,
437 detail,
438 skip_target: self.counters.skip_target.load(Ordering::Relaxed),
439 }
440 }
441}
442
443pub struct CountingReader<R: Read> {
445 inner: R,
446 report: IngestReport,
447}
448
449impl<R: Read> CountingReader<R> {
450 pub fn new(inner: R, report: IngestReport) -> Self {
452 Self { inner, report }
453 }
454}
455
456impl<R: Read> Read for CountingReader<R> {
457 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
459 let n = self.inner.read(buf)?;
460 if n > 0 {
461 self.report.add_bytes_read(n as u64);
462 }
463 Ok(n)
464 }
465}
466
467pub fn format_bytes(n: u64) -> String {
469 const KIB: f64 = 1024.0;
470 const MIB: f64 = 1024.0 * 1024.0;
471 const GIB: f64 = 1024.0 * 1024.0 * 1024.0;
472 let x = n as f64;
473 if x >= GIB {
474 format!("{:.2} GiB", x / GIB)
475 } else if x >= MIB {
476 format!("{:.1} MiB", x / MIB)
477 } else if x >= KIB {
478 format!("{:.0} KiB", x / KIB)
479 } else {
480 format!("{n} B")
481 }
482}
483
484pub fn format_count(n: u64) -> String {
486 if n >= 1_000_000_000 {
487 format!("{:.2}B", n as f64 / 1_000_000_000.0)
488 } else if n >= 1_000_000 {
489 format!("{:.2}M", n as f64 / 1_000_000.0)
490 } else if n >= 1_000 {
491 format!("{:.1}k", n as f64 / 1_000.0)
492 } else {
493 n.to_string()
494 }
495}
496
497fn parse_trailing_ms(msg: &str) -> Option<u64> {
499 let ms = msg.rsplit_once(' ')?;
500 let num = ms.1.strip_suffix("ms")?;
501 num.parse().ok()
502}
503
504fn parse_sort_path(msg: &str) -> Option<String> {
506 let rest = msg.split("path=").nth(1)?;
507 let token = rest.split_whitespace().next()?;
508 Some(token.to_string())
509}
510
511fn format_secs(secs: f64) -> String {
513 if !secs.is_finite() || secs < 0.0 {
514 return "—".into();
515 }
516 let s = secs.round() as u64;
517 let h = s / 3600;
518 let m = (s % 3600) / 60;
519 let sec = s % 60;
520 if h > 0 {
521 format!("{h}h{m:02}m")
522 } else if m > 0 {
523 format!("{m}m{sec:02}s")
524 } else {
525 format!("{sec}s")
526 }
527}
528
529fn sample_rss_bytes() -> Option<u64> {
531 use sysinfo::{Pid, ProcessesToUpdate, System};
532 let mut sys = System::new();
533 let pid = Pid::from_u32(std::process::id());
534 sys.refresh_processes(ProcessesToUpdate::Some(&[pid]), true);
535 sys.process(pid).map(|p| p.memory())
536}
537
538#[cfg(test)]
539mod tests {
540 use super::*;
541
542 #[test]
543 fn text_line_includes_phase_and_percent() {
544 let snap = IngestSnapshot {
545 phase: IngestPhase::Parsing,
546 triples: 2_000_000,
547 source_bytes: 100,
548 bytes_read: 25,
549 interned_terms: 10,
550 lex_runs: 1,
551 quin_chunks: 2,
552 workers: 6,
553 rss_bytes: Some(64 * 1024 * 1024),
554 elapsed_ms: 1000,
555 triples_per_sec: 2_000_000.0,
556 mb_per_sec: 10.0,
557 pct: 25.0,
558 eta_secs: Some(90.0),
559 message: "2.00M triples".into(),
560 detail: Some("lexrun_0.tmp".into()),
561 parse_ms: 10,
562 sort_ms: 20,
563 publish_ms: 30,
564 last_sort_path: Some("cpu".into()),
565 skip_target: 0,
566 };
567 let line = snap.format_text(0);
568 assert!(line.contains("25.0%"), "{line}");
569 assert!(line.contains("parse"), "{line}");
570 assert!(line.contains("2.00M"), "{line}");
571 let verbose = snap.format_text(2);
572 assert!(verbose.contains("lexrun_0.tmp"), "{verbose}");
573 assert!(verbose.contains("workers 6"), "{verbose}");
574 }
575
576 #[test]
577 fn skip_tick_does_not_look_like_zero_parse() {
578 let seen = Arc::new(Mutex::new(Vec::new()));
579 let slot = seen.clone();
580 let report = IngestReport::new(1, move |s| slot.lock().unwrap().push(s.clone()));
581 report.set_source_bytes(100);
582 report.add_bytes_read(40);
583 report.maybe_tick_skip(2_000_000, 1_808_000_000);
584 let snaps = seen.lock().unwrap();
585 assert_eq!(snaps.len(), 1);
586 assert_eq!(snaps[0].phase, IngestPhase::Skipping);
587 assert_eq!(snaps[0].skip_target, 1_808_000_000);
588 assert_eq!(snaps[0].triples, 2_000_000);
589 let line = snaps[0].format_text(0);
590 assert!(line.contains("skip"), "{line}");
591 assert!(line.contains("replayed"), "{line}");
592 }
593
594 #[test]
595 fn progress_file_is_written_on_emit() {
596 let dir = tempfile::TempDir::new().unwrap();
597 let path = dir.path().join("progress.json");
598 let report = IngestReport::silent();
599 report.attach_progress_file(path.clone());
600 report.set_source_bytes(10);
601 report.emit(IngestPhase::Skipping, "skip 1 / 2", None);
602 let body = std::fs::read_to_string(&path).unwrap();
603 assert!(body.contains("\"skipping\""), "{body}");
604 }
605
606 #[test]
607 fn report_ticks_and_records_triples() {
608 let seen = Arc::new(Mutex::new(Vec::new()));
609 let slot = seen.clone();
610 let report = IngestReport::new(1, move |s| slot.lock().unwrap().push(s.clone()));
611 report.set_source_bytes(1000);
612 report.add_bytes_read(250);
613 report.maybe_tick(IngestPhase::Parsing, 2_000_000);
614 let snaps = seen.lock().unwrap();
615 assert_eq!(snaps.len(), 1);
616 assert_eq!(snaps[0].triples, 2_000_000);
617 assert_eq!(snaps[0].phase, IngestPhase::Parsing);
618 }
619}