qualia_client_core/
q42_compress.rs

1//! LZ4 distribution artifacts — legacy `.c.q42` alias for browser / WebTorrent deploy.
2//!
3//! Unified v2 `.q42` volumes are self-contained (embedded lex, bidx, LZ4 blocks).
4//! `finalize_c_q42` copies v2 files unchanged; legacy v1 inputs keep the old shim.
5
6use std::fs;
7use std::io::{Read, Write};
8use std::path::Path;
9
10use serde::Serialize;
11
12const CHUNK_SIZE: usize = 393_216;
13
14#[derive(Debug, Clone, Serialize)]
15pub struct CompressStats {
16    pub input_bytes: u64,
17    pub output_bytes: u64,
18    pub blocks: u64,
19    pub ratio: f64,
20}
21
22/// Finalize `{ontology_id}.c.q42` for sharing (deprecated alias of unified v2 `.q42`).
23pub fn finalize_c_q42(q42_path: &Path, c_q42_path: &Path) -> Result<CompressStats, String> {
24    if !q42_path.is_file() {
25        return Err(format!("Missing .q42 artifact: {}", q42_path.display()));
26    }
27    if let Some(parent) = c_q42_path.parent() {
28        fs::create_dir_all(parent).map_err(|e| e.to_string())?;
29    }
30    fs::copy(q42_path, c_q42_path).map_err(|e| e.to_string())?;
31    let input_bytes = fs::metadata(q42_path).map_err(|e| e.to_string())?.len();
32    let output_bytes = fs::metadata(c_q42_path).map_err(|e| e.to_string())?.len();
33    let blocks = if qualia_core_db::q42_volume::is_unified_volume(q42_path).unwrap_or(false) {
34        qualia_core_db::q42_volume::Q42Volume::open(q42_path)
35            .map(|v| v.block_count())
36            .unwrap_or(1)
37    } else {
38        output_bytes.saturating_div(CHUNK_SIZE as u64).max(1)
39    };
40    Ok(CompressStats {
41        input_bytes,
42        output_bytes,
43        blocks,
44        ratio: if output_bytes > 0 {
45            input_bytes as f64 / output_bytes as f64
46        } else {
47            1.0
48        },
49    })
50}
51
52/// Extra LZ4 pass for non-ingest binaries (e.g. lexicon sidecars).
53pub fn compress_raw_file(input: &Path, output: &Path) -> Result<CompressStats, String> {
54    let input_bytes = fs::metadata(input).map_err(|e| e.to_string())?.len();
55    let mut reader = fs::File::open(input).map_err(|e| e.to_string())?;
56    let mut writer = fs::File::create(output).map_err(|e| e.to_string())?;
57    let mut buf = vec![0u8; CHUNK_SIZE];
58    let mut block_id: u64 = 0;
59    loop {
60        let n = read_partial(&mut reader, &mut buf).map_err(|e| e.to_string())?;
61        if n == 0 {
62            break;
63        }
64        block_id = write_lz4_block(&mut writer, block_id, &buf[..n]).map_err(|e| e.to_string())?;
65    }
66    writer.flush().map_err(|e| e.to_string())?;
67    let output_bytes = fs::metadata(output).map_err(|e| e.to_string())?.len();
68    Ok(CompressStats {
69        input_bytes,
70        output_bytes,
71        blocks: block_id,
72        ratio: if output_bytes > 0 {
73            input_bytes as f64 / output_bytes as f64
74        } else {
75            1.0
76        },
77    })
78}
79
80fn write_lz4_block(w: &mut impl Write, block_id: u64, data: &[u8]) -> std::io::Result<u64> {
81    let compressed = lz4_flex::compress_prepend_size(data);
82    w.write_all(&block_id.to_le_bytes())?;
83    w.write_all(&(compressed.len() as u32).to_le_bytes())?;
84    w.write_all(&(data.len() as u32).to_le_bytes())?;
85    w.write_all(&compressed)?;
86    Ok(block_id + 1)
87}
88
89fn read_partial(r: &mut impl Read, buf: &mut [u8]) -> std::io::Result<usize> {
90    let mut total = 0;
91    while total < buf.len() {
92        match r.read(&mut buf[total..])? {
93            0 => break,
94            n => total += n,
95        }
96    }
97    Ok(total)
98}
qualia_client_core/q42_compress.rs

qualia_client_core/
q42_compress.rs
