qualia_cli/
compress.rs

1//! LZ4 block-stream compressor for browser-deployable dataset artifacts.
2//!
3//! Unified v3 `.q42` volumes already embed LZ4-compressed SuperBlocks; the
4//! compress command copies them unchanged. Legacy v1 raw SuperBlock streams are
5//! still converted to the framed transport format for the browser VFS:
6//!
7//! ```text
8//! Per block:
9//!   [block_id:   u64 LE]
10//!   [comp_len:   u32 LE]
11//!   [uncomp_len: u32 LE]
12//!   [payload:    comp_len bytes]   — lz4_flex::compress_prepend_size output
13//! ```
14
15use std::fs::{self, File, OpenOptions};
16use std::io::{BufReader, BufWriter, Read, Write};
17use std::path::Path;
18
19/// Bytes per compressed chunk.  8 192 × 48 = 393 216 — the same size used by
20/// the `import` pipeline, so the browser decoder is identical for both sources.
21const CHUNK_SIZE: usize = 393_216;
22
23/// SuperBlock layout constants (must match qualia-core-db).
24const SUPERBLOCK_SIZE: usize = 40_960;
25const SUPERBLOCK_HEADER: usize = 160;
26const QUINS_PER_BLOCK: usize = 850;
27const QUIN_SIZE: usize = 48;
28const QUIN_DATA_PER_BLOCK: usize = QUINS_PER_BLOCK * QUIN_SIZE; // 40 800 bytes
29
30pub struct CompressStats {
31    pub input_bytes: u64,
32    pub output_bytes: u64,
33    pub blocks: u64,
34    pub ratio: f64,
35}
36
37/// Compress a `.q42` file. Unified v3 volumes are copied as-is; legacy v1
38/// SuperBlock streams are stripped and re-framed for the browser VFS.
39pub fn compress_q42(
40    input: &Path,
41    output: &Path,
42) -> Result<CompressStats, Box<dyn std::error::Error>> {
43    if qualia_core_db::q42_volume::is_unified_volume(input)? {
44        let volume = qualia_core_db::q42_volume::Q42Volume::open(input)?;
45        if volume.volume_manifest()?.is_some() {
46            return Err("cannot copy a logical Q42 root as one file: publish/copy its manifest-attested child and lexicon segments together".into());
47        }
48        fs::copy(input, output)?;
49        let input_bytes = fs::metadata(input)?.len();
50        let output_bytes = fs::metadata(output)?.len();
51        return Ok(CompressStats {
52            input_bytes,
53            output_bytes,
54            blocks: volume.block_count(),
55            ratio: 1.0,
56        });
57    }
58
59    let meta = fs::metadata(input)?;
60    let input_bytes = meta.len();
61
62    let mut reader = BufReader::new(File::open(input)?);
63    let out_file = OpenOptions::new()
64        .create(true)
65        .write(true)
66        .truncate(true)
67        .open(output)?;
68    let mut writer = BufWriter::new(out_file);
69
70    let mut chunk: Vec<u8> = Vec::with_capacity(CHUNK_SIZE);
71    let mut block_id: u64 = 0;
72    let mut sb_buf = vec![0u8; SUPERBLOCK_SIZE];
73
74    loop {
75        let n = read_exact_or_eof(&mut reader, &mut sb_buf)?;
76        if n == 0 {
77            break;
78        }
79        if n < SUPERBLOCK_SIZE {
80            break;
81        }
82
83        chunk
84            .extend_from_slice(&sb_buf[SUPERBLOCK_HEADER..SUPERBLOCK_HEADER + QUIN_DATA_PER_BLOCK]);
85
86        if chunk.len() >= CHUNK_SIZE {
87            block_id = write_lz4_block(&mut writer, block_id, &chunk[..CHUNK_SIZE])?;
88            chunk.drain(..CHUNK_SIZE);
89        }
90    }
91
92    if !chunk.is_empty() {
93        block_id = write_lz4_block(&mut writer, block_id, &chunk)?;
94    }
95
96    writer.flush()?;
97    let output_bytes = fs::metadata(output)?.len();
98
99    Ok(CompressStats {
100        input_bytes,
101        output_bytes,
102        blocks: block_id,
103        ratio: input_bytes as f64 / output_bytes as f64,
104    })
105}
106
107/// Compress any binary file (e.g. legacy `.lex` sidecar) as raw bytes.
108pub fn compress_raw(
109    input: &Path,
110    output: &Path,
111) -> Result<CompressStats, Box<dyn std::error::Error>> {
112    let meta = fs::metadata(input)?;
113    let input_bytes = meta.len();
114
115    let mut reader = BufReader::new(File::open(input)?);
116    let out_file = OpenOptions::new()
117        .create(true)
118        .write(true)
119        .truncate(true)
120        .open(output)?;
121    let mut writer = BufWriter::new(out_file);
122
123    let mut buf = vec![0u8; CHUNK_SIZE];
124    let mut block_id: u64 = 0;
125
126    loop {
127        let n = read_exact_or_eof(&mut reader, &mut buf)?;
128        if n == 0 {
129            break;
130        }
131        block_id = write_lz4_block(&mut writer, block_id, &buf[..n])?;
132    }
133
134    writer.flush()?;
135    let output_bytes = fs::metadata(output)?.len();
136
137    Ok(CompressStats {
138        input_bytes,
139        output_bytes,
140        blocks: block_id,
141        ratio: input_bytes as f64 / output_bytes as f64,
142    })
143}
144
145fn write_lz4_block(w: &mut impl Write, block_id: u64, data: &[u8]) -> std::io::Result<u64> {
146    let compressed = lz4_flex::compress_prepend_size(data);
147    w.write_all(&block_id.to_le_bytes())?;
148    w.write_all(&(compressed.len() as u32).to_le_bytes())?;
149    w.write_all(&(data.len() as u32).to_le_bytes())?;
150    w.write_all(&compressed)?;
151    Ok(block_id + 1)
152}
153
154fn read_exact_or_eof(r: &mut impl Read, buf: &mut [u8]) -> std::io::Result<usize> {
155    let mut total = 0;
156    while total < buf.len() {
157        match r.read(&mut buf[total..])? {
158            0 => break,
159            n => total += n,
160        }
161    }
162    Ok(total)
163}
qualia_cli/compress.rs

qualia_cli/
compress.rs
