Skip to main content

qualia_core_db/specialized_libs/statistical_computing/
compression.rs

1use super::*;
2
3/// Data compression engine
4pub struct DataCompressionEngine {
5    compression_algorithms: Vec<CompressionAlgorithm>,
6    compression_statistics: CompressionStatistics,
7}
8
9/// Compression algorithms
10#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
11pub enum CompressionAlgorithm {
12    Gzip,
13    LZ4,
14    ZSTD,
15    Snappy,
16    Custom(String),
17}
18
19/// Compression statistics
20///
21/// Tracks cumulative metrics across all compression/decompression operations
22/// performed by a [`DataCompressionEngine`]. The `compression_ratio` field
23/// records the ratio of the *most recent* compression operation, while
24/// [`CompressionStatistics::compression_ratio`] computes the *overall* ratio
25/// from the cumulative byte totals.
26#[derive(Debug, Clone)]
27pub struct CompressionStatistics {
28    /// Total original (uncompressed) bytes processed across all compress ops.
29    pub original_size: u64,
30    /// Total compressed bytes produced across all compress ops.
31    pub compressed_size: u64,
32    /// Ratio of the most recent compression operation (compressed / original).
33    pub compression_ratio: f64,
34    /// Total time spent compressing, in nanoseconds.
35    pub compression_time: u64,
36    /// Total time spent decompressing, in nanoseconds.
37    pub decompression_time: u64,
38    /// Number of compression operations performed.
39    pub compression_count: u64,
40    /// Number of decompression operations performed.
41    pub decompression_count: u64,
42}
43
44impl DataCompressionEngine {
45    /// Creates `DataCompressionEngine` with the given state.
46    pub fn new() -> Self {
47        Self {
48            compression_algorithms: vec![CompressionAlgorithm::LZ4, CompressionAlgorithm::ZSTD],
49            compression_statistics: CompressionStatistics::new(),
50        }
51    }
52
53    /// Initializes the component; error propagates.
54    pub fn initialize(&mut self) -> Result<(), StatisticalError> {
55        Ok(())
56    }
57
58    /// Compress `data` using a simple run-length encoding and record the
59    /// operation's statistics (original size, compressed size, ratio, time).
60    pub fn compress(&mut self, data: &[u8]) -> Result<Vec<u8>, StatisticalError> {
61        let start = Instant::now();
62        let compressed = rle_compress(data);
63        let elapsed = start.elapsed().as_nanos() as u64;
64        self.compression_statistics.record_compression(
65            data.len() as u64,
66            compressed.len() as u64,
67            elapsed,
68        );
69        Ok(compressed)
70    }
71
72    /// Decompress data previously produced by [`compress`](Self::compress) and
73    /// record the decompression statistics.
74    pub fn decompress(&mut self, data: &[u8]) -> Result<Vec<u8>, StatisticalError> {
75        let start = Instant::now();
76        let decompressed = rle_decompress(data)?;
77        let elapsed = start.elapsed().as_nanos() as u64;
78        self.compression_statistics.record_decompression(elapsed);
79        Ok(decompressed)
80    }
81
82    /// Returns a reference to the cumulative compression statistics.
83    pub fn get_statistics(&self) -> &CompressionStatistics {
84        &self.compression_statistics
85    }
86
87    /// Resets all accumulated compression statistics to zero.
88    pub fn reset_statistics(&mut self) {
89        self.compression_statistics = CompressionStatistics::new();
90    }
91
92    /// Returns the list of compression algorithms available to this engine.
93    pub fn compression_algorithms(&self) -> &[CompressionAlgorithm] {
94        &self.compression_algorithms
95    }
96
97    /// Register an additional compression algorithm.
98    pub fn add_compression_algorithm(&mut self, algorithm: CompressionAlgorithm) {
99        if !self.compression_algorithms.contains(&algorithm) {
100            self.compression_algorithms.push(algorithm);
101        }
102    }
103
104    /// Returns `true` when the given algorithm is registered.
105    pub fn supports_algorithm(&self, algorithm: &CompressionAlgorithm) -> bool {
106        self.compression_algorithms.contains(algorithm)
107    }
108}
109
110impl CompressionStatistics {
111    /// Create a fresh, zeroed statistics record.
112    pub fn new() -> Self {
113        Self {
114            original_size: 0,
115            compressed_size: 0,
116            compression_ratio: 0.0,
117            compression_time: 0,
118            decompression_time: 0,
119            compression_count: 0,
120            decompression_count: 0,
121        }
122    }
123
124    /// Record a single compression operation.
125    pub fn record_compression(&mut self, original: u64, compressed: u64, elapsed_ns: u64) {
126        self.original_size += original;
127        self.compressed_size += compressed;
128        self.compression_time += elapsed_ns;
129        self.compression_count += 1;
130        self.compression_ratio = if original == 0 {
131            0.0
132        } else {
133            compressed as f64 / original as f64
134        };
135    }
136
137    /// Record a single decompression operation.
138    pub fn record_decompression(&mut self, elapsed_ns: u64) {
139        self.decompression_time += elapsed_ns;
140        self.decompression_count += 1;
141    }
142
143    /// Overall compression ratio across all operations
144    /// (`compressed_size / original_size`). Returns `0.0` when no data has been
145    /// compressed yet.
146    pub fn compression_ratio(&self) -> f64 {
147        if self.original_size == 0 {
148            0.0
149        } else {
150            self.compressed_size as f64 / self.original_size as f64
151        }
152    }
153
154    /// Human-readable summary of the accumulated statistics.
155    pub fn summary(&self) -> String {
156        format!(
157            "CompressionStatistics: {} compress op(s), {} decompress op(s), \
158             original={} bytes, compressed={} bytes, overall ratio={:.4}, \
159             last-op ratio={:.4}, compress_time={} ns, decompress_time={} ns",
160            self.compression_count,
161            self.decompression_count,
162            self.original_size,
163            self.compressed_size,
164            self.compression_ratio(),
165            self.compression_ratio,
166            self.compression_time,
167            self.decompression_time,
168        )
169    }
170}
171
172/// Simple run-length encoding over bytes. Each run is emitted as
173/// `(count: u8, byte: u8)`; runs longer than 255 are split. Incompressible
174/// data expands by ~2x, but repetitive data (the common statistical-dataset
175/// case for constant columns) compresses well.
176fn rle_compress(data: &[u8]) -> Vec<u8> {
177    let mut out = Vec::with_capacity(data.len());
178    let mut i = 0;
179    while i < data.len() {
180        let byte = data[i];
181        let mut count: usize = 1;
182        while i + count < data.len() && data[i + count] == byte && count < 255 {
183            count += 1;
184        }
185        out.push(count as u8);
186        out.push(byte);
187        i += count;
188    }
189    out
190}
191
192/// Inverse of [`rle_compress`].
193fn rle_decompress(data: &[u8]) -> Result<Vec<u8>, StatisticalError> {
194    if data.len() % 2 != 0 {
195        return Err(StatisticalError::InvalidData(
196            "Corrupted RLE stream (odd length)".to_string(),
197        ));
198    }
199    let mut out = Vec::with_capacity(data.len());
200    let mut i = 0;
201    while i < data.len() {
202        let count = data[i] as usize;
203        let byte = data[i + 1];
204        out.resize(out.len() + count, byte);
205        i += 2;
206    }
207    Ok(out)
208}