Skip to main content

qualia_audio/models/audio_tokens/
rvq.rs

1//! Residual Vector Quantizer (RVQ) — the codec **structure** used by discrete audio token
2//! models (EnCodec / SoundStream / DAC family). This is pure codebook math: no learned weights
3//! are required, so it is real and buildable now. The caller supplies the codebooks (each stage
4//! a `[n_centroids × dim]` row-major block); the neural encoder/decoder that *produces* those
5//! codebooks and the latent vectors is a separate, `NeedsWeights` concern (out of scope here).
6//!
7//! Encoding is greedy per stage: at stage `s` we pick the centroid nearest (squared L2) to the
8//! *running residual* — the input minus the centroids already chosen at stages `0..s`. Decoding
9//! sums the chosen centroids back. Zero-heap: no scratch is allocated; the running residual is
10//! recomputed on the fly from the already-chosen tokens (stage/dim counts are small).
11
12use crate::types::AudioError;
13
14/// Row-major offset of `centroid`'s `d`-th component inside `stage`'s codebook block.
15#[inline]
16fn cb_index(stage: usize, centroid: usize, d: usize, n_centroids: usize, dim: usize) -> usize {
17    ((stage * n_centroids) + centroid) * dim + d
18}
19
20/// Value of `input[d]` minus the reconstruction from tokens chosen at stages `0..up_to`.
21#[inline]
22fn residual_component(
23    input: &[f32],
24    codebooks: &[f32],
25    tokens: &[u16],
26    up_to: usize,
27    d: usize,
28    n_centroids: usize,
29    dim: usize,
30) -> f32 {
31    let mut r = input[d];
32    for p in 0..up_to {
33        let c = tokens[p] as usize;
34        r -= codebooks[cb_index(p, c, d, n_centroids, dim)];
35    }
36    r
37}
38
39/// Encode one `dim`-vector into `n_stages` residual token indices.
40///
41/// - `input`: at least `dim` floats (one latent vector).
42/// - `codebooks`: `n_stages × n_centroids × dim` floats, row-major, stage-major:
43///   stage `s`, centroid `c`, component `d` lives at `((s*n_centroids)+c)*dim + d`.
44/// - `out_tokens`: at least `n_stages` slots; slot `s` receives the chosen centroid index
45///   (`< n_centroids`, hence `≤ 65535`).
46///
47/// Returns the number of tokens written (`n_stages`). Zero-heap.
48///
49/// # Errors
50/// - [`AudioError::InvalidParameter`] if any of `n_stages`, `n_centroids`, `dim` is 0, or
51///   `n_centroids > 65536` (index would not fit `u16`).
52/// - [`AudioError::MalformedAudio`] if `input` is shorter than `dim` or `codebooks` is shorter
53///   than `n_stages * n_centroids * dim`.
54/// - [`AudioError::OutputBufferTooSmall`] if `out_tokens` has fewer than `n_stages` slots.
55pub fn rvq_quantize(
56    input: &[f32],
57    codebooks: &[f32],
58    n_stages: usize,
59    n_centroids: usize,
60    dim: usize,
61    out_tokens: &mut [u16],
62) -> Result<usize, AudioError> {
63    if n_stages == 0 || n_centroids == 0 || dim == 0 || n_centroids > 65_536 {
64        return Err(AudioError::InvalidParameter);
65    }
66    if input.len() < dim || codebooks.len() < n_stages * n_centroids * dim {
67        return Err(AudioError::MalformedAudio);
68    }
69    if out_tokens.len() < n_stages {
70        return Err(AudioError::OutputBufferTooSmall);
71    }
72
73    for s in 0..n_stages {
74        let mut best_c = 0usize;
75        let mut best_dist = f32::INFINITY;
76        for c in 0..n_centroids {
77            let mut dist = 0.0f32;
78            for d in 0..dim {
79                let r = residual_component(input, codebooks, out_tokens, s, d, n_centroids, dim);
80                let diff = r - codebooks[cb_index(s, c, d, n_centroids, dim)];
81                dist += diff * diff;
82            }
83            if dist < best_dist {
84                best_dist = dist;
85                best_c = c;
86            }
87        }
88        out_tokens[s] = best_c as u16;
89    }
90    Ok(n_stages)
91}
92
93/// Decode `n_stages` residual token indices back into a `dim`-vector by summing the chosen
94/// centroids across all stages.
95///
96/// - `tokens`: at least `n_stages` centroid indices (each `< n_centroids`).
97/// - `codebooks`: same layout as [`rvq_quantize`].
98/// - `out`: at least `dim` floats; fully overwritten.
99///
100/// # Errors
101/// - [`AudioError::InvalidParameter`] if any of `n_stages`, `n_centroids`, `dim` is 0.
102/// - [`AudioError::MalformedAudio`] if `codebooks` is short, or a token indexes `≥ n_centroids`.
103/// - [`AudioError::OutputBufferTooSmall`] if `tokens` or `out` are too short.
104pub fn rvq_dequantize(
105    tokens: &[u16],
106    codebooks: &[f32],
107    n_stages: usize,
108    n_centroids: usize,
109    dim: usize,
110    out: &mut [f32],
111) -> Result<(), AudioError> {
112    if n_stages == 0 || n_centroids == 0 || dim == 0 {
113        return Err(AudioError::InvalidParameter);
114    }
115    if codebooks.len() < n_stages * n_centroids * dim {
116        return Err(AudioError::MalformedAudio);
117    }
118    if tokens.len() < n_stages || out.len() < dim {
119        return Err(AudioError::OutputBufferTooSmall);
120    }
121    for d in 0..dim {
122        out[d] = 0.0;
123    }
124    for (s, &tok) in tokens.iter().enumerate().take(n_stages) {
125        let c = tok as usize;
126        if c >= n_centroids {
127            return Err(AudioError::MalformedAudio);
128        }
129        for d in 0..dim {
130            out[d] += codebooks[cb_index(s, c, d, n_centroids, dim)];
131        }
132    }
133    Ok(())
134}
135
136#[cfg(test)]
137mod tests {
138    use super::*;
139
140    // Two stages, 2 centroids each, dim = 2.
141    // Stage 0 codebook: c0 = (0,0), c1 = (1,1)  -> coarse
142    // Stage 1 codebook: c0 = (-0.1,0.1), c1 = (0.1,-0.1) -> fine residual
143    fn small_codebooks() -> (Vec<f32>, usize, usize, usize) {
144        let cb = vec![
145            // stage 0
146            0.0, 0.0, // c0
147            1.0, 1.0, // c1
148            // stage 1
149            -0.1, 0.1, // c0
150            0.1, -0.1, // c1
151        ];
152        (cb, 2, 2, 2) // codebooks, n_stages, n_centroids, dim
153    }
154
155    #[test]
156    fn centroid_sum_round_trips_exactly() {
157        // Input exactly equal to c1(stage0) + c1(stage1) = (1.1, 0.9).
158        let (cb, n_stages, n_centroids, dim) = small_codebooks();
159        let input = [1.1f32, 0.9];
160        let mut tokens = [0u16; 2];
161        let n = rvq_quantize(&input, &cb, n_stages, n_centroids, dim, &mut tokens).expect("quant");
162        assert_eq!(n, 2);
163        assert_eq!(tokens, [1, 1], "should pick coarse c1 then fine c1");
164
165        let mut recon = [0.0f32; 2];
166        rvq_dequantize(&tokens, &cb, n_stages, n_centroids, dim, &mut recon).expect("dequant");
167        assert!((recon[0] - 1.1).abs() < 1e-6, "recon0={}", recon[0]);
168        assert!((recon[1] - 0.9).abs() < 1e-6, "recon1={}", recon[1]);
169    }
170
171    #[test]
172    fn reconstruction_within_codebook_resolution() {
173        // Arbitrary input near the (1,1) coarse cell; error must be within the fine step.
174        let (cb, n_stages, n_centroids, dim) = small_codebooks();
175        let input = [1.08f32, 0.92];
176        let mut tokens = [0u16; 2];
177        rvq_quantize(&input, &cb, n_stages, n_centroids, dim, &mut tokens).expect("quant");
178        let mut recon = [0.0f32; 2];
179        rvq_dequantize(&tokens, &cb, n_stages, n_centroids, dim, &mut recon).expect("dequant");
180        let err = ((recon[0] - input[0]).powi(2) + (recon[1] - input[1]).powi(2)).sqrt();
181        // Fine centroids have magnitude ~0.14; residual after two stages stays below it.
182        assert!(
183            err < 0.15,
184            "reconstruction error {err} exceeded resolution bound"
185        );
186    }
187
188    #[test]
189    fn adding_stages_does_not_increase_error() {
190        // One-stage reconstruction error >= two-stage error (greedy residual refinement).
191        let (cb, _n_stages, n_centroids, dim) = small_codebooks();
192        let input = [0.9f32, 1.05];
193        let mut t1 = [0u16; 1];
194        rvq_quantize(&input, &cb, 1, n_centroids, dim, &mut t1).expect("q1");
195        let mut r1 = [0.0f32; 2];
196        rvq_dequantize(&t1, &cb, 1, n_centroids, dim, &mut r1).expect("d1");
197        let e1 = ((r1[0] - input[0]).powi(2) + (r1[1] - input[1]).powi(2)).sqrt();
198
199        let mut t2 = [0u16; 2];
200        rvq_quantize(&input, &cb, 2, n_centroids, dim, &mut t2).expect("q2");
201        let mut r2 = [0.0f32; 2];
202        rvq_dequantize(&t2, &cb, 2, n_centroids, dim, &mut r2).expect("d2");
203        let e2 = ((r2[0] - input[0]).powi(2) + (r2[1] - input[1]).powi(2)).sqrt();
204
205        assert!(
206            e2 <= e1 + 1e-6,
207            "two-stage error {e2} worse than one-stage {e1}"
208        );
209    }
210
211    #[test]
212    fn rejects_bad_shapes() {
213        let (cb, n_stages, n_centroids, dim) = small_codebooks();
214        let input = [0.0f32; 2];
215        let mut tokens = [0u16; 2];
216        assert_eq!(
217            rvq_quantize(&input, &cb, 0, n_centroids, dim, &mut tokens),
218            Err(AudioError::InvalidParameter)
219        );
220        // input too short
221        let short = [0.0f32; 1];
222        assert_eq!(
223            rvq_quantize(&short, &cb, n_stages, n_centroids, dim, &mut tokens),
224            Err(AudioError::MalformedAudio)
225        );
226        // out_tokens too small
227        let mut one = [0u16; 1];
228        assert_eq!(
229            rvq_quantize(&input, &cb, n_stages, n_centroids, dim, &mut one),
230            Err(AudioError::OutputBufferTooSmall)
231        );
232    }
233
234    #[test]
235    fn dequantize_rejects_out_of_range_token() {
236        let (cb, n_stages, n_centroids, dim) = small_codebooks();
237        let bad = [5u16, 0];
238        let mut out = [0.0f32; 2];
239        assert_eq!(
240            rvq_dequantize(&bad, &cb, n_stages, n_centroids, dim, &mut out),
241            Err(AudioError::MalformedAudio)
242        );
243    }
244}