Skip to main content

qualia_audio/
speech.rs

1//! Speech path scaffold — greedy "phone" decode over log-mel (not full ASR).
2//!
3//! COMPLETE-WITH-GATE: seed matrix, not a licensed streaming encoder. Unknown
4//! language must not map silently — we emit only inventory phones or blank.
5
6use crate::features::log_mel_from_mono;
7use crate::hash::q_hash;
8use crate::types::{AudioError, TranscriptToken};
9
10/// Closed phone inventory (demo).
11pub const PHONES: &[&str] = &["sil", "a", "i", "u", "m", "n", "s", "t", "k", "blank"];
12
13#[derive(Debug, Clone)]
14pub struct SpeechEncoderWeights {
15    pub model_hash: u64,
16    pub n_mel: usize,
17    pub n_phones: usize,
18    /// Row-major [n_phones * n_mel]
19    pub weight: Vec<f32>,
20}
21
22pub const SPEECH_MAGIC: u32 = 0x4850_5153; // 'SQPH' LE-ish
23pub const SPEECH_VERSION: u32 = 1;
24
25impl SpeechEncoderWeights {
26    /// Constructs from seed.
27    pub fn from_seed(seed: u64, n_mel: usize) -> Self {
28        let n_phones = PHONES.len();
29        let mut weight = vec![0.0f32; n_phones * n_mel];
30        let mut h = seed ^ 0x5EEC_4010_C0DE_u64;
31        for w in weight.iter_mut() {
32            h = h.wrapping_mul(0x9e37_79b9_7f4a_7c15).rotate_left(7);
33            *w = ((h % 2000) as f32 / 1000.0) - 1.0;
34        }
35        Self {
36            model_hash: q_hash(&format!("qualia-audio-speech-seed-v1:{seed:016x}")),
37            n_mel,
38            n_phones,
39            weight,
40        }
41    }
42
43    /// Byte form.
44    pub fn to_bytes(&self) -> Vec<u8> {
45        let mut v = Vec::with_capacity(32 + self.weight.len() * 4);
46        v.extend_from_slice(&SPEECH_MAGIC.to_le_bytes());
47        v.extend_from_slice(&SPEECH_VERSION.to_le_bytes());
48        v.extend_from_slice(&(self.n_mel as u32).to_le_bytes());
49        v.extend_from_slice(&(self.n_phones as u32).to_le_bytes());
50        v.extend_from_slice(&self.model_hash.to_le_bytes());
51        for f in &self.weight {
52            v.extend_from_slice(&f.to_le_bytes());
53        }
54        v
55    }
56
57    /// Parses bytes.
58    pub fn from_bytes(bytes: &[u8]) -> Result<Self, AudioError> {
59        if bytes.len() < 24 {
60            return Err(AudioError::MalformedAudio);
61        }
62        let magic = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
63        if magic != SPEECH_MAGIC {
64            return Err(AudioError::BackendUnavailable);
65        }
66        let n_mel = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;
67        let n_phones = u32::from_le_bytes([bytes[12], bytes[13], bytes[14], bytes[15]]) as usize;
68        let model_hash = u64::from_le_bytes(
69            bytes[16..24]
70                .try_into()
71                .map_err(|_| crate::types::AudioError::MalformedAudio)?,
72        );
73        let need = 24 + n_mel * n_phones * 4;
74        if bytes.len() < need || n_phones != PHONES.len() || n_mel == 0 || n_mel > 128 {
75            return Err(AudioError::MalformedAudio);
76        }
77        let mut weight = vec![0.0f32; n_mel * n_phones];
78        let mut off = 24usize;
79        for w in weight.iter_mut() {
80            *w = f32::from_le_bytes(
81                bytes[off..off + 4]
82                    .try_into()
83                    .map_err(|_| crate::types::AudioError::MalformedAudio)?,
84            );
85            off += 4;
86        }
87        Ok(Self {
88            model_hash,
89            n_mel,
90            n_phones,
91            weight,
92        })
93    }
94
95    /// Saves path.
96    pub fn save_path(&self, path: &std::path::Path) -> Result<(), String> {
97        if let Some(p) = path.parent() {
98            std::fs::create_dir_all(p).map_err(|e| e.to_string())?;
99        }
100        std::fs::write(path, self.to_bytes()).map_err(|e| e.to_string())
101    }
102
103    /// Loads path.
104    pub fn load_path(path: &std::path::Path) -> Result<Self, String> {
105        let b = std::fs::read(path).map_err(|e| e.to_string())?;
106        Self::from_bytes(&b).map_err(|e| format!("{e:?}"))
107    }
108}
109
110/// Greedy CTC-like collapse over frames of argmax phone.
111pub fn greedy_phone_decode(
112    weights: &SpeechEncoderWeights,
113    mono: &[f32],
114    sample_rate: u32,
115    hop: usize,
116    out: &mut [TranscriptToken],
117) -> Result<usize, AudioError> {
118    let max_frames = 64;
119    let mut mel = vec![0.0f32; max_frames * weights.n_mel];
120    let n_frames = log_mel_from_mono(mono, 256, hop.max(64), sample_rate, weights.n_mel, &mut mel)?;
121    let n_frames = n_frames.min(max_frames);
122    let blank = weights.n_phones - 1;
123    let mut prev = blank;
124    let mut w = 0usize;
125    let mut frame_start = 0u64;
126    for f in 0..n_frames {
127        let row = &mel[f * weights.n_mel..(f + 1) * weights.n_mel];
128        let mut best_p = 0usize;
129        let mut best = f32::NEG_INFINITY;
130        for p in 0..weights.n_phones {
131            let mut logit = 0.0f32;
132            for i in 0..weights.n_mel {
133                logit += weights.weight[p * weights.n_mel + i] * row[i];
134            }
135            if logit > best {
136                best = logit;
137                best_p = p;
138            }
139        }
140        let conf = ((best.tanh() + 1.0) * 0.5).clamp(0.05, 0.99);
141        if best_p != blank && best_p != prev && w < out.len() {
142            let mut t = TranscriptToken::empty();
143            t.form_hash = q_hash(PHONES[best_p]);
144            t.confidence_u16 = (conf * 65535.0) as u16;
145            t.start_frame = frame_start;
146            t.end_frame = frame_start + hop as u64;
147            t.flags = 2; // low assurance proposal
148            out[w] = t;
149            w += 1;
150        }
151        prev = best_p;
152        frame_start += hop as u64;
153    }
154    for t in out.iter_mut().skip(w) {
155        *t = TranscriptToken::empty();
156    }
157    Ok(w)
158}
159
160/// Streaming CTC-greedy state: feed successive mono chunks; flush tokens.
161pub struct StreamingSpeechDecoder {
162    weights: SpeechEncoderWeights,
163    prev_phone: usize,
164    frame_cursor: u64,
165    hop: usize,
166}
167
168impl StreamingSpeechDecoder {
169    /// Creates a new instance.
170    pub fn new(weights: SpeechEncoderWeights, hop: usize) -> Self {
171        let blank = weights.n_phones.saturating_sub(1);
172        Self {
173            weights,
174            prev_phone: blank,
175            frame_cursor: 0,
176            hop: hop.max(64),
177        }
178    }
179
180    /// `model` hash.
181    pub fn model_hash(&self) -> u64 {
182        self.weights.model_hash
183    }
184
185    /// Decode one chunk into `out` (appended logical tokens; buffer overwritten from 0).
186    pub fn push_chunk(
187        &mut self,
188        mono: &[f32],
189        sample_rate: u32,
190        out: &mut [TranscriptToken],
191    ) -> Result<usize, AudioError> {
192        let max_frames = 32;
193        let mut mel = vec![0.0f32; max_frames * self.weights.n_mel];
194        let n_frames = log_mel_from_mono(
195            mono,
196            256,
197            self.hop,
198            sample_rate,
199            self.weights.n_mel,
200            &mut mel,
201        )?;
202        let n_frames = n_frames.min(max_frames);
203        let blank = self.weights.n_phones - 1;
204        let mut w = 0usize;
205        for f in 0..n_frames {
206            let row = &mel[f * self.weights.n_mel..(f + 1) * self.weights.n_mel];
207            let mut best_p = 0usize;
208            let mut best = f32::NEG_INFINITY;
209            for p in 0..self.weights.n_phones {
210                let mut logit = 0.0f32;
211                for i in 0..self.weights.n_mel {
212                    logit += self.weights.weight[p * self.weights.n_mel + i] * row[i];
213                }
214                if logit > best {
215                    best = logit;
216                    best_p = p;
217                }
218            }
219            let conf = ((best.tanh() + 1.0) * 0.5).clamp(0.05, 0.99);
220            if best_p != blank && best_p != self.prev_phone && w < out.len() {
221                let mut t = TranscriptToken::empty();
222                t.form_hash = q_hash(PHONES[best_p]);
223                t.confidence_u16 = (conf * 65535.0) as u16;
224                t.start_frame = self.frame_cursor;
225                t.end_frame = self.frame_cursor + self.hop as u64;
226                t.flags = 2;
227                out[w] = t;
228                w += 1;
229            }
230            self.prev_phone = best_p;
231            self.frame_cursor += self.hop as u64;
232        }
233        for t in out.iter_mut().skip(w) {
234            *t = TranscriptToken::empty();
235        }
236        Ok(w)
237    }
238}
239
240/// If language inventory is empty / unknown, refuse mapping (no silent map).
241pub fn decode_for_language(
242    weights: &SpeechEncoderWeights,
243    mono: &[f32],
244    sample_rate: u32,
245    language_supported: bool,
246    out: &mut [TranscriptToken],
247) -> Result<usize, AudioError> {
248    if !language_supported {
249        // Leave empty — unknown language stays untranscribed.
250        for t in out.iter_mut() {
251            *t = TranscriptToken::empty();
252        }
253        return Ok(0);
254    }
255    greedy_phone_decode(weights, mono, sample_rate, 128, out)
256}
257
258#[cfg(test)]
259mod tests {
260    use super::*;
261
262    #[test]
263    fn unknown_language_empty() {
264        let w = SpeechEncoderWeights::from_seed(1, 16);
265        let mono = [0.1f32; 2048];
266        let mut out = [TranscriptToken::empty(); 16];
267        let n = decode_for_language(&w, &mono, 16000, false, &mut out).unwrap();
268        assert_eq!(n, 0);
269    }
270
271    #[test]
272    fn supported_may_emit() {
273        let w = SpeechEncoderWeights::from_seed(2, 16);
274        let mut mono = vec![0.0f32; 4096];
275        for i in 0..mono.len() {
276            mono[i] = (2.0 * core::f32::consts::PI * 200.0 * i as f32 / 16000.0).sin() * 0.3;
277        }
278        let mut out = [TranscriptToken::empty(); 32];
279        let n = decode_for_language(&w, &mono, 16000, true, &mut out).unwrap();
280        // May be 0 if all blank — still ok; path exercised
281        assert!(n <= 32);
282    }
283
284    #[test]
285    fn speech_weight_roundtrip() {
286        let w = SpeechEncoderWeights::from_seed(9, 16);
287        let b = w.to_bytes();
288        let w2 = SpeechEncoderWeights::from_bytes(&b).unwrap();
289        assert_eq!(w.model_hash, w2.model_hash);
290        assert_eq!(w.weight, w2.weight);
291    }
292
293    #[test]
294    fn streaming_decoder_runs() {
295        let w = SpeechEncoderWeights::from_seed(3, 16);
296        let mut dec = StreamingSpeechDecoder::new(w, 128);
297        let mono = [0.2f32; 2048];
298        let mut out = [TranscriptToken::empty(); 16];
299        let n = dec.push_chunk(&mono, 16000, &mut out).unwrap();
300        assert!(n <= 16);
301        assert!(dec.model_hash() != 0);
302    }
303}