1use crate::features::log_mel_from_mono;
7use crate::hash::q_hash;
8use crate::types::{AudioError, TranscriptToken};
9
10pub const PHONES: &[&str] = &["sil", "a", "i", "u", "m", "n", "s", "t", "k", "blank"];
12
13#[derive(Debug, Clone)]
14pub struct SpeechEncoderWeights {
15 pub model_hash: u64,
16 pub n_mel: usize,
17 pub n_phones: usize,
18 pub weight: Vec<f32>,
20}
21
22pub const SPEECH_MAGIC: u32 = 0x4850_5153; pub const SPEECH_VERSION: u32 = 1;
24
25impl SpeechEncoderWeights {
26 pub fn from_seed(seed: u64, n_mel: usize) -> Self {
28 let n_phones = PHONES.len();
29 let mut weight = vec![0.0f32; n_phones * n_mel];
30 let mut h = seed ^ 0x5EEC_4010_C0DE_u64;
31 for w in weight.iter_mut() {
32 h = h.wrapping_mul(0x9e37_79b9_7f4a_7c15).rotate_left(7);
33 *w = ((h % 2000) as f32 / 1000.0) - 1.0;
34 }
35 Self {
36 model_hash: q_hash(&format!("qualia-audio-speech-seed-v1:{seed:016x}")),
37 n_mel,
38 n_phones,
39 weight,
40 }
41 }
42
43 pub fn to_bytes(&self) -> Vec<u8> {
45 let mut v = Vec::with_capacity(32 + self.weight.len() * 4);
46 v.extend_from_slice(&SPEECH_MAGIC.to_le_bytes());
47 v.extend_from_slice(&SPEECH_VERSION.to_le_bytes());
48 v.extend_from_slice(&(self.n_mel as u32).to_le_bytes());
49 v.extend_from_slice(&(self.n_phones as u32).to_le_bytes());
50 v.extend_from_slice(&self.model_hash.to_le_bytes());
51 for f in &self.weight {
52 v.extend_from_slice(&f.to_le_bytes());
53 }
54 v
55 }
56
57 pub fn from_bytes(bytes: &[u8]) -> Result<Self, AudioError> {
59 if bytes.len() < 24 {
60 return Err(AudioError::MalformedAudio);
61 }
62 let magic = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
63 if magic != SPEECH_MAGIC {
64 return Err(AudioError::BackendUnavailable);
65 }
66 let n_mel = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize;
67 let n_phones = u32::from_le_bytes([bytes[12], bytes[13], bytes[14], bytes[15]]) as usize;
68 let model_hash = u64::from_le_bytes(
69 bytes[16..24]
70 .try_into()
71 .map_err(|_| crate::types::AudioError::MalformedAudio)?,
72 );
73 let need = 24 + n_mel * n_phones * 4;
74 if bytes.len() < need || n_phones != PHONES.len() || n_mel == 0 || n_mel > 128 {
75 return Err(AudioError::MalformedAudio);
76 }
77 let mut weight = vec![0.0f32; n_mel * n_phones];
78 let mut off = 24usize;
79 for w in weight.iter_mut() {
80 *w = f32::from_le_bytes(
81 bytes[off..off + 4]
82 .try_into()
83 .map_err(|_| crate::types::AudioError::MalformedAudio)?,
84 );
85 off += 4;
86 }
87 Ok(Self {
88 model_hash,
89 n_mel,
90 n_phones,
91 weight,
92 })
93 }
94
95 pub fn save_path(&self, path: &std::path::Path) -> Result<(), String> {
97 if let Some(p) = path.parent() {
98 std::fs::create_dir_all(p).map_err(|e| e.to_string())?;
99 }
100 std::fs::write(path, self.to_bytes()).map_err(|e| e.to_string())
101 }
102
103 pub fn load_path(path: &std::path::Path) -> Result<Self, String> {
105 let b = std::fs::read(path).map_err(|e| e.to_string())?;
106 Self::from_bytes(&b).map_err(|e| format!("{e:?}"))
107 }
108}
109
110pub fn greedy_phone_decode(
112 weights: &SpeechEncoderWeights,
113 mono: &[f32],
114 sample_rate: u32,
115 hop: usize,
116 out: &mut [TranscriptToken],
117) -> Result<usize, AudioError> {
118 let max_frames = 64;
119 let mut mel = vec![0.0f32; max_frames * weights.n_mel];
120 let n_frames = log_mel_from_mono(mono, 256, hop.max(64), sample_rate, weights.n_mel, &mut mel)?;
121 let n_frames = n_frames.min(max_frames);
122 let blank = weights.n_phones - 1;
123 let mut prev = blank;
124 let mut w = 0usize;
125 let mut frame_start = 0u64;
126 for f in 0..n_frames {
127 let row = &mel[f * weights.n_mel..(f + 1) * weights.n_mel];
128 let mut best_p = 0usize;
129 let mut best = f32::NEG_INFINITY;
130 for p in 0..weights.n_phones {
131 let mut logit = 0.0f32;
132 for i in 0..weights.n_mel {
133 logit += weights.weight[p * weights.n_mel + i] * row[i];
134 }
135 if logit > best {
136 best = logit;
137 best_p = p;
138 }
139 }
140 let conf = ((best.tanh() + 1.0) * 0.5).clamp(0.05, 0.99);
141 if best_p != blank && best_p != prev && w < out.len() {
142 let mut t = TranscriptToken::empty();
143 t.form_hash = q_hash(PHONES[best_p]);
144 t.confidence_u16 = (conf * 65535.0) as u16;
145 t.start_frame = frame_start;
146 t.end_frame = frame_start + hop as u64;
147 t.flags = 2; out[w] = t;
149 w += 1;
150 }
151 prev = best_p;
152 frame_start += hop as u64;
153 }
154 for t in out.iter_mut().skip(w) {
155 *t = TranscriptToken::empty();
156 }
157 Ok(w)
158}
159
160pub struct StreamingSpeechDecoder {
162 weights: SpeechEncoderWeights,
163 prev_phone: usize,
164 frame_cursor: u64,
165 hop: usize,
166}
167
168impl StreamingSpeechDecoder {
169 pub fn new(weights: SpeechEncoderWeights, hop: usize) -> Self {
171 let blank = weights.n_phones.saturating_sub(1);
172 Self {
173 weights,
174 prev_phone: blank,
175 frame_cursor: 0,
176 hop: hop.max(64),
177 }
178 }
179
180 pub fn model_hash(&self) -> u64 {
182 self.weights.model_hash
183 }
184
185 pub fn push_chunk(
187 &mut self,
188 mono: &[f32],
189 sample_rate: u32,
190 out: &mut [TranscriptToken],
191 ) -> Result<usize, AudioError> {
192 let max_frames = 32;
193 let mut mel = vec![0.0f32; max_frames * self.weights.n_mel];
194 let n_frames = log_mel_from_mono(
195 mono,
196 256,
197 self.hop,
198 sample_rate,
199 self.weights.n_mel,
200 &mut mel,
201 )?;
202 let n_frames = n_frames.min(max_frames);
203 let blank = self.weights.n_phones - 1;
204 let mut w = 0usize;
205 for f in 0..n_frames {
206 let row = &mel[f * self.weights.n_mel..(f + 1) * self.weights.n_mel];
207 let mut best_p = 0usize;
208 let mut best = f32::NEG_INFINITY;
209 for p in 0..self.weights.n_phones {
210 let mut logit = 0.0f32;
211 for i in 0..self.weights.n_mel {
212 logit += self.weights.weight[p * self.weights.n_mel + i] * row[i];
213 }
214 if logit > best {
215 best = logit;
216 best_p = p;
217 }
218 }
219 let conf = ((best.tanh() + 1.0) * 0.5).clamp(0.05, 0.99);
220 if best_p != blank && best_p != self.prev_phone && w < out.len() {
221 let mut t = TranscriptToken::empty();
222 t.form_hash = q_hash(PHONES[best_p]);
223 t.confidence_u16 = (conf * 65535.0) as u16;
224 t.start_frame = self.frame_cursor;
225 t.end_frame = self.frame_cursor + self.hop as u64;
226 t.flags = 2;
227 out[w] = t;
228 w += 1;
229 }
230 self.prev_phone = best_p;
231 self.frame_cursor += self.hop as u64;
232 }
233 for t in out.iter_mut().skip(w) {
234 *t = TranscriptToken::empty();
235 }
236 Ok(w)
237 }
238}
239
240pub fn decode_for_language(
242 weights: &SpeechEncoderWeights,
243 mono: &[f32],
244 sample_rate: u32,
245 language_supported: bool,
246 out: &mut [TranscriptToken],
247) -> Result<usize, AudioError> {
248 if !language_supported {
249 for t in out.iter_mut() {
251 *t = TranscriptToken::empty();
252 }
253 return Ok(0);
254 }
255 greedy_phone_decode(weights, mono, sample_rate, 128, out)
256}
257
258#[cfg(test)]
259mod tests {
260 use super::*;
261
262 #[test]
263 fn unknown_language_empty() {
264 let w = SpeechEncoderWeights::from_seed(1, 16);
265 let mono = [0.1f32; 2048];
266 let mut out = [TranscriptToken::empty(); 16];
267 let n = decode_for_language(&w, &mono, 16000, false, &mut out).unwrap();
268 assert_eq!(n, 0);
269 }
270
271 #[test]
272 fn supported_may_emit() {
273 let w = SpeechEncoderWeights::from_seed(2, 16);
274 let mut mono = vec![0.0f32; 4096];
275 for i in 0..mono.len() {
276 mono[i] = (2.0 * core::f32::consts::PI * 200.0 * i as f32 / 16000.0).sin() * 0.3;
277 }
278 let mut out = [TranscriptToken::empty(); 32];
279 let n = decode_for_language(&w, &mono, 16000, true, &mut out).unwrap();
280 assert!(n <= 32);
282 }
283
284 #[test]
285 fn speech_weight_roundtrip() {
286 let w = SpeechEncoderWeights::from_seed(9, 16);
287 let b = w.to_bytes();
288 let w2 = SpeechEncoderWeights::from_bytes(&b).unwrap();
289 assert_eq!(w.model_hash, w2.model_hash);
290 assert_eq!(w.weight, w2.weight);
291 }
292
293 #[test]
294 fn streaming_decoder_runs() {
295 let w = SpeechEncoderWeights::from_seed(3, 16);
296 let mut dec = StreamingSpeechDecoder::new(w, 128);
297 let mono = [0.2f32; 2048];
298 let mut out = [TranscriptToken::empty(); 16];
299 let n = dec.push_chunk(&mono, 16000, &mut out).unwrap();
300 assert!(n <= 16);
301 assert!(dec.model_hash() != 0);
302 }
303}