1use crate::convert::to_mono_f32;
4use crate::events::{CLASS_NOISE, CLASS_SILENCE, CLASS_SPEECH_LIKE, CLASS_TONAL};
5use crate::features::{frame_energy, frame_zcr};
6use crate::hash::q_hash;
7use crate::types::{
8 AudioError, AudioView, AuditoryCapabilities, AuditoryEvent, AuditoryModel,
9 AuditoryOutputCounts, TranscriptToken, MAX_EVENTS,
10};
11
12const MODEL: &str = "qualia-audio-aed-weight-v1";
13
14#[derive(Debug, Clone)]
16pub struct AedWeightBundle {
17 pub model_hash: u64,
18 pub weight: [f32; 16],
20 pub bias: [f32; 4],
21 pub class_hashes: [u64; 4],
22}
23
24pub const AED_MAGIC: u32 = 0x4445_4151;
26pub const AED_VERSION: u32 = 1;
27
28impl AedWeightBundle {
29 pub fn from_seed(seed: u64) -> Self {
31 let mut weight = [0.0f32; 16];
32 let mut h = seed ^ 0xAED0_AED0_AED0_AED0;
33 for w in weight.iter_mut() {
34 h = h.wrapping_mul(0x9e37_79b9_7f4a_7c15).rotate_left(11);
35 *w = ((h % 1000) as f32 / 500.0) - 1.0;
36 }
37 let bias = [-0.2, 0.1, 0.05, 0.15];
39 Self {
40 model_hash: q_hash(&format!("{MODEL}:{seed:016x}")),
41 weight,
42 bias,
43 class_hashes: [
44 q_hash(CLASS_SILENCE),
45 q_hash(CLASS_SPEECH_LIKE),
46 q_hash(CLASS_NOISE),
47 q_hash(CLASS_TONAL),
48 ],
49 }
50 }
51
52 pub fn to_bytes(&self) -> Vec<u8> {
54 let mut v = Vec::with_capacity(8 + 8 + 16 * 4 + 4 * 4 + 4 * 8);
55 v.extend_from_slice(&AED_MAGIC.to_le_bytes());
56 v.extend_from_slice(&AED_VERSION.to_le_bytes());
57 v.extend_from_slice(&self.model_hash.to_le_bytes());
58 for f in &self.weight {
59 v.extend_from_slice(&f.to_le_bytes());
60 }
61 for f in &self.bias {
62 v.extend_from_slice(&f.to_le_bytes());
63 }
64 for h in &self.class_hashes {
65 v.extend_from_slice(&h.to_le_bytes());
66 }
67 v
68 }
69
70 pub fn from_bytes(bytes: &[u8]) -> Result<Self, crate::types::AudioError> {
72 let need = 4 + 4 + 8 + 16 * 4 + 4 * 4 + 4 * 8;
73 if bytes.len() < need {
74 return Err(crate::types::AudioError::MalformedAudio);
75 }
76 let magic = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
77 if magic != AED_MAGIC {
78 return Err(crate::types::AudioError::BackendUnavailable);
79 }
80 let ver = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
81 if ver != AED_VERSION {
82 return Err(crate::types::AudioError::BackendUnavailable);
83 }
84 let model_hash = u64::from_le_bytes(
85 bytes[8..16]
86 .try_into()
87 .map_err(|_| crate::types::AudioError::MalformedAudio)?,
88 );
89 let mut off = 16usize;
90 let mut weight = [0.0f32; 16];
91 for w in weight.iter_mut() {
92 *w = f32::from_le_bytes(
93 bytes[off..off + 4]
94 .try_into()
95 .map_err(|_| crate::types::AudioError::MalformedAudio)?,
96 );
97 off += 4;
98 }
99 let mut bias = [0.0f32; 4];
100 for b in bias.iter_mut() {
101 *b = f32::from_le_bytes(
102 bytes[off..off + 4]
103 .try_into()
104 .map_err(|_| crate::types::AudioError::MalformedAudio)?,
105 );
106 off += 4;
107 }
108 let mut class_hashes = [0u64; 4];
109 for c in class_hashes.iter_mut() {
110 *c = u64::from_le_bytes(
111 bytes[off..off + 8]
112 .try_into()
113 .map_err(|_| crate::types::AudioError::MalformedAudio)?,
114 );
115 off += 8;
116 }
117 Ok(Self {
118 model_hash,
119 weight,
120 bias,
121 class_hashes,
122 })
123 }
124
125 pub fn save_path(&self, path: &std::path::Path) -> Result<(), String> {
127 if let Some(p) = path.parent() {
128 std::fs::create_dir_all(p).map_err(|e| e.to_string())?;
129 }
130 std::fs::write(path, self.to_bytes()).map_err(|e| e.to_string())
131 }
132
133 pub fn load_path(path: &std::path::Path) -> Result<Self, String> {
135 let b = std::fs::read(path).map_err(|e| e.to_string())?;
136 Self::from_bytes(&b).map_err(|e| format!("{e:?}"))
137 }
138}
139
140pub struct WeightedAedModel {
141 pub bundle: AedWeightBundle,
142 pub frame_len: usize,
143 pub hop: usize,
144 pub vad_threshold: f32,
145}
146
147impl WeightedAedModel {
148 pub fn from_seed(seed: u64) -> Self {
150 Self {
151 bundle: AedWeightBundle::from_seed(seed),
152 frame_len: 512,
153 hop: 256,
154 vad_threshold: 0.015,
155 }
156 }
157
158 pub fn from_bundle(bundle: AedWeightBundle) -> Self {
160 Self {
161 bundle,
162 frame_len: 512,
163 hop: 256,
164 vad_threshold: 0.015,
165 }
166 }
167
168 pub fn model_hash(&self) -> u64 {
170 self.bundle.model_hash
171 }
172
173 fn classify_feat(&self, energy: f32, zcr: f32, flux: f32) -> (usize, f32) {
174 let feat = [energy, zcr, flux, energy * zcr];
175 let mut best_i = 0usize;
176 let mut best = f32::NEG_INFINITY;
177 for c in 0..4 {
178 let mut logit = self.bundle.bias[c];
179 for i in 0..4 {
180 logit += self.bundle.weight[c * 4 + i] * feat[i];
181 }
182 if logit > best {
183 best = logit;
184 best_i = c;
185 }
186 }
187 let conf = ((best.tanh() + 1.0) * 0.5).clamp(0.0, 1.0);
188 (best_i, conf)
189 }
190}
191
192impl AuditoryModel for WeightedAedModel {
193 fn capabilities(&self) -> AuditoryCapabilities {
194 AuditoryCapabilities {
195 max_events: MAX_EVENTS as u16,
196 embed_dim: 4,
197 supports_vad: true,
198 supports_transcript: false,
199 is_reference_backend: false,
201 }
202 }
203
204 fn infer_chunk(
205 &mut self,
206 audio: AudioView<'_>,
207 events_out: &mut [AuditoryEvent],
208 tokens_out: &mut [TranscriptToken],
209 embedding_out: &mut [f32],
210 workspace: &mut [u8],
211 ) -> Result<AuditoryOutputCounts, AudioError> {
212 if !audio.is_well_formed() {
213 return Err(AudioError::MalformedAudio);
214 }
215 let _ = workspace;
216 for t in tokens_out.iter_mut() {
217 *t = TranscriptToken::empty();
218 }
219 let mut mono = vec![0.0f32; audio.frames as usize];
220 let n = to_mono_f32(audio, &mut mono)?;
221 let mut event_n = 0usize;
222 let mut prev_e = 0.0f32;
223 let mut i = 0usize;
224 let mut active: Option<usize> = None;
225 let mut sum_e = 0.0f32;
226 let mut sum_z = 0.0f32;
227 let mut sum_f = 0.0f32;
228 let mut cnt = 0u32;
229
230 while i + self.frame_len <= n {
231 let frame = &mono[i..i + self.frame_len];
232 let e = frame_energy(frame);
233 let z = frame_zcr(frame);
234 let flux = (e - prev_e).max(0.0);
235 prev_e = e;
236 sum_e += e;
237 sum_z += z;
238 sum_f += flux;
239 cnt += 1;
240 if e >= self.vad_threshold {
241 if active.is_none() {
242 active = Some(i);
243 }
244 } else if let Some(st) = active.take() {
245 if event_n < events_out.len() {
246 let (ci, conf) = self.classify_feat(e, z, flux);
247 events_out[event_n] = make_ev(
248 self.bundle.class_hashes[ci],
249 conf,
250 st as u64,
251 i as u64,
252 self.bundle.model_hash,
253 );
254 event_n += 1;
255 }
256 }
257 i += self.hop;
258 }
259 if let Some(st) = active {
260 if event_n < events_out.len() {
261 let e = frame_energy(&mono[st.min(n)..n]);
262 let z = frame_zcr(&mono[st.min(n)..n]);
263 let (ci, conf) = self.classify_feat(e, z, 0.0);
264 events_out[event_n] = make_ev(
265 self.bundle.class_hashes[ci],
266 conf,
267 st as u64,
268 n as u64,
269 self.bundle.model_hash,
270 );
271 event_n += 1;
272 }
273 }
274 if event_n == 0 && n > 0 {
275 events_out[0] = make_ev(
276 self.bundle.class_hashes[0],
277 0.35,
278 0,
279 n as u64,
280 self.bundle.model_hash,
281 );
282 event_n = 1;
283 }
284 for e in events_out.iter_mut().skip(event_n) {
285 *e = AuditoryEvent::empty();
286 }
287 let emb_n = embedding_out.len().min(4);
288 if cnt > 0 && emb_n > 0 {
289 embedding_out[0] = sum_e / cnt as f32;
290 if emb_n > 1 {
291 embedding_out[1] = sum_z / cnt as f32;
292 }
293 if emb_n > 2 {
294 embedding_out[2] = sum_f / cnt as f32;
295 }
296 if emb_n > 3 {
297 embedding_out[3] = audio.sample_rate as f32 / 48000.0;
298 }
299 }
300 Ok(AuditoryOutputCounts {
301 events: event_n,
302 tokens: 0,
303 embedding_written: emb_n,
304 })
305 }
306}
307
308fn make_ev(class: u64, conf: f32, start: u64, end: u64, model: u64) -> AuditoryEvent {
309 let mut e = AuditoryEvent::empty();
310 e.class_hash = class;
311 e.source_hash = model ^ start;
312 e.confidence_u16 = (conf * 65535.0) as u16;
313 e.start_frame = start;
314 e.end_frame = end.max(start + 1);
315 e.flags = AuditoryEvent::FLAG_LOW_ASSURANCE | AuditoryEvent::FLAG_VAD;
316 e
317}
318
319#[cfg(test)]
320mod tests {
321 use super::*;
322 use crate::types::SampleFormat;
323
324 #[test]
325 fn aed_weight_roundtrip() {
326 let b = AedWeightBundle::from_seed(42);
327 let bytes = b.to_bytes();
328 let b2 = AedWeightBundle::from_bytes(&bytes).unwrap();
329 assert_eq!(b.model_hash, b2.model_hash);
330 assert_eq!(b.weight, b2.weight);
331 }
332
333 #[test]
334 fn weighted_aed_runs() {
335 let n = 3000usize;
336 let mut pcm = vec![0i16; n];
337 for i in 0..n {
338 pcm[i] = (0.3
339 * (2.0 * core::f32::consts::PI * 300.0 * i as f32 / 16000.0).sin()
340 * 32767.0) as i16;
341 }
342 let mut bytes = vec![0u8; n * 2];
343 for (i, s) in pcm.iter().enumerate() {
344 let b = s.to_le_bytes();
345 bytes[i * 2] = b[0];
346 bytes[i * 2 + 1] = b[1];
347 }
348 let view = AudioView {
349 bytes: &bytes,
350 frames: n as u32,
351 channels: 1,
352 sample_rate: 16000,
353 frame_stride_bytes: 2,
354 format: SampleFormat::I16,
355 };
356 let mut m = WeightedAedModel::from_seed(1);
357 assert!(!m.capabilities().is_reference_backend);
358 let mut ev = [AuditoryEvent::empty(); 16];
359 let mut tok = [TranscriptToken::empty(); 4];
360 let mut emb = [0.0f32; 4];
361 let mut ws = [0u8; 8];
362 let c = m
363 .infer_chunk(view, &mut ev, &mut tok, &mut emb, &mut ws)
364 .unwrap();
365 assert!(c.events >= 1);
366 }
367}