1use crate::convert::to_mono_f32;
4use crate::features::{frame_energy, frame_zcr};
5use crate::hash::q_hash;
6use crate::types::{
7 AudioError, AudioView, AuditoryCapabilities, AuditoryEvent, AuditoryModel,
8 AuditoryOutputCounts, TranscriptToken, MAX_EVENTS,
9};
10
11pub const CLASS_SILENCE: &str = "https://ns.webizen.org/q42/audio/class/silence";
12pub const CLASS_SPEECH_LIKE: &str = "https://ns.webizen.org/q42/audio/class/speech-like";
13pub const CLASS_NOISE: &str = "https://ns.webizen.org/q42/audio/class/noise";
14pub const CLASS_TONAL: &str = "https://ns.webizen.org/q42/audio/class/tonal";
15
16const MODEL_ID: &str = "qualia-audio-cpu-reference-v1";
17
18pub struct ReferenceEventModel {
20 model_hash: u64,
21 pub frame_len: usize,
22 pub hop: usize,
23 pub vad_threshold: f32,
24}
25
26impl Default for ReferenceEventModel {
27 fn default() -> Self {
28 Self::new()
29 }
30}
31
32impl ReferenceEventModel {
33 pub fn new() -> Self {
35 Self {
36 model_hash: q_hash(MODEL_ID),
37 frame_len: 512,
38 hop: 256,
39 vad_threshold: 0.02,
40 }
41 }
42
43 pub fn model_hash(&self) -> u64 {
45 self.model_hash
46 }
47}
48
49impl AuditoryModel for ReferenceEventModel {
50 fn capabilities(&self) -> AuditoryCapabilities {
51 AuditoryCapabilities {
52 max_events: MAX_EVENTS as u16,
53 embed_dim: 8,
54 supports_vad: true,
55 supports_transcript: false,
56 is_reference_backend: true,
57 }
58 }
59
60 fn infer_chunk(
61 &mut self,
62 audio: AudioView<'_>,
63 events_out: &mut [AuditoryEvent],
64 tokens_out: &mut [TranscriptToken],
65 embedding_out: &mut [f32],
66 workspace: &mut [u8],
67 ) -> Result<AuditoryOutputCounts, AudioError> {
68 if !audio.is_well_formed() {
69 return Err(AudioError::MalformedAudio);
70 }
71 if events_out.is_empty() {
72 return Err(AudioError::OutputBufferTooSmall);
73 }
74 let need = audio.frames as usize * 4;
75 if workspace.len() < need {
76 return Err(AudioError::WorkspaceTooSmall);
77 }
78 let mut mono = vec![0.0f32; audio.frames as usize];
80 let n = to_mono_f32(audio, &mut mono)?;
81 for t in tokens_out.iter_mut() {
82 *t = TranscriptToken::empty();
83 }
84
85 let mut event_n = 0usize;
86 let mut emb = [0.0f32; 8];
87 let mut i = 0usize;
88 let mut active_start: Option<usize> = None;
89 let mut sum_e = 0.0f32;
90 let mut sum_z = 0.0f32;
91 let mut cnt = 0u32;
92
93 while i + self.frame_len <= n {
94 let frame = &mono[i..i + self.frame_len];
95 let e = frame_energy(frame);
96 let z = frame_zcr(frame);
97 sum_e += e;
98 sum_z += z;
99 cnt += 1;
100 if e >= self.vad_threshold {
101 if active_start.is_none() {
102 active_start = Some(i);
103 }
104 } else if let Some(st) = active_start.take() {
105 if event_n < events_out.len() && event_n < MAX_EVENTS {
106 events_out[event_n] =
107 classify_segment(st as u64, i as u64, e, z, self.model_hash, audio);
108 event_n += 1;
109 }
110 }
111 i += self.hop;
112 }
113 if let Some(st) = active_start {
114 if event_n < events_out.len() {
115 let e = frame_energy(&mono[st.min(n)..n]);
116 let z = frame_zcr(&mono[st.min(n)..n]);
117 events_out[event_n] =
118 classify_segment(st as u64, n as u64, e, z, self.model_hash, audio);
119 event_n += 1;
120 }
121 }
122 if event_n == 0 && n > 0 {
124 let mut ev = AuditoryEvent::empty();
125 ev.class_hash = q_hash(CLASS_SILENCE);
126 ev.source_hash = self.model_hash ^ media_src(audio);
127 ev.confidence_u16 = 20_000;
128 ev.start_frame = 0;
129 ev.end_frame = n as u64;
130 ev.flags = AuditoryEvent::FLAG_REFERENCE_BACKEND | AuditoryEvent::FLAG_VAD;
131 events_out[0] = ev;
132 event_n = 1;
133 }
134 for e in events_out.iter_mut().skip(event_n) {
135 *e = AuditoryEvent::empty();
136 }
137 if cnt > 0 {
138 emb[0] = sum_e / cnt as f32;
139 emb[1] = sum_z / cnt as f32;
140 emb[2] = audio.sample_rate as f32 / 48000.0;
141 }
142 let emb_n = embedding_out.len().min(8);
143 embedding_out[..emb_n].copy_from_slice(&emb[..emb_n]);
144 Ok(AuditoryOutputCounts {
145 events: event_n,
146 tokens: 0,
147 embedding_written: emb_n,
148 })
149 }
150}
151
152fn media_src(audio: AudioView<'_>) -> u64 {
153 crate::hash::q_hash_bytes(&audio.bytes[..audio.bytes.len().min(1024)]) ^ (audio.frames as u64)
154}
155
156fn classify_segment(
157 start: u64,
158 end: u64,
159 energy: f32,
160 zcr: f32,
161 model_hash: u64,
162 audio: AudioView<'_>,
163) -> AuditoryEvent {
164 let (iri, conf) = if zcr < 0.1 && energy > 0.05 {
165 (CLASS_TONAL, 0.7)
166 } else if zcr > 0.15 && energy > 0.03 {
167 (CLASS_SPEECH_LIKE, 0.55)
168 } else if energy > 0.02 {
169 (CLASS_NOISE, 0.45)
170 } else {
171 (CLASS_SILENCE, 0.4)
172 };
173 let mut e = AuditoryEvent::empty();
174 e.class_hash = q_hash(iri);
175 e.source_hash = model_hash ^ start ^ media_src(audio);
176 e.confidence_u16 = (conf * 65535.0) as u16;
177 e.start_frame = start;
178 e.end_frame = end.max(start + 1);
179 e.flags = AuditoryEvent::FLAG_REFERENCE_BACKEND
180 | AuditoryEvent::FLAG_LOW_ASSURANCE
181 | AuditoryEvent::FLAG_VAD;
182 e
183}
184
185#[cfg(test)]
186mod tests {
187 use super::*;
188 use crate::types::SampleFormat;
189
190 #[test]
191 fn tone_is_tonal_or_active() {
192 let sr = 16000u32;
193 let n = 4000usize;
194 let mut pcm = vec![0i16; n];
195 for i in 0..n {
196 let t = i as f32 / sr as f32;
197 pcm[i] = (0.4 * (2.0 * core::f32::consts::PI * 440.0 * t).sin() * 32767.0) as i16;
198 }
199 let mut bytes = vec![0u8; n * 2];
200 for (i, s) in pcm.iter().enumerate() {
201 let b = s.to_le_bytes();
202 bytes[i * 2] = b[0];
203 bytes[i * 2 + 1] = b[1];
204 }
205 let view = AudioView {
206 bytes: &bytes,
207 frames: n as u32,
208 channels: 1,
209 sample_rate: sr,
210 frame_stride_bytes: 2,
211 format: SampleFormat::I16,
212 };
213 let mut m = ReferenceEventModel::new();
214 let mut ev = [AuditoryEvent::empty(); 16];
215 let mut tok = [TranscriptToken::empty(); 4];
216 let mut emb = [0.0f32; 8];
217 let mut ws = vec![0u8; n * 4];
218 let c = m
219 .infer_chunk(view, &mut ev, &mut tok, &mut emb, &mut ws)
220 .unwrap();
221 assert!(c.events >= 1);
222 assert!(m.capabilities().is_reference_backend);
223 }
224}