Skip to main content

qualia_audio/
events.rs

1//! A5 — reference VAD + energy event classifier (honest reference backend).
2
3use crate::convert::to_mono_f32;
4use crate::features::{frame_energy, frame_zcr};
5use crate::hash::q_hash;
6use crate::types::{
7    AudioError, AudioView, AuditoryCapabilities, AuditoryEvent, AuditoryModel,
8    AuditoryOutputCounts, TranscriptToken, MAX_EVENTS,
9};
10
11pub const CLASS_SILENCE: &str = "https://ns.webizen.org/q42/audio/class/silence";
12pub const CLASS_SPEECH_LIKE: &str = "https://ns.webizen.org/q42/audio/class/speech-like";
13pub const CLASS_NOISE: &str = "https://ns.webizen.org/q42/audio/class/noise";
14pub const CLASS_TONAL: &str = "https://ns.webizen.org/q42/audio/class/tonal";
15
16const MODEL_ID: &str = "qualia-audio-cpu-reference-v1";
17
18/// Energy-based VAD + coarse sound class from ZCR/energy (not a neural AED).
19pub struct ReferenceEventModel {
20    model_hash: u64,
21    pub frame_len: usize,
22    pub hop: usize,
23    pub vad_threshold: f32,
24}
25
26impl Default for ReferenceEventModel {
27    fn default() -> Self {
28        Self::new()
29    }
30}
31
32impl ReferenceEventModel {
33    /// Creates a new instance.
34    pub fn new() -> Self {
35        Self {
36            model_hash: q_hash(MODEL_ID),
37            frame_len: 512,
38            hop: 256,
39            vad_threshold: 0.02,
40        }
41    }
42
43    /// `model` hash.
44    pub fn model_hash(&self) -> u64 {
45        self.model_hash
46    }
47}
48
49impl AuditoryModel for ReferenceEventModel {
50    fn capabilities(&self) -> AuditoryCapabilities {
51        AuditoryCapabilities {
52            max_events: MAX_EVENTS as u16,
53            embed_dim: 8,
54            supports_vad: true,
55            supports_transcript: false,
56            is_reference_backend: true,
57        }
58    }
59
60    fn infer_chunk(
61        &mut self,
62        audio: AudioView<'_>,
63        events_out: &mut [AuditoryEvent],
64        tokens_out: &mut [TranscriptToken],
65        embedding_out: &mut [f32],
66        workspace: &mut [u8],
67    ) -> Result<AuditoryOutputCounts, AudioError> {
68        if !audio.is_well_formed() {
69            return Err(AudioError::MalformedAudio);
70        }
71        if events_out.is_empty() {
72            return Err(AudioError::OutputBufferTooSmall);
73        }
74        let need = audio.frames as usize * 4;
75        if workspace.len() < need {
76            return Err(AudioError::WorkspaceTooSmall);
77        }
78        // Reinterpret workspace as f32 mono buffer (alignment: use local vec for cold path safety)
79        let mut mono = vec![0.0f32; audio.frames as usize];
80        let n = to_mono_f32(audio, &mut mono)?;
81        for t in tokens_out.iter_mut() {
82            *t = TranscriptToken::empty();
83        }
84
85        let mut event_n = 0usize;
86        let mut emb = [0.0f32; 8];
87        let mut i = 0usize;
88        let mut active_start: Option<usize> = None;
89        let mut sum_e = 0.0f32;
90        let mut sum_z = 0.0f32;
91        let mut cnt = 0u32;
92
93        while i + self.frame_len <= n {
94            let frame = &mono[i..i + self.frame_len];
95            let e = frame_energy(frame);
96            let z = frame_zcr(frame);
97            sum_e += e;
98            sum_z += z;
99            cnt += 1;
100            if e >= self.vad_threshold {
101                if active_start.is_none() {
102                    active_start = Some(i);
103                }
104            } else if let Some(st) = active_start.take() {
105                if event_n < events_out.len() && event_n < MAX_EVENTS {
106                    events_out[event_n] =
107                        classify_segment(st as u64, i as u64, e, z, self.model_hash, audio);
108                    event_n += 1;
109                }
110            }
111            i += self.hop;
112        }
113        if let Some(st) = active_start {
114            if event_n < events_out.len() {
115                let e = frame_energy(&mono[st.min(n)..n]);
116                let z = frame_zcr(&mono[st.min(n)..n]);
117                events_out[event_n] =
118                    classify_segment(st as u64, n as u64, e, z, self.model_hash, audio);
119                event_n += 1;
120            }
121        }
122        // If never active, optional silence event for whole clip (low conf)
123        if event_n == 0 && n > 0 {
124            let mut ev = AuditoryEvent::empty();
125            ev.class_hash = q_hash(CLASS_SILENCE);
126            ev.source_hash = self.model_hash ^ media_src(audio);
127            ev.confidence_u16 = 20_000;
128            ev.start_frame = 0;
129            ev.end_frame = n as u64;
130            ev.flags = AuditoryEvent::FLAG_REFERENCE_BACKEND | AuditoryEvent::FLAG_VAD;
131            events_out[0] = ev;
132            event_n = 1;
133        }
134        for e in events_out.iter_mut().skip(event_n) {
135            *e = AuditoryEvent::empty();
136        }
137        if cnt > 0 {
138            emb[0] = sum_e / cnt as f32;
139            emb[1] = sum_z / cnt as f32;
140            emb[2] = audio.sample_rate as f32 / 48000.0;
141        }
142        let emb_n = embedding_out.len().min(8);
143        embedding_out[..emb_n].copy_from_slice(&emb[..emb_n]);
144        Ok(AuditoryOutputCounts {
145            events: event_n,
146            tokens: 0,
147            embedding_written: emb_n,
148        })
149    }
150}
151
152fn media_src(audio: AudioView<'_>) -> u64 {
153    crate::hash::q_hash_bytes(&audio.bytes[..audio.bytes.len().min(1024)]) ^ (audio.frames as u64)
154}
155
156fn classify_segment(
157    start: u64,
158    end: u64,
159    energy: f32,
160    zcr: f32,
161    model_hash: u64,
162    audio: AudioView<'_>,
163) -> AuditoryEvent {
164    let (iri, conf) = if zcr < 0.1 && energy > 0.05 {
165        (CLASS_TONAL, 0.7)
166    } else if zcr > 0.15 && energy > 0.03 {
167        (CLASS_SPEECH_LIKE, 0.55)
168    } else if energy > 0.02 {
169        (CLASS_NOISE, 0.45)
170    } else {
171        (CLASS_SILENCE, 0.4)
172    };
173    let mut e = AuditoryEvent::empty();
174    e.class_hash = q_hash(iri);
175    e.source_hash = model_hash ^ start ^ media_src(audio);
176    e.confidence_u16 = (conf * 65535.0) as u16;
177    e.start_frame = start;
178    e.end_frame = end.max(start + 1);
179    e.flags = AuditoryEvent::FLAG_REFERENCE_BACKEND
180        | AuditoryEvent::FLAG_LOW_ASSURANCE
181        | AuditoryEvent::FLAG_VAD;
182    e
183}
184
185#[cfg(test)]
186mod tests {
187    use super::*;
188    use crate::types::SampleFormat;
189
190    #[test]
191    fn tone_is_tonal_or_active() {
192        let sr = 16000u32;
193        let n = 4000usize;
194        let mut pcm = vec![0i16; n];
195        for i in 0..n {
196            let t = i as f32 / sr as f32;
197            pcm[i] = (0.4 * (2.0 * core::f32::consts::PI * 440.0 * t).sin() * 32767.0) as i16;
198        }
199        let mut bytes = vec![0u8; n * 2];
200        for (i, s) in pcm.iter().enumerate() {
201            let b = s.to_le_bytes();
202            bytes[i * 2] = b[0];
203            bytes[i * 2 + 1] = b[1];
204        }
205        let view = AudioView {
206            bytes: &bytes,
207            frames: n as u32,
208            channels: 1,
209            sample_rate: sr,
210            frame_stride_bytes: 2,
211            format: SampleFormat::I16,
212        };
213        let mut m = ReferenceEventModel::new();
214        let mut ev = [AuditoryEvent::empty(); 16];
215        let mut tok = [TranscriptToken::empty(); 4];
216        let mut emb = [0.0f32; 8];
217        let mut ws = vec![0u8; n * 4];
218        let c = m
219            .infer_chunk(view, &mut ev, &mut tok, &mut emb, &mut ws)
220            .unwrap();
221        assert!(c.events >= 1);
222        assert!(m.capabilities().is_reference_backend);
223    }
224}