qualia_core_db/hypermedia/processors/
audio.rs

1//! **WavProcessor** — derive searchability from an audio file by composing the
2//! project's *own* DSP (`crate::audio`), not a bolt-on library.
3//!
4//! It decodes a PCM/float WAV header, then runs the real short-time Fourier
5//! transform ([`crate::audio::forward_stft`] → [`crate::audio::stft_magnitudes`],
6//! GPU best-path with a CPU DFT floor) to summarise the recording: its duration
7//! and its dominant frequency. Those become descriptor facets so an otherwise
8//! opaque `.wav` is findable ("audio", "~440 Hz", by duration).
9//!
10//! **Honest boundary.** A *transcript* (speech-to-text) is an **ASR-model**
11//! concern — the [`Processor`] plug-in point for a `qualia-audio` speech engine,
12//! not something a spectral summary can fabricate. This derives the acoustic
13//! descriptors that are genuinely computable from the signal, and no words.
14
15use std::collections::HashMap;
16
17use super::super::{fnv60, AssetRef, AssetRole, Descriptors, Processor, ProcessorOutput};
18
19/// A model-free acoustic summary of a recording — what the DSP can honestly say.
20#[derive(Debug, Clone, Default, PartialEq)]
21pub struct AudioSpectralSummary {
22    pub sample_rate: u32,
23    pub channels: u16,
24    pub duration_secs: f32,
25    /// The dominant (peak-energy) frequency in Hz, averaged over the STFT frames.
26    pub dominant_hz: Option<f32>,
27}
28
29/// A real audio processor: WAV → duration + dominant-frequency descriptors via
30/// the project's STFT. Transcript is an ASR plug-in point, not faked here.
31#[derive(Debug, Clone, Default)]
32pub struct WavProcessor;
33
34const FRAME_SIZE: usize = 256;
35const HOP: usize = 128;
36
37impl WavProcessor {
38    /// Decode + analyse a WAV byte stream. Returns `None` if it is not a WAV.
39    pub fn analyse(bytes: &[u8]) -> Option<AudioSpectralSummary> {
40        let wav = parse_wav(bytes)?;
41        let duration_secs = if wav.sample_rate > 0 {
42            wav.samples.len() as f32 / wav.sample_rate as f32
43        } else {
44            0.0
45        };
46        let dominant_hz = dominant_frequency(&wav.samples, wav.sample_rate);
47        Some(AudioSpectralSummary {
48            sample_rate: wav.sample_rate,
49            channels: wav.channels,
50            duration_secs,
51            dominant_hz,
52        })
53    }
54}
55
56impl Processor for WavProcessor {
57    fn handles(&self, media_type: &str) -> bool {
58        matches!(media_type, "audio/wav" | "audio/x-wav" | "audio/wave")
59    }
60
61    fn process(&self, asset_uri: &str, bytes: &[u8], _media_type: &str) -> ProcessorOutput {
62        let summary = WavProcessor::analyse(bytes).unwrap_or_default();
63
64        let mut topics = vec!["audio".to_string()];
65        if let Some(hz) = summary.dominant_hz {
66            // A coarse pitch-band facet so recordings cluster by register.
67            topics.push(pitch_band(hz).to_string());
68        }
69
70        let mut text = format!(
71            "audio recording; {:.2}s; {} Hz; {} channel(s)",
72            summary.duration_secs, summary.sample_rate, summary.channels
73        );
74        if let Some(hz) = summary.dominant_hz {
75            text.push_str(&format!("; dominant {hz:.0} Hz"));
76        }
77
78        let descriptors = Descriptors {
79            topics,
80            document_type: Some("audio".to_string()),
81            ..Default::default()
82        };
83
84        let meta_uri = format!("{asset_uri}#acoustic");
85        let derived = vec![AssetRef::new(
86            &meta_uri,
87            fnv60(text.as_bytes()),
88            "text/plain",
89            AssetRole::Analysis,
90        )
91        .derived_from(asset_uri)];
92        let mut derived_bytes = HashMap::new();
93        derived_bytes.insert(meta_uri, text.into_bytes());
94
95        ProcessorOutput {
96            derived,
97            derived_bytes,
98            descriptors,
99            flags: Vec::new(),
100        }
101    }
102}
103
104fn pitch_band(hz: f32) -> &'static str {
105    match hz {
106        h if h < 250.0 => "low-frequency",
107        h if h < 2000.0 => "mid-frequency",
108        _ => "high-frequency",
109    }
110}
111
112struct WavData {
113    sample_rate: u32,
114    channels: u16,
115    /// Mono f32 samples (channel 0), normalised to roughly [-1, 1].
116    samples: Vec<f32>,
117}
118
119/// Parse a RIFF/WAVE header and decode channel 0 to f32. Supports PCM 8/16/32-bit
120/// and IEEE-float 32-bit — the common uncompressed encodings.
121fn parse_wav(b: &[u8]) -> Option<WavData> {
122    if b.len() < 12 || &b[0..4] != b"RIFF" || &b[8..12] != b"WAVE" {
123        return None;
124    }
125    let mut fmt: Option<(u16, u16, u32, u16)> = None; // (audio_format, channels, sample_rate, bits)
126    let mut data: Option<&[u8]> = None;
127    let mut i = 12;
128    while i + 8 <= b.len() {
129        let id = &b[i..i + 4];
130        let size = u32::from_le_bytes([b[i + 4], b[i + 5], b[i + 6], b[i + 7]]) as usize;
131        let start = i + 8;
132        let end = start.checked_add(size)?.min(b.len());
133        match id {
134            b"fmt " if end - start >= 16 => {
135                let c = &b[start..end];
136                let audio_format = u16::from_le_bytes([c[0], c[1]]);
137                let channels = u16::from_le_bytes([c[2], c[3]]);
138                let sample_rate = u32::from_le_bytes([c[4], c[5], c[6], c[7]]);
139                let bits = u16::from_le_bytes([c[14], c[15]]);
140                fmt = Some((audio_format, channels, sample_rate, bits));
141            }
142            b"data" => data = Some(&b[start..end]),
143            _ => {}
144        }
145        // Chunks are word-aligned (pad byte if odd size).
146        i = start + size + (size & 1);
147    }
148
149    let (audio_format, channels, sample_rate, bits) = fmt?;
150    let data = data?;
151    let channels = channels.max(1);
152    let ch = channels as usize;
153
154    let mut samples = Vec::new();
155    match (audio_format, bits) {
156        (1, 16) => {
157            // interleaved i16
158            let frame = 2 * ch;
159            let mut o = 0;
160            while o + 2 <= data.len() {
161                let v = i16::from_le_bytes([data[o], data[o + 1]]) as f32 / 32768.0;
162                samples.push(v);
163                o += frame; // channel 0 only
164            }
165        }
166        (1, 8) => {
167            let frame = ch;
168            let mut o = 0;
169            while o < data.len() {
170                let v = (data[o] as f32 - 128.0) / 128.0;
171                samples.push(v);
172                o += frame;
173            }
174        }
175        (1, 32) => {
176            let frame = 4 * ch;
177            let mut o = 0;
178            while o + 4 <= data.len() {
179                let raw = i32::from_le_bytes([data[o], data[o + 1], data[o + 2], data[o + 3]]);
180                samples.push(raw as f32 / 2_147_483_648.0);
181                o += frame;
182            }
183        }
184        (3, 32) => {
185            let frame = 4 * ch;
186            let mut o = 0;
187            while o + 4 <= data.len() {
188                let v = f32::from_le_bytes([data[o], data[o + 1], data[o + 2], data[o + 3]]);
189                samples.push(v);
190                o += frame;
191            }
192        }
193        _ => return None, // unsupported encoding — honestly decline rather than guess
194    }
195
196    Some(WavData {
197        sample_rate,
198        channels,
199        samples,
200    })
201}
202
203/// Dominant frequency (Hz) via the project's STFT: average the magnitude
204/// spectrum across frames and take the peak non-DC bin. `None` if the clip is
205/// shorter than one frame or the transform yields nothing.
206fn dominant_frequency(samples: &[f32], sample_rate: u32) -> Option<f32> {
207    if samples.len() < FRAME_SIZE || sample_rate == 0 {
208        return None;
209    }
210    let spec = crate::audio::forward_stft(samples, FRAME_SIZE, HOP).ok()?;
211    if spec.is_empty() {
212        return None;
213    }
214    let mags = crate::audio::stft_magnitudes(&spec);
215    // Average magnitude per bin over frames; only the first half is meaningful
216    // for a real signal (the spectrum is conjugate-symmetric).
217    let half = FRAME_SIZE / 2;
218    let mut acc = vec![0.0f32; half];
219    for frame in &mags {
220        for (k, a) in acc.iter_mut().enumerate() {
221            if let Some(m) = frame.get(k) {
222                *a += *m;
223            }
224        }
225    }
226    // Peak, skipping the DC bin (0).
227    let (mut best_k, mut best_v) = (0usize, -1.0f32);
228    for (k, &v) in acc.iter().enumerate().skip(1) {
229        if v > best_v {
230            best_v = v;
231            best_k = k;
232        }
233    }
234    if best_v <= 0.0 {
235        return None;
236    }
237    Some(best_k as f32 * sample_rate as f32 / FRAME_SIZE as f32)
238}
239
240#[cfg(test)]
241mod tests {
242    use super::super::super::{by_topic, ingest_with};
243    use super::*;
244    use std::f32::consts::TAU;
245
246    /// Build a mono 16-bit PCM WAV of a pure sine at `freq` Hz.
247    fn sine_wav(freq: f32, sample_rate: u32, secs: f32) -> Vec<u8> {
248        let n = (sample_rate as f32 * secs) as usize;
249        let mut data = Vec::with_capacity(n * 2);
250        for i in 0..n {
251            let t = i as f32 / sample_rate as f32;
252            let s = (TAU * freq * t).sin();
253            let q = (s * 32767.0) as i16;
254            data.extend_from_slice(&q.to_le_bytes());
255        }
256        let byte_rate = sample_rate * 2;
257        let mut b = Vec::new();
258        b.extend_from_slice(b"RIFF");
259        b.extend_from_slice(&(36 + data.len() as u32).to_le_bytes());
260        b.extend_from_slice(b"WAVE");
261        // fmt chunk
262        b.extend_from_slice(b"fmt ");
263        b.extend_from_slice(&16u32.to_le_bytes());
264        b.extend_from_slice(&1u16.to_le_bytes()); // PCM
265        b.extend_from_slice(&1u16.to_le_bytes()); // mono
266        b.extend_from_slice(&sample_rate.to_le_bytes());
267        b.extend_from_slice(&byte_rate.to_le_bytes());
268        b.extend_from_slice(&2u16.to_le_bytes()); // block align
269        b.extend_from_slice(&16u16.to_le_bytes()); // bits
270                                                   // data chunk
271        b.extend_from_slice(b"data");
272        b.extend_from_slice(&(data.len() as u32).to_le_bytes());
273        b.extend_from_slice(&data);
274        b
275    }
276
277    #[test]
278    fn wav_header_decodes_duration_and_channels() {
279        let wav = sine_wav(440.0, 8000, 0.5);
280        let s = WavProcessor::analyse(&wav).expect("wav parsed");
281        assert_eq!(s.sample_rate, 8000);
282        assert_eq!(s.channels, 1);
283        assert!(
284            (s.duration_secs - 0.5).abs() < 0.02,
285            "≈0.5s, got {}",
286            s.duration_secs
287        );
288    }
289
290    #[test]
291    fn stft_finds_the_dominant_tone() {
292        // A 1000 Hz sine at 8 kHz — the STFT peak bin should land near 1000 Hz.
293        let wav = sine_wav(1000.0, 8000, 0.5);
294        let s = WavProcessor::analyse(&wav).expect("wav parsed");
295        let hz = s.dominant_hz.expect("dominant frequency");
296        // Bin resolution = 8000/256 ≈ 31.25 Hz; allow a couple of bins.
297        assert!((hz - 1000.0).abs() < 80.0, "dominant ≈1000 Hz, got {hz}");
298    }
299
300    #[test]
301    fn audio_ingest_is_findable_by_topic() {
302        let wav = sine_wav(440.0, 8000, 0.3);
303        let proc = WavProcessor;
304        assert!(proc.handles("audio/wav"));
305        let r = ingest_with(&proc, "urn:audio:clip", "audio/wav", 0xA0D10, &wav);
306        let subj = r.container.primary.subject();
307        assert!(
308            by_topic(&r.quins, "audio").contains(&subj),
309            "findable as audio"
310        );
311    }
312}
qualia_core_db/hypermedia/processors/audio.rs

qualia_core_db/hypermedia/processors/
audio.rs
