Skip to main content

qualia_audio/
pipeline.rs

1//! End-to-end Ears MVP demo path (A5 product surface).
2
3use crate::aed_weights::WeightedAedModel;
4use crate::convert::to_mono_f32;
5use crate::events::ReferenceEventModel;
6use crate::features::{forward_cqt_mono, log_mel_from_mono};
7use crate::hash::media_digest;
8use crate::media_store::{synth_tone_f32, AudioMediaStore, RetentionClass};
9use crate::semantic::compile_auditory_quins;
10use crate::sonify::sonify_events_mono;
11use crate::speech::{decode_for_language, SpeechEncoderWeights};
12use crate::types::{AuditoryEvent, AuditoryModel, TranscriptToken};
13use crate::wav::{decode_wav, encode_wav_i16_mono};
14
15#[derive(Debug, Clone)]
16pub struct EarsDemoResult {
17    pub sample_rate: u32,
18    pub frames: u32,
19    pub n_events: usize,
20    pub n_quins: usize,
21    pub model_hash: u64,
22    pub media_hash: u64,
23    pub is_reference: bool,
24    pub mel_frames: usize,
25    pub cqt_bins: usize,
26    pub cqt_peak: f32,
27    pub event_classes: Vec<u64>,
28    pub events: Vec<AuditoryEvent>,
29    pub note: String,
30}
31
32/// Run synthetic tone → WAV → store → features → events → semantic compile.
33pub fn run_ears_demo(
34    storage_root: Option<&std::path::Path>,
35    freq_hz: f32,
36    sample_rate: u32,
37    duration_ms: u32,
38) -> Result<EarsDemoResult, String> {
39    let frames = ((sample_rate as u64 * duration_ms as u64) / 1000) as usize;
40    let frames = frames.clamp(1024, 48000);
41    let mono = synth_tone_f32(freq_hz, sample_rate, frames, 0.35);
42    let mut i16s = vec![0i16; frames];
43    for (i, &s) in mono.iter().enumerate() {
44        i16s[i] = (s.clamp(-1.0, 1.0) * 32767.0) as i16;
45    }
46    let mut wav = vec![0u8; 44 + frames * 2];
47    let wn = encode_wav_i16_mono(&i16s, sample_rate, &mut wav).map_err(|e| format!("{e:?}"))?;
48    wav.truncate(wn);
49
50    let mut media_hash = media_digest(&wav).hash;
51    if let Some(root) = storage_root {
52        let store = AudioMediaStore::open(root.join("audio_media")).map_err(|e| e)?;
53        let rec = store
54            .import_bytes(
55                &wav,
56                sample_rate,
57                1,
58                frames as u32,
59                RetentionClass::Restricted,
60            )
61            .map_err(|e| e)?;
62        media_hash = rec.digest_u64;
63    }
64
65    let decoded = decode_wav(&wav).map_err(|e| format!("{e:?}"))?;
66    let view = decoded.view();
67    let mut mono2 = vec![0.0f32; frames];
68    to_mono_f32(view, &mut mono2).map_err(|e| format!("{e:?}"))?;
69
70    let mut mel = vec![0.0f32; 32 * 16];
71    let mel_frames = log_mel_from_mono(&mono2, 256, 128, sample_rate, 16, &mut mel)
72        .map_err(|e| format!("{e:?}"))?;
73    let mut cqt = [0.0f32; 24];
74    forward_cqt_mono(&mono2, sample_rate as f32, 55.0, 12, 24, &mut cqt)
75        .map_err(|e| format!("{e:?}"))?;
76    let cqt_peak = cqt.iter().cloned().fold(0.0f32, f32::max);
77
78    let mut model = ReferenceEventModel::new();
79    let mut events = [AuditoryEvent::empty(); 32];
80    let mut tokens = [TranscriptToken::empty(); 8];
81    let mut emb = [0.0f32; 8];
82    let mut ws = vec![0u8; frames * 4];
83    let counts = model
84        .infer_chunk(view, &mut events, &mut tokens, &mut emb, &mut ws)
85        .map_err(|e| format!("{e:?}"))?;
86
87    let mut quins = [crate::semantic::AudioQuin::with_parity(0, 0, 0, 0, 0); 64];
88    let n_q = compile_auditory_quins(
89        crate::hash::MediaDigest {
90            hash: media_hash,
91            byte_len: wav.len() as u64,
92        },
93        &events[..counts.events],
94        model.model_hash(),
95        &mut quins,
96    );
97
98    let classes: Vec<u64> = events[..counts.events]
99        .iter()
100        .map(|e| e.class_hash)
101        .collect();
102
103    Ok(EarsDemoResult {
104        sample_rate,
105        frames: frames as u32,
106        n_events: counts.events,
107        n_quins: n_q,
108        model_hash: model.model_hash(),
109        media_hash,
110        is_reference: true,
111        mel_frames,
112        cqt_bins: 24,
113        cqt_peak,
114        event_classes: classes,
115        events: events[..counts.events].to_vec(),
116        note: "Ears path: tone/WAV → store → log-mel+CQT → reference VAD/events → quins. COMPLETE-WITH-GATE: not P64 AED; no mic."
117            .into(),
118    })
119}
120
121/// Import a WAV file, store, feature, classify.
122pub fn run_ears_on_wav_file(
123    storage_root: Option<&std::path::Path>,
124    path: &std::path::Path,
125) -> Result<EarsDemoResult, String> {
126    let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
127    let decoded = decode_wav(&bytes).map_err(|e| format!("{e:?}"))?;
128    let mut mono = vec![0.0f32; decoded.frames as usize];
129    to_mono_f32(decoded.view(), &mut mono).map_err(|e| format!("{e:?}"))?;
130    // Re-encode mono i16 for store consistency
131    let mut i16s = vec![0i16; mono.len()];
132    for (i, &s) in mono.iter().enumerate() {
133        i16s[i] = (s.clamp(-1.0, 1.0) * 32767.0) as i16;
134    }
135    let mut wav = vec![0u8; 44 + i16s.len() * 2];
136    let wn =
137        encode_wav_i16_mono(&i16s, decoded.sample_rate, &mut wav).map_err(|e| format!("{e:?}"))?;
138    wav.truncate(wn);
139    let media_hash = {
140        let d = media_digest(&wav);
141        if let Some(root) = storage_root {
142            let store = AudioMediaStore::open(root.join("audio_media")).map_err(|e| e)?;
143            let rec = store
144                .import_bytes(
145                    &wav,
146                    decoded.sample_rate,
147                    1,
148                    mono.len() as u32,
149                    RetentionClass::Restricted,
150                )
151                .map_err(|e| e)?;
152            rec.digest_u64
153        } else {
154            d.hash
155        }
156    };
157    let mut mel = vec![0.0f32; 32 * 16];
158    let mel_frames = log_mel_from_mono(&mono, 256, 128, decoded.sample_rate, 16, &mut mel)
159        .map_err(|e| format!("{e:?}"))?;
160    let mut cqt = [0.0f32; 24];
161    forward_cqt_mono(&mono, decoded.sample_rate as f32, 55.0, 12, 24, &mut cqt)
162        .map_err(|e| format!("{e:?}"))?;
163    let view = decoded.view();
164    let mut model = ReferenceEventModel::new();
165    let mut events = [AuditoryEvent::empty(); 32];
166    let mut tokens = [TranscriptToken::empty(); 8];
167    let mut emb = [0.0f32; 8];
168    let mut ws = vec![0u8; mono.len() * 4];
169    let counts = model
170        .infer_chunk(view, &mut events, &mut tokens, &mut emb, &mut ws)
171        .map_err(|e| format!("{e:?}"))?;
172    let mut quins = [crate::semantic::AudioQuin::with_parity(0, 0, 0, 0, 0); 64];
173    let n_q = compile_auditory_quins(
174        crate::hash::MediaDigest {
175            hash: media_hash,
176            byte_len: wav.len() as u64,
177        },
178        &events[..counts.events],
179        model.model_hash(),
180        &mut quins,
181    );
182    Ok(EarsDemoResult {
183        sample_rate: decoded.sample_rate,
184        frames: mono.len() as u32,
185        n_events: counts.events,
186        n_quins: n_q,
187        model_hash: model.model_hash(),
188        media_hash,
189        is_reference: true,
190        mel_frames,
191        cqt_bins: 24,
192        cqt_peak: cqt.iter().cloned().fold(0.0f32, f32::max),
193        event_classes: events[..counts.events]
194            .iter()
195            .map(|e| e.class_hash)
196            .collect(),
197        events: events[..counts.events].to_vec(),
198        note: format!("WAV file import: {}", path.display()),
199    })
200}
201
202/// §18 automated smoke (synthetic; COMPLETE-WITH-GATE on capture/P64).
203pub fn section18_smoke() -> Result<String, String> {
204    let r = run_ears_demo(None, 440.0, 16000, 250)?;
205    if r.n_events == 0 {
206        return Err("no events".into());
207    }
208    if r.n_quins < 2 {
209        return Err("no quins".into());
210    }
211    if r.mel_frames == 0 || r.cqt_peak <= 0.0 {
212        return Err("features missing".into());
213    }
214    Ok(format!(
215        "section18_smoke OK events={} quins={} cqt_peak={:.4} (ref backend; no mic)",
216        r.n_events, r.n_quins, r.cqt_peak
217    ))
218}
219
220/// Weighted AED path (seed P64-shaped head).
221pub fn run_ears_weighted(
222    storage_root: Option<&std::path::Path>,
223    freq_hz: f32,
224    sample_rate: u32,
225    duration_ms: u32,
226) -> Result<EarsDemoResult, String> {
227    let mut r = run_ears_demo(storage_root, freq_hz, sample_rate, duration_ms)?;
228    // Re-run with weighted model on the same synthetic path
229    let frames = r.frames as usize;
230    let mono = crate::media_store::synth_tone_f32(freq_hz, sample_rate, frames, 0.35);
231    let mut i16s = vec![0i16; frames];
232    for (i, &s) in mono.iter().enumerate() {
233        i16s[i] = (s.clamp(-1.0, 1.0) * 32767.0) as i16;
234    }
235    let mut wav = vec![0u8; 44 + frames * 2];
236    let wn = encode_wav_i16_mono(&i16s, sample_rate, &mut wav).map_err(|e| format!("{e:?}"))?;
237    let decoded = decode_wav(&wav[..wn]).map_err(|e| format!("{e:?}"))?;
238    let mut model = WeightedAedModel::from_seed(0xAED1);
239    let mut events = [crate::types::AuditoryEvent::empty(); 32];
240    let mut tokens = [TranscriptToken::empty(); 8];
241    let mut emb = [0.0f32; 4];
242    let mut ws = [0u8; 8];
243    use crate::types::AuditoryModel;
244    let counts = model
245        .infer_chunk(decoded.view(), &mut events, &mut tokens, &mut emb, &mut ws)
246        .map_err(|e| format!("{e:?}"))?;
247    r.n_events = counts.events;
248    r.model_hash = model.model_hash();
249    r.is_reference = false;
250    r.events = events[..counts.events].to_vec();
251    r.event_classes = events[..counts.events]
252        .iter()
253        .map(|e| e.class_hash)
254        .collect();
255    r.note =
256        "Weighted AED seed head (not certified P64). is_reference=false for shape only.".into();
257    Ok(r)
258}
259
260/// Sonify demo events to WAV bytes (U3-style hear navigation).
261pub fn sonify_demo_to_wav(
262    sample_rate: u32,
263    events: &[crate::types::AuditoryEvent],
264    total_frames: usize,
265) -> Result<Vec<u8>, String> {
266    let mut mono = vec![0.0f32; total_frames];
267    sonify_events_mono(events, sample_rate, total_frames, &mut mono);
268    let mut i16s = vec![0i16; total_frames];
269    for (i, &s) in mono.iter().enumerate() {
270        i16s[i] = (s.clamp(-1.0, 1.0) * 32767.0) as i16;
271    }
272    let mut wav = vec![0u8; 44 + total_frames * 2];
273    let n = encode_wav_i16_mono(&i16s, sample_rate, &mut wav).map_err(|e| format!("{e:?}"))?;
274    wav.truncate(n);
275    Ok(wav)
276}
277
278/// Speech phone demo — empty if language unsupported.
279pub fn speech_phone_demo(supported: bool) -> Result<(usize, u64), String> {
280    let w = SpeechEncoderWeights::from_seed(7, 16);
281    let mut mono = vec![0.0f32; 4096];
282    for i in 0..mono.len() {
283        mono[i] = (2.0 * core::f32::consts::PI * 180.0 * i as f32 / 16000.0).sin() * 0.25;
284    }
285    let mut tok = [TranscriptToken::empty(); 32];
286    let n =
287        decode_for_language(&w, &mono, 16000, supported, &mut tok).map_err(|e| format!("{e:?}"))?;
288    Ok((n, w.model_hash))
289}
290
291#[cfg(test)]
292mod tests {
293    use super::*;
294
295    #[test]
296    fn ears_demo_runs() {
297        let r = run_ears_demo(None, 440.0, 16000, 250).unwrap();
298        assert!(r.n_events >= 1);
299        assert!(r.n_quins >= 2);
300        assert!(r.mel_frames >= 1);
301        assert!(r.cqt_peak > 0.0);
302        assert!(r.is_reference);
303    }
304
305    #[test]
306    fn section18_smoke_passes() {
307        let s = section18_smoke().unwrap();
308        assert!(s.contains("OK"));
309    }
310}