1use bytemuck::{Pod, Zeroable};
7
8use crate::tensor::Tensor10D;
9
10pub const SPECTRAL_SIDECAR_MAGIC: u32 = 0x5134_4155; pub const SPECTRAL_PREVIEW_BINS: usize = 64;
12
13pub const SIDECAR_KIND_STFT: u16 = 0;
15pub const SIDECAR_KIND_CQT: u16 = 1;
16
17#[repr(C)]
19#[derive(Debug, Clone, Copy, PartialEq, Pod, Zeroable)]
20pub struct AudioSpectralSidecarHeader {
21 pub magic: u32,
22 pub version: u16,
23 pub _pad: u16,
24 pub bin_count: u32,
25 pub frame_count: u32,
26 pub sample_rate: u32,
27}
28
29impl AudioSpectralSidecarHeader {
30 pub const VERSION: u16 = 1;
31 pub const VERSION_V2: u16 = 2;
34
35 #[inline]
36 pub const fn empty() -> Self {
37 Self {
38 magic: SPECTRAL_SIDECAR_MAGIC,
39 version: Self::VERSION,
40 _pad: 0,
41 bin_count: 0,
42 frame_count: 0,
43 sample_rate: 48_000,
44 }
45 }
46
47 #[inline]
48 pub fn payload_bytes(&self) -> usize {
49 (self.bin_count as usize)
50 .saturating_mul(self.frame_count as usize)
51 .saturating_mul(4)
52 }
53
54 #[inline]
55 pub fn is_valid(&self) -> bool {
56 self.magic == SPECTRAL_SIDECAR_MAGIC
57 && (self.version == Self::VERSION || self.version == Self::VERSION_V2)
58 && self.bin_count > 0
59 && self.frame_count > 0
60 }
61}
62
63#[inline]
65pub fn parse_sidecar_header(bytes: &[u8]) -> Option<AudioSpectralSidecarHeader> {
66 if bytes.len() < std::mem::size_of::<AudioSpectralSidecarHeader>() {
67 return None;
68 }
69 let header = bytemuck::pod_read_unaligned::<AudioSpectralSidecarHeader>(
70 &bytes[..std::mem::size_of::<AudioSpectralSidecarHeader>()],
71 );
72 header.is_valid().then_some(header)
73}
74
75#[derive(Debug, Clone, Copy)]
77pub struct AudioSpectralSheetView<'a> {
78 pub alpha: f32,
79 pub mu: f32,
80 pub bins: &'a [f32],
81 pub position: [f32; 3],
82 pub track_v: u8,
83 pub manifold_w: u8,
84}
85
86impl<'a> AudioSpectralSheetView<'a> {
87 #[inline]
88 pub fn from_tensor_preview(t: &Tensor10D, bins: &'a [f32]) -> Self {
89 Self {
90 alpha: t.alpha,
91 mu: t.mu,
92 bins,
93 position: [t.x, t.y, t.z],
94 track_v: t.v.clamp(0.0, 255.0) as u8,
95 manifold_w: t.w.clamp(0.0, 255.0) as u8,
96 }
97 }
98}
99
100#[inline]
102pub fn preview_bins_from_tensor(t: &Tensor10D) -> [f32; SPECTRAL_PREVIEW_BINS] {
103 let src = [t.q, t.v, t.w, t.x, t.y, t.z, t.t, t.alpha, t.mu, t.sigma];
104 let mut out = [0.0_f32; SPECTRAL_PREVIEW_BINS];
105 for i in 0..SPECTRAL_PREVIEW_BINS {
106 let f = (i as f32 / SPECTRAL_PREVIEW_BINS as f32) * 10.0;
107 let idx = f.floor() as usize;
108 let next = (idx + 1).min(9);
109 let frac = f - idx as f32;
110 out[i] = src[idx] * (1.0 - frac) + src[next] * frac;
111 }
112 out
113}
114
115#[inline]
117pub fn sidecar_frame_view<'a>(bytes: &'a [u8], frame_index: u32) -> Option<&'a [f32]> {
118 let header = parse_sidecar_header(bytes)?;
119 let header_bytes = std::mem::size_of::<AudioSpectralSidecarHeader>();
120 let payload = &bytes[header_bytes..];
121 if frame_index >= header.frame_count {
122 return None;
123 }
124 let bins = header.bin_count as usize;
125 let offset = frame_index as usize * bins;
126 let need = offset.saturating_add(bins).saturating_mul(4);
127 if payload.len() < need {
128 return None;
129 }
130 let floats: &[f32] =
131 bytemuck::try_cast_slice(&payload[offset * 4..offset * 4 + bins * 4]).ok()?;
132 Some(floats)
133}
134
135#[inline]
137pub fn copy_sidecar_frame_to_preview_bins(
138 bytes: &[u8],
139 frame_index: u32,
140 out: &mut [f32; SPECTRAL_PREVIEW_BINS],
141) -> bool {
142 let Some(frame) = sidecar_frame_view(bytes, frame_index) else {
143 return false;
144 };
145 let n = frame.len().min(SPECTRAL_PREVIEW_BINS);
146 out[..n].copy_from_slice(&frame[..n]);
147 for slot in out.iter_mut().skip(n) {
148 *slot = 0.0;
149 }
150 true
151}
152
153pub const SPECTRAL_V2_SUBHEADER_MAGIC: u32 = 0x324D_3451;
159
160#[repr(C)]
165#[derive(Debug, Clone, Copy, PartialEq, Pod, Zeroable)]
166pub struct SpectralV2SubHeader {
167 pub magic: u32,
168 pub n_mel: u32,
169 pub n_mfcc: u32,
170}
171
172impl SpectralV2SubHeader {
173 #[inline]
174 pub fn is_valid(&self) -> bool {
175 self.magic == SPECTRAL_V2_SUBHEADER_MAGIC && self.n_mel > 0 && self.n_mfcc > 0
176 }
177}
178
179#[inline]
182fn v2_subheader_offset(bytes: &[u8]) -> Option<(AudioSpectralSidecarHeader, usize)> {
183 let header = parse_sidecar_header(bytes)?;
184 if header.version != AudioSpectralSidecarHeader::VERSION_V2 {
185 return None;
186 }
187 let header_bytes = std::mem::size_of::<AudioSpectralSidecarHeader>();
188 let plane0_end = header_bytes.checked_add(header.payload_bytes())?;
189 if bytes.len() < plane0_end.checked_add(std::mem::size_of::<SpectralV2SubHeader>())? {
190 return None;
191 }
192 Some((header, plane0_end))
193}
194
195#[inline]
197pub fn parse_v2_subheader(bytes: &[u8]) -> Option<SpectralV2SubHeader> {
198 let (_, off) = v2_subheader_offset(bytes)?;
199 let sub = bytemuck::pod_read_unaligned::<SpectralV2SubHeader>(
200 &bytes[off..off + std::mem::size_of::<SpectralV2SubHeader>()],
201 );
202 sub.is_valid().then_some(sub)
203}
204
205#[inline]
208pub fn sidecar_mel_frame_view(bytes: &[u8], frame_index: u32) -> Option<&[f32]> {
209 let (header, sub_off) = v2_subheader_offset(bytes)?;
210 let sub_size = std::mem::size_of::<SpectralV2SubHeader>();
211 let sub =
212 bytemuck::pod_read_unaligned::<SpectralV2SubHeader>(&bytes[sub_off..sub_off + sub_size]);
213 if !sub.is_valid() || frame_index >= header.frame_count {
214 return None;
215 }
216 let n_mel = sub.n_mel as usize;
217 let mel_base = sub_off + sub_size;
218 let off = mel_base + frame_index as usize * n_mel * 4;
219 let end = off.checked_add(n_mel * 4)?;
220 if bytes.len() < end {
221 return None;
222 }
223 bytemuck::try_cast_slice(&bytes[off..end]).ok()
224}
225
226#[inline]
229pub fn sidecar_mfcc_frame_view(bytes: &[u8], frame_index: u32) -> Option<&[f32]> {
230 let (header, sub_off) = v2_subheader_offset(bytes)?;
231 let sub_size = std::mem::size_of::<SpectralV2SubHeader>();
232 let sub =
233 bytemuck::pod_read_unaligned::<SpectralV2SubHeader>(&bytes[sub_off..sub_off + sub_size]);
234 if !sub.is_valid() || frame_index >= header.frame_count {
235 return None;
236 }
237 let n_mel = sub.n_mel as usize;
238 let n_mfcc = sub.n_mfcc as usize;
239 let mel_bytes = header.frame_count as usize * n_mel * 4;
240 let mfcc_base = sub_off + sub_size + mel_bytes;
241 let off = mfcc_base + frame_index as usize * n_mfcc * 4;
242 let end = off.checked_add(n_mfcc * 4)?;
243 if bytes.len() < end {
244 return None;
245 }
246 bytemuck::try_cast_slice(&bytes[off..end]).ok()
247}
248
249#[inline]
251pub fn v2_sidecar_size(frame_count: usize, bin_count: usize, n_mel: usize, n_mfcc: usize) -> usize {
252 std::mem::size_of::<AudioSpectralSidecarHeader>()
253 + frame_count * bin_count * 4
254 + std::mem::size_of::<SpectralV2SubHeader>()
255 + frame_count * n_mel * 4
256 + frame_count * n_mfcc * 4
257}
258
259#[cfg(not(target_arch = "wasm32"))]
275pub fn bake_spectral_v2_from_samples(
276 samples: &[f32],
277 frame_size: usize,
278 hop: usize,
279 sample_rate: u32,
280 n_mel: usize,
281 n_mfcc: usize,
282 out: &mut [u8],
283) -> Result<usize, crate::audio::stft_bake::StftBakeError> {
284 use crate::audio::stft::{forward_stft, stft_magnitudes};
285 use crate::audio::stft_bake::StftBakeError;
286 use qualia_audio::features::mel::{build_mel_bank, mel_bands, mfcc};
287
288 if n_mel == 0 || n_mfcc == 0 || n_mfcc > n_mel {
289 return Err(StftBakeError::InvalidFrameCount);
290 }
291
292 let spec = forward_stft(samples, frame_size, hop)?;
293 let mags = stft_magnitudes(&spec);
294 let frame_count = mags.len();
295 if frame_count == 0 || frame_count > 4096 {
296 return Err(StftBakeError::InvalidFrameCount);
297 }
298 let bin_count = frame_size / 2 + 1;
299
300 let need = v2_sidecar_size(frame_count, bin_count, n_mel, n_mfcc);
301 if out.len() < need {
302 return Err(StftBakeError::OutputTooSmall);
303 }
304
305 let mut bank = vec![0.0f32; n_mel * bin_count];
307 build_mel_bank(
308 bin_count,
309 n_mel,
310 sample_rate as f32,
311 0.0,
312 sample_rate as f32 * 0.5,
313 &mut bank,
314 )
315 .map_err(|_| StftBakeError::InvalidFrameCount)?;
316
317 let header = AudioSpectralSidecarHeader {
318 magic: SPECTRAL_SIDECAR_MAGIC,
319 version: AudioSpectralSidecarHeader::VERSION_V2,
320 _pad: SIDECAR_KIND_STFT,
321 bin_count: bin_count as u32,
322 frame_count: frame_count as u32,
323 sample_rate,
324 };
325
326 let header_bytes = std::mem::size_of::<AudioSpectralSidecarHeader>();
327 out[..header_bytes].copy_from_slice(bytemuck::bytes_of(&header));
328
329 let mut off = header_bytes;
331 for frame_mags in &mags {
332 if frame_mags.len() != bin_count {
334 return Err(StftBakeError::InvalidFrameCount);
335 }
336 out[off..off + bin_count * 4].copy_from_slice(bytemuck::cast_slice(frame_mags));
337 off += bin_count * 4;
338 }
339
340 let sub = SpectralV2SubHeader {
342 magic: SPECTRAL_V2_SUBHEADER_MAGIC,
343 n_mel: n_mel as u32,
344 n_mfcc: n_mfcc as u32,
345 };
346 out[off..off + std::mem::size_of::<SpectralV2SubHeader>()]
347 .copy_from_slice(bytemuck::bytes_of(&sub));
348 off += std::mem::size_of::<SpectralV2SubHeader>();
349
350 let mel_base = off;
352 let mfcc_base = mel_base + frame_count * n_mel * 4;
353 let mut power = vec![0.0f32; bin_count];
354 let mut mel_out = vec![0.0f32; n_mel];
355 let mut mfcc_out = vec![0.0f32; n_mfcc];
356 let mut scratch = vec![0.0f32; 2 * n_mel];
357 for (f, frame_mags) in mags.iter().enumerate() {
358 for (p, &m) in power.iter_mut().zip(frame_mags.iter()) {
359 *p = m * m;
360 }
361 mel_bands(&power, &bank, n_mel, &mut mel_out)
362 .map_err(|_| StftBakeError::InvalidFrameCount)?;
363 mfcc(&power, &bank, n_mel, n_mfcc, &mut mfcc_out, &mut scratch)
364 .map_err(|_| StftBakeError::InvalidFrameCount)?;
365
366 let mel_off = mel_base + f * n_mel * 4;
367 out[mel_off..mel_off + n_mel * 4].copy_from_slice(bytemuck::cast_slice(&mel_out));
368 let mfcc_off = mfcc_base + f * n_mfcc * 4;
369 out[mfcc_off..mfcc_off + n_mfcc * 4].copy_from_slice(bytemuck::cast_slice(&mfcc_out));
370 }
371
372 Ok(need)
373}
374
375#[cfg(test)]
376mod tests {
377 use super::*;
378
379 #[test]
380 fn sidecar_header_size_and_magic() {
381 assert_eq!(std::mem::size_of::<AudioSpectralSidecarHeader>(), 20);
382 let h = AudioSpectralSidecarHeader::empty();
383 assert_eq!(h.magic, SPECTRAL_SIDECAR_MAGIC);
384 }
385
386 #[test]
387 fn preview_bins_len_fixed() {
388 let t = Tensor10D::new(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0);
389 let bins = preview_bins_from_tensor(&t);
390 assert_eq!(bins.len(), SPECTRAL_PREVIEW_BINS);
391 assert!(bins[0] > 0.0);
392 assert!(bins[63] > 0.0);
393 }
394
395 #[test]
396 fn parse_header_rejects_short_slice() {
397 assert!(parse_sidecar_header(&[0u8; 8]).is_none());
398 }
399
400 #[test]
401 fn parse_header_accepts_valid() {
402 let h = AudioSpectralSidecarHeader {
403 magic: SPECTRAL_SIDECAR_MAGIC,
404 version: 1,
405 _pad: 0,
406 bin_count: 64,
407 frame_count: 100,
408 sample_rate: 48_000,
409 };
410 let bytes = bytemuck::bytes_of(&h);
411 let parsed = parse_sidecar_header(bytes).expect("valid header");
412 assert_eq!(parsed.bin_count, 64);
413 assert_eq!(parsed.payload_bytes(), 64 * 100 * 4);
414 }
415
416 #[test]
417 fn v2_subheader_size_is_12() {
418 assert_eq!(std::mem::size_of::<SpectralV2SubHeader>(), 12);
419 }
420
421 #[test]
422 fn v1_header_still_valid_after_widening() {
423 let h = AudioSpectralSidecarHeader {
425 magic: SPECTRAL_SIDECAR_MAGIC,
426 version: AudioSpectralSidecarHeader::VERSION,
427 _pad: SIDECAR_KIND_STFT,
428 bin_count: 64,
429 frame_count: 4,
430 sample_rate: 48_000,
431 };
432 let mut buf = vec![0u8; std::mem::size_of::<AudioSpectralSidecarHeader>() + 64 * 4 * 4];
433 buf[..std::mem::size_of::<AudioSpectralSidecarHeader>()]
434 .copy_from_slice(bytemuck::bytes_of(&h));
435 let po = std::mem::size_of::<AudioSpectralSidecarHeader>();
437 let f2 = po + 2 * 64 * 4;
438 buf[f2..f2 + 4].copy_from_slice(&7.5f32.to_le_bytes());
439 let parsed = parse_sidecar_header(&buf).expect("v1 still valid");
440 assert_eq!(parsed.version, 1);
441 let frame = sidecar_frame_view(&buf, 2).expect("frame 2");
442 assert_eq!(frame.len(), 64);
443 assert_eq!(frame[0], 7.5);
444 assert!(parse_v2_subheader(&buf).is_none());
446 assert!(sidecar_mel_frame_view(&buf, 0).is_none());
447 }
448
449 #[cfg(not(target_arch = "wasm32"))]
450 #[test]
451 fn v2_bake_round_trips_and_tone_energy_is_concentrated() {
452 use core::f32::consts::TAU;
453 const FRAME: usize = 256;
454 const HOP: usize = 128;
455 const N_MEL: usize = 26;
456 const N_MFCC: usize = 13;
457 let sr = 16_000u32;
458 let freq = 2000.0f32;
460 let samples: Vec<f32> = (0..FRAME * 6)
461 .map(|i| (TAU * freq * i as f32 / sr as f32).sin())
462 .collect();
463
464 let bin_count = FRAME / 2 + 1;
465 let frame_count = (samples.len() - FRAME) / HOP + 1;
467 let mut buf = vec![0u8; v2_sidecar_size(frame_count, bin_count, N_MEL, N_MFCC)];
468 let n = bake_spectral_v2_from_samples(&samples, FRAME, HOP, sr, N_MEL, N_MFCC, &mut buf)
469 .expect("v2 bake");
470 assert_eq!(n, buf.len());
471
472 let header = parse_sidecar_header(&buf).expect("v2 header parses");
473 assert_eq!(header.version, AudioSpectralSidecarHeader::VERSION_V2);
474 assert_eq!(header.bin_count as usize, bin_count);
475 assert_eq!(header.frame_count as usize, frame_count);
476
477 let sub = parse_v2_subheader(&buf).expect("v2 subheader parses");
478 assert_eq!(sub.n_mel as usize, N_MEL);
479 assert_eq!(sub.n_mfcc as usize, N_MFCC);
480
481 let mel = sidecar_mel_frame_view(&buf, 1).expect("mel frame 1");
483 assert_eq!(mel.len(), N_MEL);
484 let mfcc_v = sidecar_mfcc_frame_view(&buf, 1).expect("mfcc frame 1");
485 assert_eq!(mfcc_v.len(), N_MFCC);
486 assert!(sidecar_mel_frame_view(&buf, header.frame_count).is_none());
488
489 let total: f32 = mel.iter().sum();
492 assert!(total > 0.0, "mel energy present for a tone");
493 let (peak_idx, &peak_val) = mel
494 .iter()
495 .enumerate()
496 .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
497 .unwrap();
498 assert!(
499 peak_val > 0.25 * total,
500 "tone not concentrated: peak {peak_val} vs total {total}"
501 );
502 assert!(
504 peak_idx > 0 && peak_idx < N_MEL - 1,
505 "peak mel band {peak_idx} at edge"
506 );
507 }
508
509 #[cfg(not(target_arch = "wasm32"))]
510 #[test]
511 fn v1_reader_reads_plane0_of_v2_file() {
512 use core::f32::consts::TAU;
513 const FRAME: usize = 256;
514 const HOP: usize = 128;
515 let sr = 16_000u32;
516 let samples: Vec<f32> = (0..FRAME * 4)
517 .map(|i| (TAU * 1000.0 * i as f32 / sr as f32).sin())
518 .collect();
519 let bin_count = FRAME / 2 + 1;
520 let frame_count = (samples.len() - FRAME) / HOP + 1;
521 let mut buf = vec![0u8; v2_sidecar_size(frame_count, bin_count, 20, 10)];
522 bake_spectral_v2_from_samples(&samples, FRAME, HOP, sr, 20, 10, &mut buf).expect("bake");
523
524 let header = parse_sidecar_header(&buf).expect("header");
527 assert_eq!(header.bin_count as usize, bin_count);
528 let frame0 = sidecar_frame_view(&buf, 0).expect("plane-0 frame 0");
529 assert_eq!(frame0.len(), bin_count);
530 assert!(
531 frame0.iter().any(|&v| v > 0.0),
532 "plane-0 has real STFT energy"
533 );
534 let last = sidecar_frame_view(&buf, header.frame_count - 1).expect("last plane-0 frame");
536 assert_eq!(last.len(), bin_count);
537 }
538}