1use bytemuck::{Pod, Zeroable};
7
8use crate::tensor::Tensor10D;
9
10pub const SPECTRAL_SIDECAR_MAGIC: u32 = 0x5134_4155; pub const SPECTRAL_PREVIEW_BINS: usize = 64;
12
13pub const SIDECAR_KIND_STFT: u16 = 0;
15pub const SIDECAR_KIND_CQT: u16 = 1;
16
17#[repr(C)]
19#[derive(Debug, Clone, Copy, PartialEq, Pod, Zeroable)]
20pub struct AudioSpectralSidecarHeader {
21 pub magic: u32,
23 pub version: u16,
25 pub _pad: u16,
27 pub bin_count: u32,
29 pub frame_count: u32,
31 pub sample_rate: u32,
33}
34
35impl AudioSpectralSidecarHeader {
36 pub const VERSION: u16 = 1;
37 pub const VERSION_V2: u16 = 2;
40
41 #[inline]
43 pub const fn empty() -> Self {
44 Self {
45 magic: SPECTRAL_SIDECAR_MAGIC,
46 version: Self::VERSION,
47 _pad: 0,
48 bin_count: 0,
49 frame_count: 0,
50 sample_rate: 48_000,
51 }
52 }
53
54 #[inline]
57 pub fn payload_bytes(&self) -> usize {
58 (self.bin_count as usize)
59 .saturating_mul(self.frame_count as usize)
60 .saturating_mul(4)
61 }
62
63 #[inline]
66 pub fn is_valid(&self) -> bool {
67 self.magic == SPECTRAL_SIDECAR_MAGIC
68 && (self.version == Self::VERSION || self.version == Self::VERSION_V2)
69 && self.bin_count > 0
70 && self.frame_count > 0
71 }
72}
73
74#[inline]
76pub fn parse_sidecar_header(bytes: &[u8]) -> Option<AudioSpectralSidecarHeader> {
77 if bytes.len() < std::mem::size_of::<AudioSpectralSidecarHeader>() {
78 return None;
79 }
80 let header = bytemuck::pod_read_unaligned::<AudioSpectralSidecarHeader>(
81 &bytes[..std::mem::size_of::<AudioSpectralSidecarHeader>()],
82 );
83 header.is_valid().then_some(header)
84}
85
86#[derive(Debug, Clone, Copy)]
88pub struct AudioSpectralSheetView<'a> {
89 pub alpha: f32,
91 pub mu: f32,
93 pub bins: &'a [f32],
95 pub position: [f32; 3],
97 pub track_v: u8,
99 pub manifold_w: u8,
101}
102
103impl<'a> AudioSpectralSheetView<'a> {
104 #[inline]
107 pub fn from_tensor_preview(t: &Tensor10D, bins: &'a [f32]) -> Self {
108 Self {
109 alpha: t.alpha,
110 mu: t.mu,
111 bins,
112 position: [t.x, t.y, t.z],
113 track_v: t.v.clamp(0.0, 255.0) as u8,
114 manifold_w: t.w.clamp(0.0, 255.0) as u8,
115 }
116 }
117}
118
119#[inline]
121pub fn preview_bins_from_tensor(t: &Tensor10D) -> [f32; SPECTRAL_PREVIEW_BINS] {
122 let src = [t.q, t.v, t.w, t.x, t.y, t.z, t.t, t.alpha, t.mu, t.sigma];
123 let mut out = [0.0_f32; SPECTRAL_PREVIEW_BINS];
124 for i in 0..SPECTRAL_PREVIEW_BINS {
125 let f = (i as f32 / SPECTRAL_PREVIEW_BINS as f32) * 10.0;
126 let idx = f.floor() as usize;
127 let next = (idx + 1).min(9);
128 let frac = f - idx as f32;
129 out[i] = src[idx] * (1.0 - frac) + src[next] * frac;
130 }
131 out
132}
133
134#[inline]
136pub fn sidecar_frame_view(bytes: &[u8], frame_index: u32) -> Option<&[f32]> {
137 let header = parse_sidecar_header(bytes)?;
138 let frame_count = usize::try_from(header.frame_count).ok()?;
139 let bins = usize::try_from(header.bin_count).ok()?;
140 let start = std::mem::size_of::<AudioSpectralSidecarHeader>();
141 let end = start.checked_add(spectral_plane_bytes(frame_count, bins)?)?;
142 spectral_frame_view(bytes.get(start..end)?, frame_index, frame_count, bins)
143}
144
145#[inline]
147fn spectral_plane_bytes(frame_count: usize, width: usize) -> Option<usize> {
148 frame_count
149 .checked_mul(width)?
150 .checked_mul(std::mem::size_of::<f32>())
151}
152
153#[inline]
154fn spectral_frame_view(
155 plane: &[u8],
156 frame_index: u32,
157 frame_count: usize,
158 width: usize,
159) -> Option<&[f32]> {
160 let frame_index = usize::try_from(frame_index).ok()?;
161 if frame_index >= frame_count {
162 return None;
163 }
164 let frame_bytes = spectral_plane_bytes(1, width)?;
165 let start = frame_index.checked_mul(frame_bytes)?;
166 let end = start.checked_add(frame_bytes)?;
167 bytemuck::try_cast_slice(plane.get(start..end)?).ok()
168}
169
170#[inline]
172pub fn copy_sidecar_frame_to_preview_bins(
173 bytes: &[u8],
174 frame_index: u32,
175 out: &mut [f32; SPECTRAL_PREVIEW_BINS],
176) -> bool {
177 let Some(frame) = sidecar_frame_view(bytes, frame_index) else {
178 return false;
179 };
180 let n = frame.len().min(SPECTRAL_PREVIEW_BINS);
181 out[..n].copy_from_slice(&frame[..n]);
182 for slot in out.iter_mut().skip(n) {
183 *slot = 0.0;
184 }
185 true
186}
187
188pub const SPECTRAL_V2_SUBHEADER_MAGIC: u32 = 0x324D_3451;
194
195#[repr(C)]
200#[derive(Debug, Clone, Copy, PartialEq, Pod, Zeroable)]
201pub struct SpectralV2SubHeader {
202 pub magic: u32,
203 pub n_mel: u32,
204 pub n_mfcc: u32,
205}
206
207impl SpectralV2SubHeader {
208 #[inline]
211 pub fn is_valid(&self) -> bool {
212 self.magic == SPECTRAL_V2_SUBHEADER_MAGIC && self.n_mel > 0 && self.n_mfcc > 0
213 }
214}
215
216#[inline]
219fn v2_subheader_offset(bytes: &[u8]) -> Option<(AudioSpectralSidecarHeader, usize)> {
220 let header = parse_sidecar_header(bytes)?;
221 if header.version != AudioSpectralSidecarHeader::VERSION_V2 {
222 return None;
223 }
224 let header_bytes = std::mem::size_of::<AudioSpectralSidecarHeader>();
225 let plane0_bytes = spectral_plane_bytes(
226 usize::try_from(header.frame_count).ok()?,
227 usize::try_from(header.bin_count).ok()?,
228 )?;
229 let plane0_end = header_bytes.checked_add(plane0_bytes)?;
230 if bytes.len() < plane0_end.checked_add(std::mem::size_of::<SpectralV2SubHeader>())? {
231 return None;
232 }
233 Some((header, plane0_end))
234}
235
236#[inline]
238pub fn parse_v2_subheader(bytes: &[u8]) -> Option<SpectralV2SubHeader> {
239 let (_, off) = v2_subheader_offset(bytes)?;
240 let end = off.checked_add(std::mem::size_of::<SpectralV2SubHeader>())?;
241 let sub = bytemuck::pod_read_unaligned::<SpectralV2SubHeader>(bytes.get(off..end)?);
242 sub.is_valid().then_some(sub)
243}
244
245#[inline]
247fn v2_feature_planes(
248 bytes: &[u8],
249) -> Option<(
250 AudioSpectralSidecarHeader,
251 SpectralV2SubHeader,
252 &[u8],
253 &[u8],
254)> {
255 let (header, sub_off) = v2_subheader_offset(bytes)?;
256 let sub = parse_v2_subheader(bytes)?;
257 let frames = usize::try_from(header.frame_count).ok()?;
258 let mel_bytes = spectral_plane_bytes(frames, usize::try_from(sub.n_mel).ok()?)?;
259 let mfcc_bytes = spectral_plane_bytes(frames, usize::try_from(sub.n_mfcc).ok()?)?;
260 let mel_start = sub_off.checked_add(std::mem::size_of::<SpectralV2SubHeader>())?;
261 let mfcc_start = mel_start.checked_add(mel_bytes)?;
262 let end = mfcc_start.checked_add(mfcc_bytes)?;
263 Some((
264 header,
265 sub,
266 bytes.get(mel_start..mfcc_start)?,
267 bytes.get(mfcc_start..end)?,
268 ))
269}
270
271#[inline]
274pub fn sidecar_mel_frame_view(bytes: &[u8], frame_index: u32) -> Option<&[f32]> {
275 let (header, sub, mel, _) = v2_feature_planes(bytes)?;
276 spectral_frame_view(
277 mel,
278 frame_index,
279 usize::try_from(header.frame_count).ok()?,
280 usize::try_from(sub.n_mel).ok()?,
281 )
282}
283
284#[inline]
287pub fn sidecar_mfcc_frame_view(bytes: &[u8], frame_index: u32) -> Option<&[f32]> {
288 let (header, sub, _, mfcc) = v2_feature_planes(bytes)?;
289 spectral_frame_view(
290 mfcc,
291 frame_index,
292 usize::try_from(header.frame_count).ok()?,
293 usize::try_from(sub.n_mfcc).ok()?,
294 )
295}
296
297#[inline]
300pub fn v2_sidecar_size(frame_count: usize, bin_count: usize, n_mel: usize, n_mfcc: usize) -> usize {
301 checked_v2_sidecar_size(frame_count, bin_count, n_mel, n_mfcc).unwrap_or(usize::MAX)
302}
303
304#[inline]
306pub fn checked_v2_sidecar_size(
307 frame_count: usize,
308 bin_count: usize,
309 n_mel: usize,
310 n_mfcc: usize,
311) -> Option<usize> {
312 std::mem::size_of::<AudioSpectralSidecarHeader>()
313 .checked_add(spectral_plane_bytes(frame_count, bin_count)?)?
314 .checked_add(std::mem::size_of::<SpectralV2SubHeader>())?
315 .checked_add(spectral_plane_bytes(frame_count, n_mel)?)?
316 .checked_add(spectral_plane_bytes(frame_count, n_mfcc)?)
317}
318
319#[cfg(not(target_arch = "wasm32"))]
335pub fn bake_spectral_v2_from_samples(
336 samples: &[f32],
337 frame_size: usize,
338 hop: usize,
339 sample_rate: u32,
340 n_mel: usize,
341 n_mfcc: usize,
342 out: &mut [u8],
343) -> Result<usize, crate::audio::stft_bake::StftBakeError> {
344 use crate::audio::stft::{forward_stft, stft_magnitudes};
345 use crate::audio::stft_bake::StftBakeError;
346 use qualia_audio::features::mel::{build_mel_bank, mel_bands, mfcc};
347
348 if n_mel == 0 || n_mfcc == 0 || n_mfcc > n_mel {
349 return Err(StftBakeError::InvalidFrameCount);
350 }
351
352 let spec = forward_stft(samples, frame_size, hop)?;
353 let mags = stft_magnitudes(&spec);
354 let frame_count = mags.len();
355 if frame_count == 0 || frame_count > 4096 {
356 return Err(StftBakeError::InvalidFrameCount);
357 }
358 let bin_count = frame_size / 2 + 1;
359
360 let need = checked_v2_sidecar_size(frame_count, bin_count, n_mel, n_mfcc)
361 .ok_or(StftBakeError::InvalidFrameCount)?;
362 if out.len() < need {
363 return Err(StftBakeError::OutputTooSmall);
364 }
365
366 let mut bank = vec![0.0f32; n_mel * bin_count];
368 build_mel_bank(
369 bin_count,
370 n_mel,
371 sample_rate as f32,
372 0.0,
373 sample_rate as f32 * 0.5,
374 &mut bank,
375 )
376 .map_err(|_| StftBakeError::InvalidFrameCount)?;
377
378 let header = AudioSpectralSidecarHeader {
379 magic: SPECTRAL_SIDECAR_MAGIC,
380 version: AudioSpectralSidecarHeader::VERSION_V2,
381 _pad: SIDECAR_KIND_STFT,
382 bin_count: bin_count as u32,
383 frame_count: frame_count as u32,
384 sample_rate,
385 };
386
387 let header_bytes = std::mem::size_of::<AudioSpectralSidecarHeader>();
388 out[..header_bytes].copy_from_slice(bytemuck::bytes_of(&header));
389
390 let mut off = header_bytes;
392 for frame_mags in &mags {
393 if frame_mags.len() != bin_count {
395 return Err(StftBakeError::InvalidFrameCount);
396 }
397 out[off..off + bin_count * 4].copy_from_slice(bytemuck::cast_slice(frame_mags));
398 off += bin_count * 4;
399 }
400
401 let sub = SpectralV2SubHeader {
403 magic: SPECTRAL_V2_SUBHEADER_MAGIC,
404 n_mel: n_mel as u32,
405 n_mfcc: n_mfcc as u32,
406 };
407 out[off..off + std::mem::size_of::<SpectralV2SubHeader>()]
408 .copy_from_slice(bytemuck::bytes_of(&sub));
409 off += std::mem::size_of::<SpectralV2SubHeader>();
410
411 let mel_base = off;
413 let mfcc_base = mel_base + frame_count * n_mel * 4;
414 let mut power = vec![0.0f32; bin_count];
415 let mut mel_out = vec![0.0f32; n_mel];
416 let mut mfcc_out = vec![0.0f32; n_mfcc];
417 let mut scratch = vec![0.0f32; 2 * n_mel];
418 for (f, frame_mags) in mags.iter().enumerate() {
419 for (p, &m) in power.iter_mut().zip(frame_mags.iter()) {
420 *p = m * m;
421 }
422 mel_bands(&power, &bank, n_mel, &mut mel_out)
423 .map_err(|_| StftBakeError::InvalidFrameCount)?;
424 mfcc(&power, &bank, n_mel, n_mfcc, &mut mfcc_out, &mut scratch)
425 .map_err(|_| StftBakeError::InvalidFrameCount)?;
426
427 let mel_off = mel_base + f * n_mel * 4;
428 out[mel_off..mel_off + n_mel * 4].copy_from_slice(bytemuck::cast_slice(&mel_out));
429 let mfcc_off = mfcc_base + f * n_mfcc * 4;
430 out[mfcc_off..mfcc_off + n_mfcc * 4].copy_from_slice(bytemuck::cast_slice(&mfcc_out));
431 }
432
433 Ok(need)
434}
435
436#[cfg(test)]
437mod tests {
438 use super::*;
439
440 #[test]
441 fn sidecar_header_size_and_magic() {
442 assert_eq!(std::mem::size_of::<AudioSpectralSidecarHeader>(), 20);
443 let h = AudioSpectralSidecarHeader::empty();
444 assert_eq!(h.magic, SPECTRAL_SIDECAR_MAGIC);
445 }
446
447 #[test]
448 fn preview_bins_len_fixed() {
449 let t = Tensor10D::new(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0);
450 let bins = preview_bins_from_tensor(&t);
451 assert_eq!(bins.len(), SPECTRAL_PREVIEW_BINS);
452 assert!(bins[0] > 0.0);
453 assert!(bins[63] > 0.0);
454 }
455
456 #[test]
457 fn parse_header_rejects_short_slice() {
458 assert!(parse_sidecar_header(&[0u8; 8]).is_none());
459 }
460
461 #[test]
462 fn parse_header_accepts_valid() {
463 let h = AudioSpectralSidecarHeader {
464 magic: SPECTRAL_SIDECAR_MAGIC,
465 version: 1,
466 _pad: 0,
467 bin_count: 64,
468 frame_count: 100,
469 sample_rate: 48_000,
470 };
471 let bytes = bytemuck::bytes_of(&h);
472 let parsed = parse_sidecar_header(bytes).expect("valid header");
473 assert_eq!(parsed.bin_count, 64);
474 assert_eq!(parsed.payload_bytes(), 64 * 100 * 4);
475 }
476
477 #[test]
478 fn v2_subheader_size_is_12() {
479 assert_eq!(std::mem::size_of::<SpectralV2SubHeader>(), 12);
480 }
481
482 #[test]
483 fn v1_header_still_valid_after_widening() {
484 let h = AudioSpectralSidecarHeader {
486 magic: SPECTRAL_SIDECAR_MAGIC,
487 version: AudioSpectralSidecarHeader::VERSION,
488 _pad: SIDECAR_KIND_STFT,
489 bin_count: 64,
490 frame_count: 4,
491 sample_rate: 48_000,
492 };
493 let mut buf = vec![0u8; std::mem::size_of::<AudioSpectralSidecarHeader>() + 64 * 4 * 4];
494 buf[..std::mem::size_of::<AudioSpectralSidecarHeader>()]
495 .copy_from_slice(bytemuck::bytes_of(&h));
496 let po = std::mem::size_of::<AudioSpectralSidecarHeader>();
498 let f2 = po + 2 * 64 * 4;
499 buf[f2..f2 + 4].copy_from_slice(&7.5f32.to_le_bytes());
500 let parsed = parse_sidecar_header(&buf).expect("v1 still valid");
501 assert_eq!(parsed.version, 1);
502 let frame = sidecar_frame_view(&buf, 2).expect("frame 2");
503 assert_eq!(frame.len(), 64);
504 assert_eq!(frame[0], 7.5);
505 assert!(parse_v2_subheader(&buf).is_none());
507 assert!(sidecar_mel_frame_view(&buf, 0).is_none());
508 }
509
510 #[cfg(not(target_arch = "wasm32"))]
511 #[test]
512 fn v2_bake_round_trips_and_tone_energy_is_concentrated() {
513 use core::f32::consts::TAU;
514 const FRAME: usize = 256;
515 const HOP: usize = 128;
516 const N_MEL: usize = 26;
517 const N_MFCC: usize = 13;
518 let sr = 16_000u32;
519 let freq = 2000.0f32;
521 let samples: Vec<f32> = (0..FRAME * 6)
522 .map(|i| (TAU * freq * i as f32 / sr as f32).sin())
523 .collect();
524
525 let bin_count = FRAME / 2 + 1;
526 let frame_count = (samples.len() - FRAME) / HOP + 1;
528 let mut buf = vec![0u8; v2_sidecar_size(frame_count, bin_count, N_MEL, N_MFCC)];
529 let n = bake_spectral_v2_from_samples(&samples, FRAME, HOP, sr, N_MEL, N_MFCC, &mut buf)
530 .expect("v2 bake");
531 assert_eq!(n, buf.len());
532
533 let header = parse_sidecar_header(&buf).expect("v2 header parses");
534 assert_eq!(header.version, AudioSpectralSidecarHeader::VERSION_V2);
535 assert_eq!(header.bin_count as usize, bin_count);
536 assert_eq!(header.frame_count as usize, frame_count);
537
538 let sub = parse_v2_subheader(&buf).expect("v2 subheader parses");
539 assert_eq!(sub.n_mel as usize, N_MEL);
540 assert_eq!(sub.n_mfcc as usize, N_MFCC);
541
542 let mel = sidecar_mel_frame_view(&buf, 1).expect("mel frame 1");
544 assert_eq!(mel.len(), N_MEL);
545 let mfcc_v = sidecar_mfcc_frame_view(&buf, 1).expect("mfcc frame 1");
546 assert_eq!(mfcc_v.len(), N_MFCC);
547 assert!(sidecar_mel_frame_view(&buf, header.frame_count).is_none());
549
550 let total: f32 = mel.iter().sum();
553 assert!(total > 0.0, "mel energy present for a tone");
554 let (peak_idx, &peak_val) = mel
555 .iter()
556 .enumerate()
557 .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
558 .unwrap();
559 assert!(
560 peak_val > 0.25 * total,
561 "tone not concentrated: peak {peak_val} vs total {total}"
562 );
563 assert!(
565 peak_idx > 0 && peak_idx < N_MEL - 1,
566 "peak mel band {peak_idx} at edge"
567 );
568 }
569
570 #[cfg(not(target_arch = "wasm32"))]
571 #[test]
572 fn v1_reader_reads_plane0_of_v2_file() {
573 use core::f32::consts::TAU;
574 const FRAME: usize = 256;
575 const HOP: usize = 128;
576 let sr = 16_000u32;
577 let samples: Vec<f32> = (0..FRAME * 4)
578 .map(|i| (TAU * 1000.0 * i as f32 / sr as f32).sin())
579 .collect();
580 let bin_count = FRAME / 2 + 1;
581 let frame_count = (samples.len() - FRAME) / HOP + 1;
582 let mut buf = vec![0u8; v2_sidecar_size(frame_count, bin_count, 20, 10)];
583 bake_spectral_v2_from_samples(&samples, FRAME, HOP, sr, 20, 10, &mut buf).expect("bake");
584
585 let header = parse_sidecar_header(&buf).expect("header");
588 assert_eq!(header.bin_count as usize, bin_count);
589 let frame0 = sidecar_frame_view(&buf, 0).expect("plane-0 frame 0");
590 assert_eq!(frame0.len(), bin_count);
591 assert!(
592 frame0.iter().any(|&v| v > 0.0),
593 "plane-0 has real STFT energy"
594 );
595 let last = sidecar_frame_view(&buf, header.frame_count - 1).expect("last plane-0 frame");
597 assert_eq!(last.len(), bin_count);
598 }
599}
600
601#[cfg(test)]
602#[path = "audio_spectral_sheet_view_tests.rs"]
603mod view_tests;