Skip to main content

qualia_audio/
cross_modal.rs

1//! Swarm X — eyes + ears correlation (overlap ≠ causality).
2
3use crate::hash::q_hash;
4
5/// Proposed co-occurrence of a visual instance and auditory instance on shared media time.
6#[derive(Debug, Clone, Copy)]
7pub struct AvCorrelationProposal {
8    pub media_hash: u64,
9    pub visual_instance: u64,
10    pub auditory_instance: u64,
11    /// Overlap start/end in media_time_ms.
12    pub overlap_start_ms: u64,
13    pub overlap_end_ms: u64,
14    pub confidence_u16: u16,
15    /// Always false for pure temporal join — causal claims need separate evidence.
16    pub asserts_causality: bool,
17}
18
19/// Interval on media timeline (ms).
20#[derive(Debug, Clone, Copy)]
21pub struct TimeIntervalMs {
22    pub start_ms: u64,
23    pub end_ms: u64,
24    pub instance: u64,
25}
26
27fn overlap(a: TimeIntervalMs, b: TimeIntervalMs) -> Option<(u64, u64)> {
28    let s = a.start_ms.max(b.start_ms);
29    let e = a.end_ms.min(b.end_ms);
30    if s < e {
31        Some((s, e))
32    } else {
33        None
34    }
35}
36
37/// Propose correlations where visual and auditory intervals overlap.
38/// `asserts_causality` is **always false** here.
39pub fn propose_temporal_correlations(
40    media_hash: u64,
41    visual: &[TimeIntervalMs],
42    auditory: &[TimeIntervalMs],
43    out: &mut [AvCorrelationProposal],
44) -> usize {
45    let mut w = 0usize;
46    for v in visual {
47        for a in auditory {
48            if let Some((s, e)) = overlap(*v, *a) {
49                if w >= out.len() {
50                    return w;
51                }
52                let dur = e - s;
53                let conf = ((dur.min(5000) as f32 / 5000.0) * 40000.0) as u16;
54                out[w] = AvCorrelationProposal {
55                    media_hash,
56                    visual_instance: v.instance,
57                    auditory_instance: a.instance,
58                    overlap_start_ms: s,
59                    overlap_end_ms: e,
60                    confidence_u16: conf,
61                    asserts_causality: false,
62                };
63                w += 1;
64            }
65        }
66    }
67    w
68}
69
70/// Convert audio frames to media_time_ms given sample rate and media time origin.
71pub fn frames_to_media_ms(frame: u64, sample_rate: u32, origin_ms: u64) -> u64 {
72    if sample_rate == 0 {
73        return origin_ms;
74    }
75    origin_ms + (frame * 1000) / sample_rate as u64
76}
77
78/// Correlation predicate.
79pub fn correlation_predicate() -> u64 {
80    q_hash("https://ns.webizen.org/q42/proposesAvCooccurrence")
81}
82
83/// Shared media clock: maps vision frame index and audio sample to one timeline.
84#[derive(Debug, Clone, Copy)]
85pub struct SharedMediaClock {
86    pub media_hash: u64,
87    pub origin_ms: u64,
88    pub video_fps_num: u32,
89    pub video_fps_den: u32,
90    pub audio_sample_rate: u32,
91}
92
93impl SharedMediaClock {
94    /// Creates a new instance.
95    pub fn new(media_hash: u64, audio_sample_rate: u32, video_fps: f32) -> Self {
96        let fps = if video_fps > 0.0 { video_fps } else { 30.0 };
97        Self {
98            media_hash,
99            origin_ms: 0,
100            video_fps_num: (fps * 1000.0) as u32,
101            video_fps_den: 1000,
102            audio_sample_rate,
103        }
104    }
105
106    /// Video frame to ms.
107    pub fn video_frame_to_ms(&self, frame_index: u64) -> u64 {
108        if self.video_fps_num == 0 {
109            return self.origin_ms;
110        }
111        self.origin_ms
112            + (frame_index * 1000 * self.video_fps_den as u64) / self.video_fps_num as u64
113    }
114
115    /// Audio frame to ms.
116    pub fn audio_frame_to_ms(&self, sample_frame: u64) -> u64 {
117        frames_to_media_ms(sample_frame, self.audio_sample_rate, self.origin_ms)
118    }
119
120    /// Absolute drift between mapped times (ms).
121    pub fn drift_ms(&self, video_frame: u64, audio_sample: u64) -> i64 {
122        let v = self.video_frame_to_ms(video_frame) as i64;
123        let a = self.audio_frame_to_ms(audio_sample) as i64;
124        v - a
125    }
126}
127
128/// Joint query: events whose intervals overlap a window (caller-buffered).
129pub fn events_overlapping_window(
130    intervals: &[TimeIntervalMs],
131    window: TimeIntervalMs,
132    out_instances: &mut [u64],
133) -> usize {
134    let mut w = 0usize;
135    for iv in intervals {
136        if overlap(*iv, window).is_some() {
137            if w >= out_instances.len() {
138                return w;
139            }
140            out_instances[w] = iv.instance;
141            w += 1;
142        }
143    }
144    w
145}
146
147#[cfg(test)]
148mod tests {
149    use super::*;
150
151    #[test]
152    fn overlap_not_causal() {
153        let v = [TimeIntervalMs {
154            start_ms: 0,
155            end_ms: 1000,
156            instance: 1,
157        }];
158        let a = [TimeIntervalMs {
159            start_ms: 500,
160            end_ms: 1500,
161            instance: 2,
162        }];
163        let mut out = [AvCorrelationProposal {
164            media_hash: 0,
165            visual_instance: 0,
166            auditory_instance: 0,
167            overlap_start_ms: 0,
168            overlap_end_ms: 0,
169            confidence_u16: 0,
170            asserts_causality: true,
171        }; 4];
172        let n = propose_temporal_correlations(9, &v, &a, &mut out);
173        assert_eq!(n, 1);
174        assert!(!out[0].asserts_causality);
175        assert_eq!(out[0].overlap_start_ms, 500);
176        assert_eq!(out[0].overlap_end_ms, 1000);
177    }
178
179    #[test]
180    fn shared_clock_zero_drift_at_origin() {
181        let c = SharedMediaClock::new(1, 16000, 25.0);
182        assert_eq!(c.video_frame_to_ms(0), 0);
183        assert_eq!(c.audio_frame_to_ms(0), 0);
184        assert_eq!(c.drift_ms(0, 0), 0);
185    }
186
187    #[test]
188    fn joint_window_query() {
189        let intervals = [
190            TimeIntervalMs {
191                start_ms: 0,
192                end_ms: 100,
193                instance: 11,
194            },
195            TimeIntervalMs {
196                start_ms: 200,
197                end_ms: 300,
198                instance: 22,
199            },
200        ];
201        let win = TimeIntervalMs {
202            start_ms: 50,
203            end_ms: 250,
204            instance: 0,
205        };
206        let mut out = [0u64; 4];
207        let n = events_overlapping_window(&intervals, win, &mut out);
208        assert_eq!(n, 2);
209        assert!(out[..2].contains(&11) && out[..2].contains(&22));
210    }
211}