Skip to main content

qualia_core_db/query/
query_engine.rs

1use crate::NQuin;
2use std::fs::File;
3use std::io::{Read, Seek, SeekFrom};
4use std::path::Path;
5use std::time::Instant;
6
7/// Bounded sample of a `.q42` graph — unified v3 first, then flat packed NQuins.
8pub fn mmap_sample_quins(
9    file_path: &str,
10    max_quins: usize,
11) -> Result<Vec<NQuin>, Box<dyn std::error::Error>> {
12    #[cfg(not(target_arch = "wasm32"))]
13    {
14        use crate::q42_volume::{Q42Volume, Q42VolumeSet};
15
16        if max_quins == 0 {
17            return Ok(Vec::new());
18        }
19        let path = Path::new(file_path);
20        if crate::q42_volume::is_unified_volume(path)? {
21            let volume = Q42Volume::open(path)?;
22            if volume.volume_manifest()?.is_some() {
23                let set = Q42VolumeSet::open_root(path)?;
24                let mut out = Vec::new();
25                for segment in set.segments() {
26                    sample_volume_blocks(&segment, max_quins.saturating_sub(out.len()), &mut out)?;
27                    if out.len() >= max_quins {
28                        break;
29                    }
30                }
31                return Ok(out);
32            }
33            let mut out = Vec::new();
34            sample_volume_blocks(&volume, max_quins, &mut out)?;
35            return Ok(out);
36        }
37
38        if let Ok(quins) = crate::q42_reader::read_q42_quins(path) {
39            return Ok(stride_sample(&quins, max_quins));
40        }
41
42        use memmap2::MmapOptions;
43        const QUIN_SIZE: usize = std::mem::size_of::<NQuin>();
44        let file = File::open(path)?;
45        let mmap = unsafe { MmapOptions::new().map(&file)? };
46        let len = mmap.len();
47        if len % QUIN_SIZE != 0 {
48            return Err(format!(
49                "File size {} is not a multiple of NQuin ({} bytes)",
50                len, QUIN_SIZE
51            )
52            .into());
53        }
54        let count = len / QUIN_SIZE;
55        let quins: &[NQuin] =
56            unsafe { std::slice::from_raw_parts(mmap.as_ptr() as *const NQuin, count) };
57        Ok(stride_sample(quins, max_quins))
58    }
59    #[cfg(target_arch = "wasm32")]
60    {
61        let _ = file_path;
62        let _ = max_quins;
63        Err("mmap_sample_quins is not available on wasm32".into())
64    }
65}
66
67#[cfg(not(target_arch = "wasm32"))]
68fn sample_volume_blocks(
69    volume: &crate::q42_volume::Q42Volume,
70    max_quins: usize,
71    out: &mut Vec<NQuin>,
72) -> Result<(), Box<dyn std::error::Error>> {
73    use crate::q42_volume::{decode_superblock_quins, SUPERBLOCK_SIZE};
74    let blocks = volume.block_count() as usize;
75    if blocks == 0 || max_quins == 0 {
76        return Ok(());
77    }
78    let mut decoded = [0u8; SUPERBLOCK_SIZE];
79    let stride = (blocks / max_quins).max(1);
80    let mut index = 0usize;
81    while out.len() < max_quins && index < blocks {
82        volume.read_superblock_into(index, &mut decoded)?;
83        for quin in decode_superblock_quins(&decoded)? {
84            if quin.subject != 0 || quin.predicate != 0 {
85                out.push(quin);
86                if out.len() >= max_quins {
87                    break;
88                }
89            }
90        }
91        index = index.saturating_add(stride);
92    }
93    Ok(())
94}
95
96#[cfg(not(target_arch = "wasm32"))]
97fn collect_subject(
98    volume: &crate::q42_volume::Q42Volume,
99    subject_id: u64,
100    out: &mut Vec<NQuin>,
101) -> Result<(), Box<dyn std::error::Error>> {
102    use crate::q42_volume::{decode_superblock_quins, SUPERBLOCK_SIZE};
103    let mut decoded = [0u8; SUPERBLOCK_SIZE];
104    for index in 0..volume.block_count() as usize {
105        volume.read_superblock_into(index, &mut decoded)?;
106        for quin in decode_superblock_quins(&decoded)? {
107            if quin.subject == subject_id {
108                out.push(quin);
109            }
110        }
111    }
112    Ok(())
113}
114
115/// Bounded stride sample of an in-memory Quin slice. Filesystem `.q42` sampling
116/// stays on [`mmap_sample_quins`]; WASM query paths use this over packed frames.
117pub fn sample_quins(quins: &[NQuin], max_quins: usize) -> Vec<NQuin> {
118    stride_sample(quins, max_quins)
119}
120
121fn stride_sample(quins: &[NQuin], max_quins: usize) -> Vec<NQuin> {
122    if max_quins == 0 || quins.is_empty() {
123        return Vec::new();
124    }
125    let cap = max_quins.min(quins.len());
126    let stride = (quins.len() / cap).max(1);
127    let mut out = Vec::with_capacity(cap);
128    let mut idx = 0usize;
129    while out.len() < cap && idx < quins.len() {
130        let quin = quins[idx];
131        if quin.subject != 0 || quin.predicate != 0 {
132            out.push(quin);
133        }
134        idx += stride;
135    }
136    out
137}
138
139/// Query a `.q42` for Quins whose `subject` matches `subject_id`.
140/// Unified v3 volumes are decoded; flat packed files remain a fallback.
141pub fn mmap_query_subject(
142    file_path: &str,
143    subject_id: u64,
144) -> Result<Vec<NQuin>, Box<dyn std::error::Error>> {
145    #[cfg(not(target_arch = "wasm32"))]
146    {
147        use crate::q42_volume::{Q42Volume, Q42VolumeSet};
148
149        let path = Path::new(file_path);
150        if crate::q42_volume::is_unified_volume(path)? {
151            let volume = Q42Volume::open(path)?;
152            if volume.volume_manifest()?.is_some() {
153                let set = Q42VolumeSet::open_root(path)?;
154                let mut out = Vec::new();
155                for segment in set.segments() {
156                    collect_subject(&segment, subject_id, &mut out)?;
157                }
158                return Ok(out);
159            }
160            let mut out = Vec::new();
161            collect_subject(&volume, subject_id, &mut out)?;
162            return Ok(out);
163        }
164
165        if let Ok(quins) = crate::q42_reader::read_q42_quins(path) {
166            return Ok(quins
167                .into_iter()
168                .filter(|q| q.subject == subject_id)
169                .collect());
170        }
171
172        use memmap2::MmapOptions;
173        const QUIN_SIZE: usize = std::mem::size_of::<NQuin>();
174        let file = File::open(path)?;
175        let mmap = unsafe { MmapOptions::new().map(&file)? };
176        let len = mmap.len();
177        if len % QUIN_SIZE != 0 {
178            return Err(format!(
179                "File size {} is not a multiple of NQuin ({} bytes)",
180                len, QUIN_SIZE
181            )
182            .into());
183        }
184        let count = len / QUIN_SIZE;
185        let quins: &[NQuin] =
186            unsafe { std::slice::from_raw_parts(mmap.as_ptr() as *const NQuin, count) };
187        Ok(quins
188            .iter()
189            .filter(|q| q.subject == subject_id)
190            .copied()
191            .collect())
192    }
193    #[cfg(target_arch = "wasm32")]
194    {
195        let _ = file_path;
196        let _ = subject_id;
197        Err("mmap_query_subject is not available on wasm32".into())
198    }
199}
200
201/// Telemetry counters for `lazy_superblock_query`.
202pub struct TelemetryHook {
203    pub blocks_loaded: usize,
204    pub bytes_decompressed: usize,
205    /// Reserved for future WebRTC P2P streaming telemetry.
206    pub remote_blocks_streamed: usize,
207}
208
209/// Reads a SuperBlock file lazily: unified v3 volumes decode selected blocks;
210/// legacy 16-byte framed transport remains a fallback (O(1) seek on skip).
211pub fn lazy_superblock_query(
212    file_path: &str,
213    target_percent: u8,
214) -> Result<TelemetryHook, Box<dyn std::error::Error>> {
215    let start_time = Instant::now();
216    let path = Path::new(file_path);
217    let mut telemetry = TelemetryHook {
218        blocks_loaded: 0,
219        bytes_decompressed: 0,
220        remote_blocks_streamed: 0,
221    };
222
223    #[cfg(not(target_arch = "wasm32"))]
224    if crate::q42_volume::is_unified_volume(path).ok() == Some(true) {
225        use crate::q42_volume::{Q42Volume, SUPERBLOCK_SIZE};
226        let volume = Q42Volume::open(path)?;
227        let mut decoded = [0u8; SUPERBLOCK_SIZE];
228        for index in 0..volume.block_count() {
229            let is_relevant = (index % 100) < target_percent as u64;
230            if !is_relevant {
231                continue;
232            }
233            let n = volume.read_superblock_into(index as usize, &mut decoded)?;
234            telemetry.blocks_loaded += 1;
235            telemetry.bytes_decompressed += n;
236        }
237        let _duration = start_time.elapsed();
238        return Ok(telemetry);
239    }
240
241    let mut file = File::open(path)?;
242    let file_len = file.metadata()?.len();
243    let mut offset = 0u64;
244    let mut block_index = 0u64;
245
246    while offset < file_len {
247        let mut header = [0u8; 16];
248        if file.read_exact(&mut header).is_err() {
249            break;
250        }
251        offset += 16;
252
253        let _block_id = u64::from_le_bytes(header[0..8].try_into().unwrap());
254        let compressed_len = u32::from_le_bytes(header[8..12].try_into().unwrap()) as usize;
255        let uncompressed_len = u32::from_le_bytes(header[12..16].try_into().unwrap()) as usize;
256
257        let is_relevant = (block_index % 100) < target_percent as u64;
258
259        if is_relevant {
260            let mut compressed_buf = vec![0u8; compressed_len];
261            file.read_exact(&mut compressed_buf)?;
262            telemetry.blocks_loaded += 1;
263            let _uncompressed = lz4_flex::decompress_size_prepended(&compressed_buf)?;
264            telemetry.bytes_decompressed += uncompressed_len;
265        } else {
266            file.seek(SeekFrom::Current(compressed_len as i64))?;
267        }
268
269        offset += compressed_len as u64;
270        block_index += 1;
271    }
272
273    let _duration = start_time.elapsed();
274
275    Ok(telemetry)
276}
277
278/// Filter a slice of NQuin by context hash
279pub fn filter_by_context(quins: &[NQuin], context_hash: u64) -> Vec<NQuin> {
280    if context_hash == 0 {
281        return quins.to_vec();
282    }
283    quins
284        .iter()
285        .filter(|q| q.context == context_hash)
286        .copied()
287        .collect()
288}
289
290/// Filter a slice of NQuin by multiple context hashes
291pub fn filter_by_contexts(quins: &[NQuin], context_hashes: &[u64]) -> Vec<NQuin> {
292    if context_hashes.is_empty() {
293        return quins.to_vec();
294    }
295    let context_set: std::collections::HashSet<u64> = context_hashes.iter().copied().collect();
296    quins
297        .iter()
298        .filter(|q| context_set.contains(&q.context))
299        .copied()
300        .collect()
301}
302
303/// Count Quins per context hash
304pub fn count_by_context(quins: &[NQuin]) -> std::collections::HashMap<u64, usize> {
305    let mut counts = std::collections::HashMap::new();
306    for quin in quins {
307        *counts.entry(quin.context).or_insert(0) += 1;
308    }
309    counts
310}
311
312/// Get unique context hashes from a slice of NQuin
313pub fn unique_contexts(quins: &[NQuin]) -> Vec<u64> {
314    let mut contexts = std::collections::HashSet::new();
315    for quin in quins {
316        contexts.insert(quin.context);
317    }
318    contexts.into_iter().collect()
319}
320
321/// Filter Quins by context and subject
322pub fn filter_by_context_and_subject(
323    quins: &[NQuin],
324    context_hash: u64,
325    subject: u64,
326) -> Vec<NQuin> {
327    quins
328        .iter()
329        .filter(|q| (context_hash == 0 || q.context == context_hash) && q.subject == subject)
330        .copied()
331        .collect()
332}
333
334/// Filter Quins by context and predicate
335pub fn filter_by_context_and_predicate(
336    quins: &[NQuin],
337    context_hash: u64,
338    predicate: u64,
339) -> Vec<NQuin> {
340    quins
341        .iter()
342        .filter(|q| (context_hash == 0 || q.context == context_hash) && q.predicate == predicate)
343        .copied()
344        .collect()
345}
346
347/// Filter Quins by context and object
348pub fn filter_by_context_and_object(quins: &[NQuin], context_hash: u64, object: u64) -> Vec<NQuin> {
349    quins
350        .iter()
351        .filter(|q| (context_hash == 0 || q.context == context_hash) && q.object == object)
352        .copied()
353        .collect()
354}
355
356#[cfg(test)]
357mod volume_query_tests {
358    use super::*;
359    use crate::q42_volume::write_sorted_quins_volume;
360
361    fn quin(subject: u64, object: u64) -> NQuin {
362        NQuin {
363            subject,
364            predicate: 2,
365            object,
366            context: 0,
367            metadata: 0,
368            parity: NQuin::calculate_parity(subject, 2, object, 0, 0),
369        }
370    }
371
372    #[test]
373    fn sample_and_subject_query_read_unified_v3() {
374        let dir = tempfile::tempdir().unwrap();
375        let path = dir.path().join("graph.q42");
376        write_sorted_quins_volume(&path, &[quin(7, 1), quin(9, 2), quin(7, 3)]).unwrap();
377        let sampled = mmap_sample_quins(path.to_str().unwrap(), 8).unwrap();
378        assert_eq!(sampled.len(), 3);
379        let hits = mmap_query_subject(path.to_str().unwrap(), 7).unwrap();
380        assert_eq!(hits.len(), 2);
381        assert!(hits.iter().all(|q| q.subject == 7));
382        let telemetry = lazy_superblock_query(path.to_str().unwrap(), 100).unwrap();
383        assert_eq!(telemetry.blocks_loaded, 1);
384        assert!(telemetry.bytes_decompressed >= crate::q42_volume::SUPERBLOCK_SIZE);
385    }
386}
387
388#[cfg(test)]
389mod context_tests {
390    use super::*;
391
392    #[test]
393    fn test_filter_by_context() {
394        let quins = vec![
395            NQuin {
396                subject: 1,
397                predicate: 2,
398                object: 3,
399                context: 100,
400                metadata: 0,
401                parity: 0,
402            },
403            NQuin {
404                subject: 4,
405                predicate: 5,
406                object: 6,
407                context: 200,
408                metadata: 0,
409                parity: 0,
410            },
411            NQuin {
412                subject: 7,
413                predicate: 8,
414                object: 9,
415                context: 100,
416                metadata: 0,
417                parity: 0,
418            },
419        ];
420
421        let filtered = filter_by_context(&quins, 100);
422        assert_eq!(filtered.len(), 2);
423        assert_eq!(filtered[0].context, 100);
424        assert_eq!(filtered[1].context, 100);
425    }
426
427    #[test]
428    fn test_filter_by_context_wildcard() {
429        let quins = vec![
430            NQuin {
431                subject: 1,
432                predicate: 2,
433                object: 3,
434                context: 100,
435                metadata: 0,
436                parity: 0,
437            },
438            NQuin {
439                subject: 4,
440                predicate: 5,
441                object: 6,
442                context: 200,
443                metadata: 0,
444                parity: 0,
445            },
446        ];
447
448        let filtered = filter_by_context(&quins, 0);
449        assert_eq!(filtered.len(), 2);
450    }
451
452    #[test]
453    fn test_count_by_context() {
454        let quins = vec![
455            NQuin {
456                subject: 1,
457                predicate: 2,
458                object: 3,
459                context: 100,
460                metadata: 0,
461                parity: 0,
462            },
463            NQuin {
464                subject: 4,
465                predicate: 5,
466                object: 6,
467                context: 200,
468                metadata: 0,
469                parity: 0,
470            },
471            NQuin {
472                subject: 7,
473                predicate: 8,
474                object: 9,
475                context: 100,
476                metadata: 0,
477                parity: 0,
478            },
479        ];
480
481        let counts = count_by_context(&quins);
482        assert_eq!(counts.get(&100), Some(&2));
483        assert_eq!(counts.get(&200), Some(&1));
484    }
485
486    #[test]
487    fn test_filter_by_context_and_subject() {
488        let quins = vec![
489            NQuin {
490                subject: 1,
491                predicate: 2,
492                object: 3,
493                context: 100,
494                metadata: 0,
495                parity: 0,
496            },
497            NQuin {
498                subject: 1,
499                predicate: 5,
500                object: 6,
501                context: 200,
502                metadata: 0,
503                parity: 0,
504            },
505            NQuin {
506                subject: 7,
507                predicate: 8,
508                object: 9,
509                context: 100,
510                metadata: 0,
511                parity: 0,
512            },
513        ];
514
515        let filtered = filter_by_context_and_subject(&quins, 100, 1);
516        assert_eq!(filtered.len(), 1);
517        assert_eq!(filtered[0].subject, 1);
518        assert_eq!(filtered[0].context, 100);
519    }
520}