1use crate::NQuin;
2use std::fs::File;
3use std::io::{Read, Seek, SeekFrom};
4use std::path::Path;
5use std::time::Instant;
6
7pub fn mmap_sample_quins(
9 file_path: &str,
10 max_quins: usize,
11) -> Result<Vec<NQuin>, Box<dyn std::error::Error>> {
12 #[cfg(not(target_arch = "wasm32"))]
13 {
14 use crate::q42_volume::{Q42Volume, Q42VolumeSet};
15
16 if max_quins == 0 {
17 return Ok(Vec::new());
18 }
19 let path = Path::new(file_path);
20 if crate::q42_volume::is_unified_volume(path)? {
21 let volume = Q42Volume::open(path)?;
22 if volume.volume_manifest()?.is_some() {
23 let set = Q42VolumeSet::open_root(path)?;
24 let mut out = Vec::new();
25 for segment in set.segments() {
26 sample_volume_blocks(&segment, max_quins.saturating_sub(out.len()), &mut out)?;
27 if out.len() >= max_quins {
28 break;
29 }
30 }
31 return Ok(out);
32 }
33 let mut out = Vec::new();
34 sample_volume_blocks(&volume, max_quins, &mut out)?;
35 return Ok(out);
36 }
37
38 if let Ok(quins) = crate::q42_reader::read_q42_quins(path) {
39 return Ok(stride_sample(&quins, max_quins));
40 }
41
42 use memmap2::MmapOptions;
43 const QUIN_SIZE: usize = std::mem::size_of::<NQuin>();
44 let file = File::open(path)?;
45 let mmap = unsafe { MmapOptions::new().map(&file)? };
46 let len = mmap.len();
47 if len % QUIN_SIZE != 0 {
48 return Err(format!(
49 "File size {} is not a multiple of NQuin ({} bytes)",
50 len, QUIN_SIZE
51 )
52 .into());
53 }
54 let count = len / QUIN_SIZE;
55 let quins: &[NQuin] =
56 unsafe { std::slice::from_raw_parts(mmap.as_ptr() as *const NQuin, count) };
57 Ok(stride_sample(quins, max_quins))
58 }
59 #[cfg(target_arch = "wasm32")]
60 {
61 let _ = file_path;
62 let _ = max_quins;
63 Err("mmap_sample_quins is not available on wasm32".into())
64 }
65}
66
67#[cfg(not(target_arch = "wasm32"))]
68fn sample_volume_blocks(
69 volume: &crate::q42_volume::Q42Volume,
70 max_quins: usize,
71 out: &mut Vec<NQuin>,
72) -> Result<(), Box<dyn std::error::Error>> {
73 use crate::q42_volume::{decode_superblock_quins, SUPERBLOCK_SIZE};
74 let blocks = volume.block_count() as usize;
75 if blocks == 0 || max_quins == 0 {
76 return Ok(());
77 }
78 let mut decoded = [0u8; SUPERBLOCK_SIZE];
79 let stride = (blocks / max_quins).max(1);
80 let mut index = 0usize;
81 while out.len() < max_quins && index < blocks {
82 volume.read_superblock_into(index, &mut decoded)?;
83 for quin in decode_superblock_quins(&decoded)? {
84 if quin.subject != 0 || quin.predicate != 0 {
85 out.push(quin);
86 if out.len() >= max_quins {
87 break;
88 }
89 }
90 }
91 index = index.saturating_add(stride);
92 }
93 Ok(())
94}
95
96#[cfg(not(target_arch = "wasm32"))]
97fn collect_subject(
98 volume: &crate::q42_volume::Q42Volume,
99 subject_id: u64,
100 out: &mut Vec<NQuin>,
101) -> Result<(), Box<dyn std::error::Error>> {
102 use crate::q42_volume::{decode_superblock_quins, SUPERBLOCK_SIZE};
103 let mut decoded = [0u8; SUPERBLOCK_SIZE];
104 for index in 0..volume.block_count() as usize {
105 volume.read_superblock_into(index, &mut decoded)?;
106 for quin in decode_superblock_quins(&decoded)? {
107 if quin.subject == subject_id {
108 out.push(quin);
109 }
110 }
111 }
112 Ok(())
113}
114
115pub fn sample_quins(quins: &[NQuin], max_quins: usize) -> Vec<NQuin> {
118 stride_sample(quins, max_quins)
119}
120
121fn stride_sample(quins: &[NQuin], max_quins: usize) -> Vec<NQuin> {
122 if max_quins == 0 || quins.is_empty() {
123 return Vec::new();
124 }
125 let cap = max_quins.min(quins.len());
126 let stride = (quins.len() / cap).max(1);
127 let mut out = Vec::with_capacity(cap);
128 let mut idx = 0usize;
129 while out.len() < cap && idx < quins.len() {
130 let quin = quins[idx];
131 if quin.subject != 0 || quin.predicate != 0 {
132 out.push(quin);
133 }
134 idx += stride;
135 }
136 out
137}
138
139pub fn mmap_query_subject(
142 file_path: &str,
143 subject_id: u64,
144) -> Result<Vec<NQuin>, Box<dyn std::error::Error>> {
145 #[cfg(not(target_arch = "wasm32"))]
146 {
147 use crate::q42_volume::{Q42Volume, Q42VolumeSet};
148
149 let path = Path::new(file_path);
150 if crate::q42_volume::is_unified_volume(path)? {
151 let volume = Q42Volume::open(path)?;
152 if volume.volume_manifest()?.is_some() {
153 let set = Q42VolumeSet::open_root(path)?;
154 let mut out = Vec::new();
155 for segment in set.segments() {
156 collect_subject(&segment, subject_id, &mut out)?;
157 }
158 return Ok(out);
159 }
160 let mut out = Vec::new();
161 collect_subject(&volume, subject_id, &mut out)?;
162 return Ok(out);
163 }
164
165 if let Ok(quins) = crate::q42_reader::read_q42_quins(path) {
166 return Ok(quins
167 .into_iter()
168 .filter(|q| q.subject == subject_id)
169 .collect());
170 }
171
172 use memmap2::MmapOptions;
173 const QUIN_SIZE: usize = std::mem::size_of::<NQuin>();
174 let file = File::open(path)?;
175 let mmap = unsafe { MmapOptions::new().map(&file)? };
176 let len = mmap.len();
177 if len % QUIN_SIZE != 0 {
178 return Err(format!(
179 "File size {} is not a multiple of NQuin ({} bytes)",
180 len, QUIN_SIZE
181 )
182 .into());
183 }
184 let count = len / QUIN_SIZE;
185 let quins: &[NQuin] =
186 unsafe { std::slice::from_raw_parts(mmap.as_ptr() as *const NQuin, count) };
187 Ok(quins
188 .iter()
189 .filter(|q| q.subject == subject_id)
190 .copied()
191 .collect())
192 }
193 #[cfg(target_arch = "wasm32")]
194 {
195 let _ = file_path;
196 let _ = subject_id;
197 Err("mmap_query_subject is not available on wasm32".into())
198 }
199}
200
201pub struct TelemetryHook {
203 pub blocks_loaded: usize,
204 pub bytes_decompressed: usize,
205 pub remote_blocks_streamed: usize,
207}
208
209pub fn lazy_superblock_query(
212 file_path: &str,
213 target_percent: u8,
214) -> Result<TelemetryHook, Box<dyn std::error::Error>> {
215 let start_time = Instant::now();
216 let path = Path::new(file_path);
217 let mut telemetry = TelemetryHook {
218 blocks_loaded: 0,
219 bytes_decompressed: 0,
220 remote_blocks_streamed: 0,
221 };
222
223 #[cfg(not(target_arch = "wasm32"))]
224 if crate::q42_volume::is_unified_volume(path).ok() == Some(true) {
225 use crate::q42_volume::{Q42Volume, SUPERBLOCK_SIZE};
226 let volume = Q42Volume::open(path)?;
227 let mut decoded = [0u8; SUPERBLOCK_SIZE];
228 for index in 0..volume.block_count() {
229 let is_relevant = (index % 100) < target_percent as u64;
230 if !is_relevant {
231 continue;
232 }
233 let n = volume.read_superblock_into(index as usize, &mut decoded)?;
234 telemetry.blocks_loaded += 1;
235 telemetry.bytes_decompressed += n;
236 }
237 let _duration = start_time.elapsed();
238 return Ok(telemetry);
239 }
240
241 let mut file = File::open(path)?;
242 let file_len = file.metadata()?.len();
243 let mut offset = 0u64;
244 let mut block_index = 0u64;
245
246 while offset < file_len {
247 let mut header = [0u8; 16];
248 if file.read_exact(&mut header).is_err() {
249 break;
250 }
251 offset += 16;
252
253 let _block_id = u64::from_le_bytes(header[0..8].try_into().unwrap());
254 let compressed_len = u32::from_le_bytes(header[8..12].try_into().unwrap()) as usize;
255 let uncompressed_len = u32::from_le_bytes(header[12..16].try_into().unwrap()) as usize;
256
257 let is_relevant = (block_index % 100) < target_percent as u64;
258
259 if is_relevant {
260 let mut compressed_buf = vec![0u8; compressed_len];
261 file.read_exact(&mut compressed_buf)?;
262 telemetry.blocks_loaded += 1;
263 let _uncompressed = lz4_flex::decompress_size_prepended(&compressed_buf)?;
264 telemetry.bytes_decompressed += uncompressed_len;
265 } else {
266 file.seek(SeekFrom::Current(compressed_len as i64))?;
267 }
268
269 offset += compressed_len as u64;
270 block_index += 1;
271 }
272
273 let _duration = start_time.elapsed();
274
275 Ok(telemetry)
276}
277
278pub fn filter_by_context(quins: &[NQuin], context_hash: u64) -> Vec<NQuin> {
280 if context_hash == 0 {
281 return quins.to_vec();
282 }
283 quins
284 .iter()
285 .filter(|q| q.context == context_hash)
286 .copied()
287 .collect()
288}
289
290pub fn filter_by_contexts(quins: &[NQuin], context_hashes: &[u64]) -> Vec<NQuin> {
292 if context_hashes.is_empty() {
293 return quins.to_vec();
294 }
295 let context_set: std::collections::HashSet<u64> = context_hashes.iter().copied().collect();
296 quins
297 .iter()
298 .filter(|q| context_set.contains(&q.context))
299 .copied()
300 .collect()
301}
302
303pub fn count_by_context(quins: &[NQuin]) -> std::collections::HashMap<u64, usize> {
305 let mut counts = std::collections::HashMap::new();
306 for quin in quins {
307 *counts.entry(quin.context).or_insert(0) += 1;
308 }
309 counts
310}
311
312pub fn unique_contexts(quins: &[NQuin]) -> Vec<u64> {
314 let mut contexts = std::collections::HashSet::new();
315 for quin in quins {
316 contexts.insert(quin.context);
317 }
318 contexts.into_iter().collect()
319}
320
321pub fn filter_by_context_and_subject(
323 quins: &[NQuin],
324 context_hash: u64,
325 subject: u64,
326) -> Vec<NQuin> {
327 quins
328 .iter()
329 .filter(|q| (context_hash == 0 || q.context == context_hash) && q.subject == subject)
330 .copied()
331 .collect()
332}
333
334pub fn filter_by_context_and_predicate(
336 quins: &[NQuin],
337 context_hash: u64,
338 predicate: u64,
339) -> Vec<NQuin> {
340 quins
341 .iter()
342 .filter(|q| (context_hash == 0 || q.context == context_hash) && q.predicate == predicate)
343 .copied()
344 .collect()
345}
346
347pub fn filter_by_context_and_object(quins: &[NQuin], context_hash: u64, object: u64) -> Vec<NQuin> {
349 quins
350 .iter()
351 .filter(|q| (context_hash == 0 || q.context == context_hash) && q.object == object)
352 .copied()
353 .collect()
354}
355
356#[cfg(test)]
357mod volume_query_tests {
358 use super::*;
359 use crate::q42_volume::write_sorted_quins_volume;
360
361 fn quin(subject: u64, object: u64) -> NQuin {
362 NQuin {
363 subject,
364 predicate: 2,
365 object,
366 context: 0,
367 metadata: 0,
368 parity: NQuin::calculate_parity(subject, 2, object, 0, 0),
369 }
370 }
371
372 #[test]
373 fn sample_and_subject_query_read_unified_v3() {
374 let dir = tempfile::tempdir().unwrap();
375 let path = dir.path().join("graph.q42");
376 write_sorted_quins_volume(&path, &[quin(7, 1), quin(9, 2), quin(7, 3)]).unwrap();
377 let sampled = mmap_sample_quins(path.to_str().unwrap(), 8).unwrap();
378 assert_eq!(sampled.len(), 3);
379 let hits = mmap_query_subject(path.to_str().unwrap(), 7).unwrap();
380 assert_eq!(hits.len(), 2);
381 assert!(hits.iter().all(|q| q.subject == 7));
382 let telemetry = lazy_superblock_query(path.to_str().unwrap(), 100).unwrap();
383 assert_eq!(telemetry.blocks_loaded, 1);
384 assert!(telemetry.bytes_decompressed >= crate::q42_volume::SUPERBLOCK_SIZE);
385 }
386}
387
388#[cfg(test)]
389mod context_tests {
390 use super::*;
391
392 #[test]
393 fn test_filter_by_context() {
394 let quins = vec![
395 NQuin {
396 subject: 1,
397 predicate: 2,
398 object: 3,
399 context: 100,
400 metadata: 0,
401 parity: 0,
402 },
403 NQuin {
404 subject: 4,
405 predicate: 5,
406 object: 6,
407 context: 200,
408 metadata: 0,
409 parity: 0,
410 },
411 NQuin {
412 subject: 7,
413 predicate: 8,
414 object: 9,
415 context: 100,
416 metadata: 0,
417 parity: 0,
418 },
419 ];
420
421 let filtered = filter_by_context(&quins, 100);
422 assert_eq!(filtered.len(), 2);
423 assert_eq!(filtered[0].context, 100);
424 assert_eq!(filtered[1].context, 100);
425 }
426
427 #[test]
428 fn test_filter_by_context_wildcard() {
429 let quins = vec![
430 NQuin {
431 subject: 1,
432 predicate: 2,
433 object: 3,
434 context: 100,
435 metadata: 0,
436 parity: 0,
437 },
438 NQuin {
439 subject: 4,
440 predicate: 5,
441 object: 6,
442 context: 200,
443 metadata: 0,
444 parity: 0,
445 },
446 ];
447
448 let filtered = filter_by_context(&quins, 0);
449 assert_eq!(filtered.len(), 2);
450 }
451
452 #[test]
453 fn test_count_by_context() {
454 let quins = vec![
455 NQuin {
456 subject: 1,
457 predicate: 2,
458 object: 3,
459 context: 100,
460 metadata: 0,
461 parity: 0,
462 },
463 NQuin {
464 subject: 4,
465 predicate: 5,
466 object: 6,
467 context: 200,
468 metadata: 0,
469 parity: 0,
470 },
471 NQuin {
472 subject: 7,
473 predicate: 8,
474 object: 9,
475 context: 100,
476 metadata: 0,
477 parity: 0,
478 },
479 ];
480
481 let counts = count_by_context(&quins);
482 assert_eq!(counts.get(&100), Some(&2));
483 assert_eq!(counts.get(&200), Some(&1));
484 }
485
486 #[test]
487 fn test_filter_by_context_and_subject() {
488 let quins = vec![
489 NQuin {
490 subject: 1,
491 predicate: 2,
492 object: 3,
493 context: 100,
494 metadata: 0,
495 parity: 0,
496 },
497 NQuin {
498 subject: 1,
499 predicate: 5,
500 object: 6,
501 context: 200,
502 metadata: 0,
503 parity: 0,
504 },
505 NQuin {
506 subject: 7,
507 predicate: 8,
508 object: 9,
509 context: 100,
510 metadata: 0,
511 parity: 0,
512 },
513 ];
514
515 let filtered = filter_by_context_and_subject(&quins, 100, 1);
516 assert_eq!(filtered.len(), 1);
517 assert_eq!(filtered[0].subject, 1);
518 assert_eq!(filtered[0].context, 100);
519 }
520}