Skip to main content

qualia_core_db/bundle/
streaming_hydration.rs

1//! Byte-Backed WASM `.q42` Host Seam, Progressive Range-Streamed `.hmc` Delivery,
2//! Webizen-Governed Worker Pool & Hardware Texture Transcoding Pipeline.
3//!
4//! Unites Epic 7 (BC7/ETC2/ASTC/KTX2 Hardware Pipeline) and Epic 10 (Byte-Backed WASM
5//! `.q42` Host Seam & Governed Worker Pool) from `qualia-core-db`:
6//! - [`HmcPackStreamer`]: Orchestrates progressive range streaming for `.hmc` bundles
7//!   across 3 priority tiers:
8//!   - Tier 0: `.q42` semantic graph + `.10d` `PROC1`/`EONT1`/`IMPO`/`SURF` sections (<50 KB)
9//!   - Tier 1: Mid-LOD (`LOD2` / `LOD1`) mesh sections
10//!   - Tier 2: High-poly `LOD0` B-Rep geometry and `KTX2` mip0 textures
11//! - Zero-heap `NQuin` and `Q42LEX` dictionary resolution via [`WasmSliceQ42Reader`] and
12//!   `vibe::catalog::local::LocalHost::with_q42_pack`.
13//! - Capability-gated background hydration worker pool ([`GovernedHydrationPool`]) enforcing
14//!   fuel and byte budgets using [`HydrationCapabilityToken`] so main-thread frame times never stall.
15//! - Direct GPU hardware-compressed texture selection ([`select_hardware_compressed_format`])
16//!   and zero-heap CPU block fallback transcoding ([`decode_bc7_block`], [`decode_etc2_eac_rgba_block`],
17//!   [`decode_astc_4x4_block`]).
18
19use crate::bundle::streaming_loader::{
20    verify_arrived_range, HmcHeaderProbe, HmcProgressiveRangePlanner, HmcRangeRequest,
21    StreamPriorityTier,
22};
23use crate::bundle::BundleError;
24use crate::governance::worker_pool::{
25    GovernedHydrationPool, GovernancePoolError, HydrationCapabilityToken, HydrationReceipt,
26    HydrationTask, HydrationTaskKind, TranscodeCodec, CAP_ALL_HYDRATION,
27};
28use crate::q42::q42_volume::WasmSliceQ42Reader;
29use crate::render::texture_ktx2::{
30    select_hardware_compressed_format, HardwareCompressedUploadFormat, Ktx2Document, Ktx2Error,
31};
32use crate::render::texture_transcode::{
33    decode_astc_4x4_block, decode_bc7_block, decode_etc2_eac_rgba_block,
34};
35use vibe::Host;
36
37/// Maximum capability tokens issued for hydration worker threads.
38pub const DEFAULT_HYDRATION_QUEUE_CAPACITY: usize = 128;
39
40/// Extension trait equipping [`vibe::LocalHost`] with direct `.q42` pack admission
41/// and local in-memory SPARQL execution.
42pub trait LocalHostQ42Ext {
43    /// Creates a new `LocalHost` initialized with the admitted `.q42` slice.
44    fn with_q42_pack(q42_bytes: &[u8]) -> Self;
45    /// Admits a slice of 48-byte `NQuin` records into the local SPARQL store.
46    fn admit_q42_slice(&mut self, q42_bytes: &[u8]) -> Result<usize, String>;
47    /// Evaluates a local zero-heap SPARQL pattern query.
48    fn query_sparql_local(
49        &mut self,
50        subject: Option<u64>,
51        predicate: Option<u64>,
52        object: Option<u64>,
53        context: Option<u64>,
54    ) -> Result<Vec<vibe::Value>, vibe::Diagnostic>;
55}
56
57impl LocalHostQ42Ext for vibe::LocalHost {
58    fn with_q42_pack(q42_bytes: &[u8]) -> Self {
59        let mut host = Self::default();
60        let _ = host.admit_q42_slice(q42_bytes);
61        host
62    }
63
64    fn admit_q42_slice(&mut self, q42_bytes: &[u8]) -> Result<usize, String> {
65        let mut map = std::collections::BTreeMap::new();
66        let byte_list = q42_bytes.iter().map(|&b| vibe::Value::U64(b as u64)).collect();
67        map.insert("bytes".into(), vibe::Value::List(byte_list));
68        map.insert("key".into(), vibe::Value::String("pack.q42".into()));
69        map.insert("load".into(), vibe::Value::Bool(true));
70        let args = vibe::Value::Record(map);
71        let span = vibe::Span::new(0, 0);
72        match self.capability_invoke("GraphDatabase.volume_open", &args, span) {
73            Ok(vibe::Value::Record(res)) => {
74                let count = match res.get("quin_count") {
75                    Some(vibe::Value::U64(c)) => *c as usize,
76                    _ => 0,
77                };
78                Ok(count)
79            }
80            Ok(_) => Ok(0),
81            Err(e) => Err(format!("{:?}", e)),
82        }
83    }
84
85    fn query_sparql_local(
86        &mut self,
87        subject: Option<u64>,
88        predicate: Option<u64>,
89        object: Option<u64>,
90        context: Option<u64>,
91    ) -> Result<Vec<vibe::Value>, vibe::Diagnostic> {
92        let mut map = std::collections::BTreeMap::new();
93        if let Some(s) = subject {
94            map.insert("subject".into(), vibe::Value::U64(s));
95        }
96        if let Some(p) = predicate {
97            map.insert("predicate".into(), vibe::Value::U64(p));
98        }
99        if let Some(o) = object {
100            map.insert("object".into(), vibe::Value::U64(o));
101        }
102        if let Some(c) = context {
103            map.insert("context".into(), vibe::Value::U64(c));
104        }
105        let args = vibe::Value::Record(map);
106        let span = vibe::Span::new(0, 0);
107        let res = self.capability_invoke("GraphDatabase.sparql", &args, span)?;
108        match res {
109            vibe::Value::List(rows) => Ok(rows),
110            _ => Ok(Vec::new()),
111        }
112    }
113}
114
115/// Progressive `.hmc` package streamer, governed hydration worker pool, and hardware texture bridge.
116pub struct HmcPackStreamer {
117    /// In-process Vibe local host for executing local SPARQL and graph queries.
118    pub local_host: vibe::LocalHost,
119    /// Capability-gated background hydration worker pool.
120    pub pool: GovernedHydrationPool,
121    /// Primary capability token governing background task dispatch.
122    pub token: HydrationCapabilityToken,
123    /// Scheduled range requests partitioned across priority tiers.
124    pub planned_requests: Vec<HmcRangeRequest>,
125    /// Tier 0 requests successfully verified and admitted.
126    pub tier0_completed: usize,
127    /// Tier 1 requests successfully verified and admitted.
128    pub tier1_completed: usize,
129    /// Tier 2 requests successfully verified and admitted.
130    pub tier2_completed: usize,
131    /// Total payload bytes admitted and verified.
132    pub total_bytes_admitted: u64,
133    /// Monotonically increasing hydration task identifier.
134    next_task_id: u64,
135}
136
137/// Compatibility alias for [`HmcPackStreamer`].
138pub type EcopunkPackStreamer = HmcPackStreamer;
139
140impl HmcPackStreamer {
141    /// Creates a new streamer with governed worker pool, capability token, and Vibe local host.
142    pub fn new(holder_did_hash: u64, fuel_budget: u64, byte_budget: u64) -> Self {
143        let token = HydrationCapabilityToken::new_sealed(
144            holder_did_hash,
145            CAP_ALL_HYDRATION,
146            StreamPriorityTier::Tier2HighPolyAndMip0,
147            fuel_budget,
148            byte_budget,
149        );
150        Self {
151            local_host: vibe::LocalHost::default(),
152            pool: GovernedHydrationPool::new(DEFAULT_HYDRATION_QUEUE_CAPACITY),
153            token,
154            planned_requests: Vec::new(),
155            tier0_completed: 0,
156            tier1_completed: 0,
157            tier2_completed: 0,
158            total_bytes_admitted: 0,
159            next_task_id: 1,
160        }
161    }
162
163    /// Plans progressive `.hmc` stream delivery from the 64-byte header probe and CBOR index footer.
164    ///
165    /// Orders all entries strictly by `(tier, offset)` so Tier 0 arrives first.
166    pub fn plan_hmc_stream(
167        &mut self,
168        header_probe: HmcHeaderProbe,
169        index_footer_bytes: &[u8],
170    ) -> Result<&[HmcRangeRequest], BundleError> {
171        let planner = HmcProgressiveRangePlanner::from_header_and_index(header_probe, index_footer_bytes)?;
172        self.planned_requests = planner.plan_entry_ranges();
173        Ok(&self.planned_requests)
174    }
175
176    /// Verifies an arrived byte range against its planned request and admits it into the active session.
177    pub fn admit_and_verify_range(
178        &mut self,
179        req_idx: usize,
180        arrived_bytes: &[u8],
181    ) -> Result<(), GovernancePoolError> {
182        if req_idx >= self.planned_requests.len() {
183            return Err(GovernancePoolError::IntegrityMismatch {
184                task_id: req_idx as u64,
185            });
186        }
187        let req = &self.planned_requests[req_idx];
188        verify_arrived_range(req, arrived_bytes).map_err(|_| {
189            GovernancePoolError::IntegrityMismatch {
190                task_id: req_idx as u64,
191            }
192        })?;
193
194        // If this is a Tier 0 .q42 semantic slice, admit it into the Vibe LocalHost
195        if req.entry_kind.eq_ignore_ascii_case("q42") || req.entry_key.ends_with(".q42") {
196            let _ = self.local_host.admit_q42_slice(arrived_bytes);
197        }
198
199        match req.tier {
200            StreamPriorityTier::Tier0SemanticAndProcedural => self.tier0_completed += 1,
201            StreamPriorityTier::Tier1MidLodMeshes => self.tier1_completed += 1,
202            StreamPriorityTier::Tier2HighPolyAndMip0 => self.tier2_completed += 1,
203        }
204        self.total_bytes_admitted += arrived_bytes.len() as u64;
205        Ok(())
206    }
207
208    /// Opens a zero-heap [`WasmSliceQ42Reader`] directly over borrowed in-memory bytes.
209    #[inline]
210    pub fn open_wasm_slice_reader<'a>(
211        &self,
212        q42_bytes: &'a [u8],
213    ) -> std::io::Result<WasmSliceQ42Reader<'a>> {
214        WasmSliceQ42Reader::open(q42_bytes)
215    }
216
217    /// Submits a background `.10d` section CRC-32C verification task to the governed pool.
218    pub fn submit_section_crc_check(
219        &mut self,
220        tier: StreamPriorityTier,
221        payload: Vec<u8>,
222        expected_crc32c: u32,
223    ) -> Result<u64, GovernancePoolError> {
224        let task_id = self.next_task_id;
225        self.next_task_id += 1;
226        self.pool.submit(
227            &mut self.token,
228            HydrationTask {
229                task_id,
230                tier,
231                kind: HydrationTaskKind::VerifySectionCrc32c {
232                    payload,
233                    expected_crc32c,
234                },
235            },
236        )?;
237        Ok(task_id)
238    }
239
240    /// Submits a background `.hmc` entry SHA-256 verification task to the governed pool.
241    pub fn submit_entry_sha256_check(
242        &mut self,
243        tier: StreamPriorityTier,
244        payload: Vec<u8>,
245        expected_sha256: [u8; 32],
246    ) -> Result<u64, GovernancePoolError> {
247        let task_id = self.next_task_id;
248        self.next_task_id += 1;
249        self.pool.submit(
250            &mut self.token,
251            HydrationTask {
252                task_id,
253                tier,
254                kind: HydrationTaskKind::VerifyEntrySha256 {
255                    payload,
256                    expected_sha256,
257                },
258            },
259        )?;
260        Ok(task_id)
261    }
262
263    /// Submits a background 16-byte texture block transcode task (BC7 / ETC2 / ASTC) to the governed pool.
264    pub fn submit_texture_transcode(
265        &mut self,
266        tier: StreamPriorityTier,
267        codec: TranscodeCodec,
268        block: [u8; 16],
269    ) -> Result<u64, GovernancePoolError> {
270        let task_id = self.next_task_id;
271        self.next_task_id += 1;
272        self.pool.submit(
273            &mut self.token,
274            HydrationTask {
275                task_id,
276                tier,
277                kind: HydrationTaskKind::TranscodeTextureBlock { codec, block },
278            },
279        )?;
280        Ok(task_id)
281    }
282
283    /// Executes a bounded slice of hydration work (draining up to `max_tasks_this_frame`)
284    /// strictly prioritizing Tier 0 over Tier 1 over Tier 2, ensuring frame times never stall.
285    #[inline]
286    pub fn step_hydration_frame(
287        &mut self,
288        max_tasks_this_frame: usize,
289    ) -> Result<Vec<HydrationReceipt>, GovernancePoolError> {
290        self.pool.drain_bounded(max_tasks_this_frame)
291    }
292
293    /// Inspects a borrowed KTX2 container and selects a direct hardware-compressed format
294    /// (avoiding CPU RGBA8 inflation) if supported by the active GPU profile.
295    #[inline]
296    pub fn inspect_and_select_gpu_texture_format(
297        &self,
298        ktx2_bytes: &[u8],
299        gpu_bc: bool,
300        gpu_astc: bool,
301        gpu_etc2: bool,
302    ) -> Result<Option<HardwareCompressedUploadFormat>, Ktx2Error> {
303        let doc = Ktx2Document::parse(ktx2_bytes)?;
304        Ok(select_hardware_compressed_format(&doc, gpu_bc, gpu_astc, gpu_etc2))
305    }
306
307    /// Executes a zero-heap single 16-byte block transcode into a 64-byte 4x4 RGBA8 tile.
308    #[inline]
309    pub fn transcode_block_zero_heap(
310        codec: TranscodeCodec,
311        block: &[u8; 16],
312        out_rgba: &mut [u8; 64],
313    ) {
314        let pixels: &mut [[u8; 4]; 16] = bytemuck::cast_mut(out_rgba);
315        match codec {
316            TranscodeCodec::Bc7 => decode_bc7_block(block, pixels),
317            TranscodeCodec::Etc2EacRgba8 => decode_etc2_eac_rgba_block(block, pixels),
318            TranscodeCodec::Astc4x4 => decode_astc_4x4_block(block, pixels),
319        }
320    }
321}
322
323#[cfg(test)]
324mod tests {
325    use super::*;
326    use crate::bundle::BundleWriter;
327    use crate::container_10d::crc32c::crc32c;
328    use crate::NQuin;
329
330    #[test]
331    fn progressive_hmc_streaming_schedules_priority_tiers() {
332        let mut writer = BundleWriter::new();
333        // Add entries belonging to Tier 0, Tier 1, and Tier 2
334        writer
335            .add_file("ecopunk_terrain_lod0.10d", "10d", vec![0xAA; 1024], None)
336            .unwrap();
337        writer
338            .add_file("ecopunk_terrain_lod2.10d", "10d", vec![0xBB; 256], None)
339            .unwrap();
340        writer
341            .add_file("ecopunk_ecology.q42", "q42", vec![0xCC; 96], None)
342            .unwrap();
343        writer
344            .add_file("snow_gum_bark.ktx2", "ktx2", vec![0xDD; 512], None)
345            .unwrap();
346        let bundle_bytes = writer.build().unwrap();
347
348        let probe = HmcHeaderProbe::parse_header_range(&bundle_bytes[..64]).unwrap();
349        assert_eq!(probe.entry_count, 4);
350
351        let (idx_off, idx_len) = probe.index_footer_range();
352        let footer_slice = &bundle_bytes[idx_off as usize..(idx_off + idx_len) as usize];
353
354        let mut streamer = HmcPackStreamer::new(0x10D_E64_A1, 50_000, 100_000);
355        let schedule = streamer.plan_hmc_stream(probe, footer_slice).unwrap();
356        assert_eq!(schedule.len(), 4);
357
358        // Tier 0 (.q42) must be scheduled first
359        assert_eq!(schedule[0].tier, StreamPriorityTier::Tier0SemanticAndProcedural);
360        assert_eq!(schedule[0].entry_key, "ecopunk_ecology.q42");
361
362        // Tier 1 (lod2) must be scheduled second
363        assert_eq!(schedule[1].tier, StreamPriorityTier::Tier1MidLodMeshes);
364        assert_eq!(schedule[1].entry_key, "ecopunk_terrain_lod2.10d");
365
366        // Tier 2 (lod0, ktx2) must be scheduled last
367        assert_eq!(schedule[2].tier, StreamPriorityTier::Tier2HighPolyAndMip0);
368        assert_eq!(schedule[3].tier, StreamPriorityTier::Tier2HighPolyAndMip0);
369
370        // Verify arrived slices pass verification
371        for i in 0..schedule.len() {
372            let start = streamer.planned_requests[i].offset as usize;
373            let end = start + streamer.planned_requests[i].length as usize;
374            streamer
375                .admit_and_verify_range(i, &bundle_bytes[start..end])
376                .unwrap();
377        }
378        assert_eq!(streamer.tier0_completed, 1);
379        assert_eq!(streamer.tier1_completed, 1);
380        assert_eq!(streamer.tier2_completed, 2);
381    }
382
383    #[test]
384    fn governed_worker_pool_hydrates_within_budget() {
385        let mut streamer = EcopunkPackStreamer::new(0x8888_7777, 500, 2048);
386        assert!(streamer.token.verify_seal());
387
388        // Submit Tier 2 texture transcode task
389        let mut block = [0u8; 16];
390        block[0] = 1 << 6; // BC7 Mode 6
391        let t1 = streamer
392            .submit_texture_transcode(StreamPriorityTier::Tier2HighPolyAndMip0, TranscodeCodec::Bc7, block)
393            .unwrap();
394
395        // Submit Tier 0 CRC-32C verification task
396        let payload = b"ecopunk procedural botanical section".to_vec();
397        let expected_crc = crc32c(&payload);
398        let t0 = streamer
399            .submit_section_crc_check(
400                StreamPriorityTier::Tier0SemanticAndProcedural,
401                payload.clone(),
402                expected_crc,
403            )
404            .unwrap();
405
406        // Step pool bounded by 2 tasks; Tier 0 MUST complete first!
407        let receipts = streamer.step_hydration_frame(2).unwrap();
408        assert_eq!(receipts.len(), 2);
409        assert_eq!(receipts[0].task_id, t0);
410        assert_eq!(receipts[0].tier, StreamPriorityTier::Tier0SemanticAndProcedural);
411        assert!(receipts[0].verified);
412        assert_eq!(receipts[1].task_id, t1);
413        assert_eq!(receipts[1].tier, StreamPriorityTier::Tier2HighPolyAndMip0);
414        assert!(receipts[1].transcoded_rgba8_4x4.is_some());
415
416        // Budgets were correctly deducted and seal preserved
417        assert!(streamer.token.fuel_remaining < 500);
418        assert!(streamer.token.byte_budget_remaining < 2048);
419        assert!(streamer.token.verify_seal());
420    }
421
422    #[test]
423    fn hardware_compressed_format_selection_and_transcode() {
424        // Test BC7 Modes 0..=7 decoding
425        for mode in 0..8usize {
426            let mut block = [0xAAu8; 16];
427            block[0] = 1u8 << mode;
428            let mut out = [0u8; 64];
429            HmcPackStreamer::transcode_block_zero_heap(TranscodeCodec::Bc7, &block, &mut out);
430            assert!(out.iter().any(|&b| b > 0));
431        }
432
433        // Test ETC2 EAC RGBA8 block decoding
434        let mut etc2_block = [0u8; 16];
435        etc2_block[0] = 220; // base alpha
436        etc2_block[8] = 0xAA;
437        let mut etc2_out = [0u8; 64];
438        HmcPackStreamer::transcode_block_zero_heap(TranscodeCodec::Etc2EacRgba8, &etc2_block, &mut etc2_out);
439        assert!(etc2_out.iter().any(|&b| b > 0));
440
441        // Test ASTC 4x4 void-extent block decoding
442        let mut astc_block = [0u8; 16];
443        astc_block[0] = 0xFC;
444        astc_block[1] = 0xFD; // Void-extent flag
445        astc_block[8..10].copy_from_slice(&0x8000u16.to_le_bytes()); // R
446        astc_block[10..12].copy_from_slice(&0x4000u16.to_le_bytes()); // G
447        astc_block[12..14].copy_from_slice(&0xFF00u16.to_le_bytes()); // B
448        astc_block[14..16].copy_from_slice(&0xFF00u16.to_le_bytes()); // A
449        let mut astc_out = [0u8; 64];
450        HmcPackStreamer::transcode_block_zero_heap(TranscodeCodec::Astc4x4, &astc_block, &mut astc_out);
451        assert_eq!(astc_out[0], 0x80);
452        assert_eq!(astc_out[1], 0x40);
453        assert_eq!(astc_out[2], 0xFF);
454        assert_eq!(astc_out[3], 0xFF);
455    }
456
457    #[test]
458    fn q42_wasm_slice_and_localhost_query() {
459        let mut q1 = NQuin {
460            subject: 0x1111,
461            predicate: 0x2222,
462            object: 0x3333,
463            context: 0x4444,
464            metadata: 0,
465            parity: 0,
466        };
467        q1.parity = NQuin::calculate_parity(q1.subject, q1.predicate, q1.object, q1.context, q1.metadata);
468
469        let mut q2 = NQuin {
470            subject: 0x1111,
471            predicate: 0x2222,
472            object: 0x9999,
473            context: 0x4444,
474            metadata: 0,
475            parity: 0,
476        };
477        q2.parity = NQuin::calculate_parity(q2.subject, q2.predicate, q2.object, q2.context, q2.metadata);
478
479        let quins = [q1, q2];
480        let raw_bytes: &[u8] = bytemuck::cast_slice(&quins);
481
482        // 1. WasmSliceQ42Reader
483        let reader = WasmSliceQ42Reader::open(raw_bytes).expect("open raw q42 slice");
484        assert_eq!(reader.byte_len(), 96);
485        let all = reader.collect_all_quins().expect("collect quins");
486        assert_eq!(all.len(), 2);
487        assert_eq!(all[0].subject, 0x1111);
488        assert_eq!(all[1].object, 0x9999);
489
490        // 2. vibe::LocalHost::with_q42_pack
491        let mut host = vibe::LocalHost::with_q42_pack(raw_bytes);
492        let rows = host
493            .query_sparql_local(Some(0x1111), None, None, None)
494            .expect("local sparql query");
495        assert_eq!(rows.len(), 2);
496    }
497}