1use crate::bundle::streaming_loader::{
20 verify_arrived_range, HmcHeaderProbe, HmcProgressiveRangePlanner, HmcRangeRequest,
21 StreamPriorityTier,
22};
23use crate::bundle::BundleError;
24use crate::governance::worker_pool::{
25 GovernedHydrationPool, GovernancePoolError, HydrationCapabilityToken, HydrationReceipt,
26 HydrationTask, HydrationTaskKind, TranscodeCodec, CAP_ALL_HYDRATION,
27};
28use crate::q42::q42_volume::WasmSliceQ42Reader;
29use crate::render::texture_ktx2::{
30 select_hardware_compressed_format, HardwareCompressedUploadFormat, Ktx2Document, Ktx2Error,
31};
32use crate::render::texture_transcode::{
33 decode_astc_4x4_block, decode_bc7_block, decode_etc2_eac_rgba_block,
34};
35use vibe::Host;
36
37pub const DEFAULT_HYDRATION_QUEUE_CAPACITY: usize = 128;
39
40pub trait LocalHostQ42Ext {
43 fn with_q42_pack(q42_bytes: &[u8]) -> Self;
45 fn admit_q42_slice(&mut self, q42_bytes: &[u8]) -> Result<usize, String>;
47 fn query_sparql_local(
49 &mut self,
50 subject: Option<u64>,
51 predicate: Option<u64>,
52 object: Option<u64>,
53 context: Option<u64>,
54 ) -> Result<Vec<vibe::Value>, vibe::Diagnostic>;
55}
56
57impl LocalHostQ42Ext for vibe::LocalHost {
58 fn with_q42_pack(q42_bytes: &[u8]) -> Self {
59 let mut host = Self::default();
60 let _ = host.admit_q42_slice(q42_bytes);
61 host
62 }
63
64 fn admit_q42_slice(&mut self, q42_bytes: &[u8]) -> Result<usize, String> {
65 let mut map = std::collections::BTreeMap::new();
66 let byte_list = q42_bytes.iter().map(|&b| vibe::Value::U64(b as u64)).collect();
67 map.insert("bytes".into(), vibe::Value::List(byte_list));
68 map.insert("key".into(), vibe::Value::String("pack.q42".into()));
69 map.insert("load".into(), vibe::Value::Bool(true));
70 let args = vibe::Value::Record(map);
71 let span = vibe::Span::new(0, 0);
72 match self.capability_invoke("GraphDatabase.volume_open", &args, span) {
73 Ok(vibe::Value::Record(res)) => {
74 let count = match res.get("quin_count") {
75 Some(vibe::Value::U64(c)) => *c as usize,
76 _ => 0,
77 };
78 Ok(count)
79 }
80 Ok(_) => Ok(0),
81 Err(e) => Err(format!("{:?}", e)),
82 }
83 }
84
85 fn query_sparql_local(
86 &mut self,
87 subject: Option<u64>,
88 predicate: Option<u64>,
89 object: Option<u64>,
90 context: Option<u64>,
91 ) -> Result<Vec<vibe::Value>, vibe::Diagnostic> {
92 let mut map = std::collections::BTreeMap::new();
93 if let Some(s) = subject {
94 map.insert("subject".into(), vibe::Value::U64(s));
95 }
96 if let Some(p) = predicate {
97 map.insert("predicate".into(), vibe::Value::U64(p));
98 }
99 if let Some(o) = object {
100 map.insert("object".into(), vibe::Value::U64(o));
101 }
102 if let Some(c) = context {
103 map.insert("context".into(), vibe::Value::U64(c));
104 }
105 let args = vibe::Value::Record(map);
106 let span = vibe::Span::new(0, 0);
107 let res = self.capability_invoke("GraphDatabase.sparql", &args, span)?;
108 match res {
109 vibe::Value::List(rows) => Ok(rows),
110 _ => Ok(Vec::new()),
111 }
112 }
113}
114
115pub struct HmcPackStreamer {
117 pub local_host: vibe::LocalHost,
119 pub pool: GovernedHydrationPool,
121 pub token: HydrationCapabilityToken,
123 pub planned_requests: Vec<HmcRangeRequest>,
125 pub tier0_completed: usize,
127 pub tier1_completed: usize,
129 pub tier2_completed: usize,
131 pub total_bytes_admitted: u64,
133 next_task_id: u64,
135}
136
137pub type EcopunkPackStreamer = HmcPackStreamer;
139
140impl HmcPackStreamer {
141 pub fn new(holder_did_hash: u64, fuel_budget: u64, byte_budget: u64) -> Self {
143 let token = HydrationCapabilityToken::new_sealed(
144 holder_did_hash,
145 CAP_ALL_HYDRATION,
146 StreamPriorityTier::Tier2HighPolyAndMip0,
147 fuel_budget,
148 byte_budget,
149 );
150 Self {
151 local_host: vibe::LocalHost::default(),
152 pool: GovernedHydrationPool::new(DEFAULT_HYDRATION_QUEUE_CAPACITY),
153 token,
154 planned_requests: Vec::new(),
155 tier0_completed: 0,
156 tier1_completed: 0,
157 tier2_completed: 0,
158 total_bytes_admitted: 0,
159 next_task_id: 1,
160 }
161 }
162
163 pub fn plan_hmc_stream(
167 &mut self,
168 header_probe: HmcHeaderProbe,
169 index_footer_bytes: &[u8],
170 ) -> Result<&[HmcRangeRequest], BundleError> {
171 let planner = HmcProgressiveRangePlanner::from_header_and_index(header_probe, index_footer_bytes)?;
172 self.planned_requests = planner.plan_entry_ranges();
173 Ok(&self.planned_requests)
174 }
175
176 pub fn admit_and_verify_range(
178 &mut self,
179 req_idx: usize,
180 arrived_bytes: &[u8],
181 ) -> Result<(), GovernancePoolError> {
182 if req_idx >= self.planned_requests.len() {
183 return Err(GovernancePoolError::IntegrityMismatch {
184 task_id: req_idx as u64,
185 });
186 }
187 let req = &self.planned_requests[req_idx];
188 verify_arrived_range(req, arrived_bytes).map_err(|_| {
189 GovernancePoolError::IntegrityMismatch {
190 task_id: req_idx as u64,
191 }
192 })?;
193
194 if req.entry_kind.eq_ignore_ascii_case("q42") || req.entry_key.ends_with(".q42") {
196 let _ = self.local_host.admit_q42_slice(arrived_bytes);
197 }
198
199 match req.tier {
200 StreamPriorityTier::Tier0SemanticAndProcedural => self.tier0_completed += 1,
201 StreamPriorityTier::Tier1MidLodMeshes => self.tier1_completed += 1,
202 StreamPriorityTier::Tier2HighPolyAndMip0 => self.tier2_completed += 1,
203 }
204 self.total_bytes_admitted += arrived_bytes.len() as u64;
205 Ok(())
206 }
207
208 #[inline]
210 pub fn open_wasm_slice_reader<'a>(
211 &self,
212 q42_bytes: &'a [u8],
213 ) -> std::io::Result<WasmSliceQ42Reader<'a>> {
214 WasmSliceQ42Reader::open(q42_bytes)
215 }
216
217 pub fn submit_section_crc_check(
219 &mut self,
220 tier: StreamPriorityTier,
221 payload: Vec<u8>,
222 expected_crc32c: u32,
223 ) -> Result<u64, GovernancePoolError> {
224 let task_id = self.next_task_id;
225 self.next_task_id += 1;
226 self.pool.submit(
227 &mut self.token,
228 HydrationTask {
229 task_id,
230 tier,
231 kind: HydrationTaskKind::VerifySectionCrc32c {
232 payload,
233 expected_crc32c,
234 },
235 },
236 )?;
237 Ok(task_id)
238 }
239
240 pub fn submit_entry_sha256_check(
242 &mut self,
243 tier: StreamPriorityTier,
244 payload: Vec<u8>,
245 expected_sha256: [u8; 32],
246 ) -> Result<u64, GovernancePoolError> {
247 let task_id = self.next_task_id;
248 self.next_task_id += 1;
249 self.pool.submit(
250 &mut self.token,
251 HydrationTask {
252 task_id,
253 tier,
254 kind: HydrationTaskKind::VerifyEntrySha256 {
255 payload,
256 expected_sha256,
257 },
258 },
259 )?;
260 Ok(task_id)
261 }
262
263 pub fn submit_texture_transcode(
265 &mut self,
266 tier: StreamPriorityTier,
267 codec: TranscodeCodec,
268 block: [u8; 16],
269 ) -> Result<u64, GovernancePoolError> {
270 let task_id = self.next_task_id;
271 self.next_task_id += 1;
272 self.pool.submit(
273 &mut self.token,
274 HydrationTask {
275 task_id,
276 tier,
277 kind: HydrationTaskKind::TranscodeTextureBlock { codec, block },
278 },
279 )?;
280 Ok(task_id)
281 }
282
283 #[inline]
286 pub fn step_hydration_frame(
287 &mut self,
288 max_tasks_this_frame: usize,
289 ) -> Result<Vec<HydrationReceipt>, GovernancePoolError> {
290 self.pool.drain_bounded(max_tasks_this_frame)
291 }
292
293 #[inline]
296 pub fn inspect_and_select_gpu_texture_format(
297 &self,
298 ktx2_bytes: &[u8],
299 gpu_bc: bool,
300 gpu_astc: bool,
301 gpu_etc2: bool,
302 ) -> Result<Option<HardwareCompressedUploadFormat>, Ktx2Error> {
303 let doc = Ktx2Document::parse(ktx2_bytes)?;
304 Ok(select_hardware_compressed_format(&doc, gpu_bc, gpu_astc, gpu_etc2))
305 }
306
307 #[inline]
309 pub fn transcode_block_zero_heap(
310 codec: TranscodeCodec,
311 block: &[u8; 16],
312 out_rgba: &mut [u8; 64],
313 ) {
314 let pixels: &mut [[u8; 4]; 16] = bytemuck::cast_mut(out_rgba);
315 match codec {
316 TranscodeCodec::Bc7 => decode_bc7_block(block, pixels),
317 TranscodeCodec::Etc2EacRgba8 => decode_etc2_eac_rgba_block(block, pixels),
318 TranscodeCodec::Astc4x4 => decode_astc_4x4_block(block, pixels),
319 }
320 }
321}
322
323#[cfg(test)]
324mod tests {
325 use super::*;
326 use crate::bundle::BundleWriter;
327 use crate::container_10d::crc32c::crc32c;
328 use crate::NQuin;
329
330 #[test]
331 fn progressive_hmc_streaming_schedules_priority_tiers() {
332 let mut writer = BundleWriter::new();
333 writer
335 .add_file("ecopunk_terrain_lod0.10d", "10d", vec![0xAA; 1024], None)
336 .unwrap();
337 writer
338 .add_file("ecopunk_terrain_lod2.10d", "10d", vec![0xBB; 256], None)
339 .unwrap();
340 writer
341 .add_file("ecopunk_ecology.q42", "q42", vec![0xCC; 96], None)
342 .unwrap();
343 writer
344 .add_file("snow_gum_bark.ktx2", "ktx2", vec![0xDD; 512], None)
345 .unwrap();
346 let bundle_bytes = writer.build().unwrap();
347
348 let probe = HmcHeaderProbe::parse_header_range(&bundle_bytes[..64]).unwrap();
349 assert_eq!(probe.entry_count, 4);
350
351 let (idx_off, idx_len) = probe.index_footer_range();
352 let footer_slice = &bundle_bytes[idx_off as usize..(idx_off + idx_len) as usize];
353
354 let mut streamer = HmcPackStreamer::new(0x10D_E64_A1, 50_000, 100_000);
355 let schedule = streamer.plan_hmc_stream(probe, footer_slice).unwrap();
356 assert_eq!(schedule.len(), 4);
357
358 assert_eq!(schedule[0].tier, StreamPriorityTier::Tier0SemanticAndProcedural);
360 assert_eq!(schedule[0].entry_key, "ecopunk_ecology.q42");
361
362 assert_eq!(schedule[1].tier, StreamPriorityTier::Tier1MidLodMeshes);
364 assert_eq!(schedule[1].entry_key, "ecopunk_terrain_lod2.10d");
365
366 assert_eq!(schedule[2].tier, StreamPriorityTier::Tier2HighPolyAndMip0);
368 assert_eq!(schedule[3].tier, StreamPriorityTier::Tier2HighPolyAndMip0);
369
370 for i in 0..schedule.len() {
372 let start = streamer.planned_requests[i].offset as usize;
373 let end = start + streamer.planned_requests[i].length as usize;
374 streamer
375 .admit_and_verify_range(i, &bundle_bytes[start..end])
376 .unwrap();
377 }
378 assert_eq!(streamer.tier0_completed, 1);
379 assert_eq!(streamer.tier1_completed, 1);
380 assert_eq!(streamer.tier2_completed, 2);
381 }
382
383 #[test]
384 fn governed_worker_pool_hydrates_within_budget() {
385 let mut streamer = EcopunkPackStreamer::new(0x8888_7777, 500, 2048);
386 assert!(streamer.token.verify_seal());
387
388 let mut block = [0u8; 16];
390 block[0] = 1 << 6; let t1 = streamer
392 .submit_texture_transcode(StreamPriorityTier::Tier2HighPolyAndMip0, TranscodeCodec::Bc7, block)
393 .unwrap();
394
395 let payload = b"ecopunk procedural botanical section".to_vec();
397 let expected_crc = crc32c(&payload);
398 let t0 = streamer
399 .submit_section_crc_check(
400 StreamPriorityTier::Tier0SemanticAndProcedural,
401 payload.clone(),
402 expected_crc,
403 )
404 .unwrap();
405
406 let receipts = streamer.step_hydration_frame(2).unwrap();
408 assert_eq!(receipts.len(), 2);
409 assert_eq!(receipts[0].task_id, t0);
410 assert_eq!(receipts[0].tier, StreamPriorityTier::Tier0SemanticAndProcedural);
411 assert!(receipts[0].verified);
412 assert_eq!(receipts[1].task_id, t1);
413 assert_eq!(receipts[1].tier, StreamPriorityTier::Tier2HighPolyAndMip0);
414 assert!(receipts[1].transcoded_rgba8_4x4.is_some());
415
416 assert!(streamer.token.fuel_remaining < 500);
418 assert!(streamer.token.byte_budget_remaining < 2048);
419 assert!(streamer.token.verify_seal());
420 }
421
422 #[test]
423 fn hardware_compressed_format_selection_and_transcode() {
424 for mode in 0..8usize {
426 let mut block = [0xAAu8; 16];
427 block[0] = 1u8 << mode;
428 let mut out = [0u8; 64];
429 HmcPackStreamer::transcode_block_zero_heap(TranscodeCodec::Bc7, &block, &mut out);
430 assert!(out.iter().any(|&b| b > 0));
431 }
432
433 let mut etc2_block = [0u8; 16];
435 etc2_block[0] = 220; etc2_block[8] = 0xAA;
437 let mut etc2_out = [0u8; 64];
438 HmcPackStreamer::transcode_block_zero_heap(TranscodeCodec::Etc2EacRgba8, &etc2_block, &mut etc2_out);
439 assert!(etc2_out.iter().any(|&b| b > 0));
440
441 let mut astc_block = [0u8; 16];
443 astc_block[0] = 0xFC;
444 astc_block[1] = 0xFD; astc_block[8..10].copy_from_slice(&0x8000u16.to_le_bytes()); astc_block[10..12].copy_from_slice(&0x4000u16.to_le_bytes()); astc_block[12..14].copy_from_slice(&0xFF00u16.to_le_bytes()); astc_block[14..16].copy_from_slice(&0xFF00u16.to_le_bytes()); let mut astc_out = [0u8; 64];
450 HmcPackStreamer::transcode_block_zero_heap(TranscodeCodec::Astc4x4, &astc_block, &mut astc_out);
451 assert_eq!(astc_out[0], 0x80);
452 assert_eq!(astc_out[1], 0x40);
453 assert_eq!(astc_out[2], 0xFF);
454 assert_eq!(astc_out[3], 0xFF);
455 }
456
457 #[test]
458 fn q42_wasm_slice_and_localhost_query() {
459 let mut q1 = NQuin {
460 subject: 0x1111,
461 predicate: 0x2222,
462 object: 0x3333,
463 context: 0x4444,
464 metadata: 0,
465 parity: 0,
466 };
467 q1.parity = NQuin::calculate_parity(q1.subject, q1.predicate, q1.object, q1.context, q1.metadata);
468
469 let mut q2 = NQuin {
470 subject: 0x1111,
471 predicate: 0x2222,
472 object: 0x9999,
473 context: 0x4444,
474 metadata: 0,
475 parity: 0,
476 };
477 q2.parity = NQuin::calculate_parity(q2.subject, q2.predicate, q2.object, q2.context, q2.metadata);
478
479 let quins = [q1, q2];
480 let raw_bytes: &[u8] = bytemuck::cast_slice(&quins);
481
482 let reader = WasmSliceQ42Reader::open(raw_bytes).expect("open raw q42 slice");
484 assert_eq!(reader.byte_len(), 96);
485 let all = reader.collect_all_quins().expect("collect quins");
486 assert_eq!(all.len(), 2);
487 assert_eq!(all[0].subject, 0x1111);
488 assert_eq!(all[1].object, 0x9999);
489
490 let mut host = vibe::LocalHost::with_q42_pack(raw_bytes);
492 let rows = host
493 .query_sparql_local(Some(0x1111), None, None, None)
494 .expect("local sparql query");
495 assert_eq!(rows.len(), 2);
496 }
497}