Skip to main content

qualia_core_db/bundle/
mod.rs

1//! `.hmc` — the **hyper-media-container**: a **transparent container-of-files**
2//! bundle for shipping a set of sealed Qualia assets (`.emf` spectral media,
3//! `.10d` meshes, `.q42` graph volumes, `.p64` weights, …)
4//! as one attestable unit (a release artefact, a downloadable pack, a signed
5//! asset set).
6//!
7//! # Why a bundle, and why *transparent*
8//! The Qualia container family (`.10d` / `.q42` / `.p64`) are all internally
9//! navigable — you can pull a *segment* (a mesh section, a subgraph, a weight
10//! shard) without reading the whole file, ideally `mmap` zero-copy. A bundle
11//! must not destroy that. So a `.hmc` bundle only **concatenates intact files,
12//! page-aligned, with an index of absolute offsets** — it never compresses,
13//! re-chunks, or reframes an entry. Consequences:
14//!
15//! - `reader.get(key)` returns a zero-copy slice that **is** a byte-identical
16//!   standalone file — feed it straight to the existing `.q42`/`.10d`/`.p64`
17//!   reader; its interior offsets resolve unchanged.
18//! - `reader.segment(key, off, len)` reaches an interior segment of an entry
19//!   directly — the bundle does not interfere with segment-level access.
20//! - HTTP range-fetching one interior segment works: `entry.offset + seg.off`.
21//!
22//! It lives in `qualia-core-db` so it is **one reader for both channels** —
23//! native (`BundleMmap`, zero-copy) and WASM (parse fetched/ranged bytes).
24//!
25//! See [`format`] for the on-disk layout.
26
27pub mod cbor_ld_manifest;
28pub mod entity_compiler;
29pub mod entity_verifier;
30pub mod field_range_planner;
31pub(crate) mod format;
32mod reader;
33pub use entity_compiler as twin_compiler;
34pub use entity_verifier as twin_verifier;
35pub mod species_meta;
36pub mod stream;
37pub mod streaming_hydration;
38pub mod streaming_loader;
39pub mod variant_matrix;
40mod writer;
41
42pub use streaming_hydration::{
43    DEFAULT_HYDRATION_QUEUE_CAPACITY, EcopunkPackStreamer, HmcPackStreamer, LocalHostQ42Ext,
44};
45
46pub use entity_compiler::{
47    compile_digital_twin, compile_entity, EntityCompileInput, EntityCompileOutput,
48    TwinCompileInput, TwinCompileOutput,
49};
50
51pub use entity_verifier::{
52    verify_digital_twin_10d, verify_digital_twin_bundle, verify_entity_10d, verify_entity_bundle,
53    AffordanceVerificationSummary, CommonsObligationSummary, DigitalTwin10dVerification,
54    DigitalTwinBundleVerification, Entity10dVerification, EntityBundleVerification,
55    ImpostorVerificationSummary, SurfelVerificationSummary,
56};
57
58pub use cbor_ld_manifest::{
59    AcousticAnchors, CborLdManifest, CommonsObligation, CredentialProof, CredentialSubject,
60    GeometryLayout, SectionLayoutEntry, SubjectAffordances, CBOR_LD_CONTEXT_CREDENTIALS_V1,
61    CBOR_LD_CONTEXT_WEBCIVICS_V1, CBOR_LD_MANIFEST_ENTRY_KEY, CBOR_LD_MANIFEST_KIND,
62};
63pub use field_range_planner::*;
64pub use format::{
65    BundleEntry, BundleError, BUNDLE_ENTRY_ALIGN, BUNDLE_FLAG_PAGE_ALIGN_4096, BUNDLE_HEADER_SIZE,
66    BUNDLE_MAGIC, BUNDLE_PAGE_ALIGN_4096, BUNDLE_VERSION, SHA256_LEN,
67};
68pub use reader::{BundleReadLimits, BundleReader};
69pub use species_meta::{
70    FaunaSpeciesMeta, FloraSpeciesMeta, HolisticSpeciesMeta, MechanicalAssemblyMeta,
71    SPECIES_META_ENTRY_KEY, SPECIES_META_KIND,
72};
73pub use streaming_loader::{
74    classify_10d_section_tier, classify_entry_tier, verify_arrived_range, HmcHeaderProbe,
75    HmcProgressiveRangePlanner, HmcRangeRequest, StreamPriorityTier,
76};
77pub use variant_matrix::{
78    BiologicalPhenotype, EntityFamilyManifest, LodTier, ModelVariantDescriptor, StructuralState,
79    TwinFamilyManifest, FAMILY_MANIFEST_ENTRY_KEY, FAMILY_MANIFEST_KIND,
80};
81pub use writer::BundleWriter;
82
83#[cfg(not(target_arch = "wasm32"))]
84pub use reader::BundleMmap;
85
86#[cfg(test)]
87mod tests {
88    use super::*;
89    use crate::entity_view::entity_ontologies::LifecycleStage;
90
91    #[test]
92    fn heterogeneous_pack_of_three_kinds_roundtrips() {
93        // A bundle can carry different sealed formats side by side.
94        let ten_d: Vec<u8> = (0..300u32).map(|i| (i * 7) as u8).collect();
95        let q42 = b"q42 graph volume bytes".to_vec();
96        let p64: Vec<u8> = vec![0x40; 129];
97
98        let mut w = BundleWriter::new();
99        w.add_file("body.10d", "10d", ten_d.clone(), None).unwrap();
100        w.add_file("wordnet.q42", "q42", q42.clone(), None).unwrap();
101        w.add_file("model.p64", "p64", p64.clone(), None).unwrap();
102        let bytes = w.build().unwrap();
103
104        let r = BundleReader::parse(&bytes).unwrap();
105        assert_eq!(r.entries().len(), 3);
106        assert_eq!(r.get("body.10d").unwrap(), ten_d.as_slice());
107        assert_eq!(r.get("wordnet.q42").unwrap(), q42.as_slice());
108        assert_eq!(r.get("model.p64").unwrap(), p64.as_slice());
109        assert_eq!(r.entry("model.p64").unwrap().kind, "p64");
110        // Every entry is page-aligned so its interior alignment is preserved.
111        for e in r.entries() {
112            assert_eq!(e.offset % BUNDLE_ENTRY_ALIGN as u64, 0);
113        }
114        // Every entry verifies against its recorded hash.
115        assert!(r.entries().iter().all(|e| r.verify_entry(&e.key)));
116    }
117
118    #[test]
119    fn empty_bundle_is_valid() {
120        let bytes = BundleWriter::new().build().unwrap();
121        let r = BundleReader::parse(&bytes).unwrap();
122        assert!(r.entries().is_empty());
123    }
124
125    #[test]
126    fn tampered_bundle_fails_verification() {
127        let mut w = BundleWriter::new();
128        w.add_file("asset.10d", "10d", b"valid asset bytes".to_vec(), None)
129            .unwrap();
130        let mut bytes = w.build().unwrap();
131
132        // Tamper with one byte in the payload
133        let r = BundleReader::parse(&bytes).unwrap();
134        let offset = r.entry("asset.10d").unwrap().offset as usize;
135        bytes[offset] ^= 0xFF;
136
137        // Bundle parse must reject with CRC mismatch
138        let res = BundleReader::parse(&bytes);
139        assert!(matches!(res, Err(BundleError::CrcMismatch { .. })));
140    }
141
142    #[test]
143    fn family_manifest_bundle_roundtrip() {
144        let mut manifest = TwinFamilyManifest::new(
145            "urn:taxon:water_pump_v3",
146            "Water Pump V3",
147            "machinery",
148            "operational_lod0",
149        );
150        manifest.add_variant(ModelVariantDescriptor {
151            variant_id: "operational_lod0".to_string(),
152            phenotype: BiologicalPhenotype::Unspecified,
153            stage: LifecycleStage::Operational,
154            lod: LodTier::Lod0,
155            state: StructuralState::Pristine,
156            asset_key: "pump_hero.10d".to_string(),
157            entity_ontology_key: Some("pump_ontology.q42".to_string()),
158            procedural_seed: Some(101),
159        });
160
161        let mut w = BundleWriter::new();
162        w.add_family_manifest(&manifest).unwrap();
163        w.add_file("pump_hero.10d", "10d", vec![0x10; 64], None)
164            .unwrap();
165        w.add_file("pump_ontology.q42", "q42", vec![0x42; 64], None)
166            .unwrap();
167        let bytes = w.build().unwrap();
168
169        let r = BundleReader::parse(&bytes).unwrap();
170        let decoded_manifest = r.get_family_manifest().unwrap().expect("manifest exists");
171        assert_eq!(decoded_manifest.family_id, "urn:taxon:water_pump_v3");
172        let resolved = decoded_manifest
173            .resolve_variant(
174                BiologicalPhenotype::Unspecified,
175                LifecycleStage::Operational,
176                LodTier::Lod0,
177                StructuralState::Pristine,
178            )
179            .unwrap();
180        assert_eq!(resolved.variant_id, "operational_lod0");
181        assert_eq!(resolved.asset_key, "pump_hero.10d");
182    }
183
184    #[test]
185    fn species_meta_and_4096_alignment_roundtrip() {
186        let flora = HolisticSpeciesMeta::Flora(FloraSpeciesMeta::new("urn:taxon:eucalyptus_regnans"));
187        let mut w = BundleWriter::new().with_page_align_4096(true);
188        w.add_species_meta(&flora).unwrap();
189        w.add_file("tree.10d", "10d", vec![0x10; 128], None).unwrap();
190        let bytes = w.build().unwrap();
191
192        let r = BundleReader::parse(&bytes).unwrap();
193        assert!(r.is_page_aligned_4096());
194        for e in r.entries() {
195            assert_eq!(e.offset % BUNDLE_PAGE_ALIGN_4096 as u64, 0);
196        }
197        let decoded = r.get_species_meta().unwrap().expect("species meta exists");
198        assert_eq!(decoded, flora);
199    }
200}