Skip to main content

qualia_core_db/query/
lexicon.rs

1//! Lexicon Dictionary Manager
2//! Maps multi-modal semantic concepts (Text, Audio, Visual) into the deterministic 60-bit integers
3//! required by the NQuin data structure.
4
5// Re-export the embedded triple tag from resolver for SPARQL-Star support
6pub use crate::resolver::{TAG_EMBEDDED, TAG_WEBIZEN};
7
8/// Represents the pluralistic forms that a semantic concept can take.
9/// We explicitly reject the assumption that knowledge is exclusively bound to Unicode strings.
10pub enum SemanticModality<'a> {
11    Text(&'a str),
12    AudioHash(&'a [u8]),                  // For mother tongues / oral traditions
13    CeremonialVisual(&'a [u8]),           // For heraldry / visual concepts
14    PhoneticSchema(&'a [u8]),             // For non-western phonetics
15    Visual(&'a [u8]),                     // Generalized RGB/RGBA images & video frames
16    Spatial3D(&'a [u8]),                  // 3D MeshIR, STL/3MF, point clouds, volumetric grids
17    Biosignal(&'a [u8]),                  // PPG/rPPG, ECG, EEG, respiratory telemetry
18    Molecular(&'a [u8]),                  // SMILES, InChI, protein sequences, chemical structures
19    QuantumPhysical(&'a [u8]), // Quantum state vectors, density functional grids, ODE fields
20    SpatioTemporalTrace(&'a [u8]), // LTL temporal traces, geospatial paths, motion kinematics
21    Custom { tag: u32, bytes: &'a [u8] }, // Arbitrary domain modality with custom tag
22}
23
24/// Generates a deterministic, collision-resistant 60-bit token from a raw byte stream.
25/// Uses a custom FNV-1a inspired hash restricted to 60 bits.
26#[inline(always)]
27pub fn generate_60bit_token(bytes: &[u8]) -> u64 {
28    let mut hash: u64 = 0xcbf29ce484222325; // FNV offset basis
29    for &byte in bytes {
30        hash ^= byte as u64;
31        hash = hash.wrapping_mul(0x100000001b3); // FNV prime
32    }
33    // Truncate to 60 bits (the top 4 bits are reserved for datatype tags in the O vector)
34    hash & 0x0FFF_FFFF_FFFF_FFFF
35}
36
37/// Generates a Virtual ID for a SPARQL-Star embedded triple `<<s p o>>`.
38///
39/// This function serializes the three u64 component IDs into a 24-byte array
40/// and hashes them using FNV-1a, then tags the result with TAG_EMBEDDED.
41///
42/// # Arguments
43/// * subject - The subject u64 ID
44/// * predicate - The predicate u64 ID  
45/// * object - The object u64 ID
46///
47/// # Returns
48/// A 64-bit Virtual ID with the TAG_EMBEDDED bit set, suitable for
49/// storage in the Subject or Object position of a NQuin.
50#[inline(always)]
51pub fn generate_embedded_triple_id(subject: u64, predicate: u64, object: u64) -> u64 {
52    // Serialize the three u64 IDs into a 24-byte array
53    let mut bytes = [0u8; 24];
54    bytes[0..8].copy_from_slice(&subject.to_le_bytes());
55    bytes[8..16].copy_from_slice(&predicate.to_le_bytes());
56    bytes[16..24].copy_from_slice(&object.to_le_bytes());
57
58    // Hash the bytes and tag with EMBEDDED marker
59    generate_60bit_token(&bytes) | TAG_EMBEDDED
60}
61
62/// In-memory Lexicon manager to handle reverse lookups in the future.
63/// For now, ingestion purely maps forward (Bytes -> u64) via the hash.
64pub struct LexiconManager {
65    // A production database would memory-map a reverse lookup file here (u64 -> Modality)
66}
67
68impl LexiconManager {
69    /// Creates a default `LexiconManager`.
70    pub fn new() -> Self {
71        Self {}
72    }
73
74    /// Converts a multi-modal semantic concept into its 60-bit hardware representation.
75    pub fn tokenize_modal(&self, modality: &SemanticModality) -> u64 {
76        match modality {
77            SemanticModality::Text(text) => generate_60bit_token(text.as_bytes()),
78            SemanticModality::AudioHash(bytes) => generate_60bit_token(bytes),
79            SemanticModality::CeremonialVisual(bytes) => generate_60bit_token(bytes),
80            SemanticModality::PhoneticSchema(bytes) => generate_60bit_token(bytes),
81            SemanticModality::Visual(bytes) => generate_60bit_token(bytes),
82            SemanticModality::Spatial3D(bytes) => generate_60bit_token(bytes),
83            SemanticModality::Biosignal(bytes) => generate_60bit_token(bytes),
84            SemanticModality::Molecular(bytes) => generate_60bit_token(bytes),
85            SemanticModality::QuantumPhysical(bytes) => generate_60bit_token(bytes),
86            SemanticModality::SpatioTemporalTrace(bytes) => generate_60bit_token(bytes),
87            SemanticModality::Custom { tag, bytes } => {
88                let mut combined = Vec::with_capacity(4 + bytes.len());
89                combined.extend_from_slice(&tag.to_le_bytes());
90                combined.extend_from_slice(bytes);
91                generate_60bit_token(&combined)
92            }
93        }
94    }
95
96    /// Legacy support for text strings
97    pub fn tokenize(&self, literal: &str) -> u64 {
98        self.tokenize_modal(&SemanticModality::Text(literal))
99    }
100}
101
102use std::collections::HashMap;
103
104/// Outcome of interning a `(handle, value)` pair into a collision-aware lexicon.
105#[derive(Debug, PartialEq, Eq, Clone, Copy)]
106pub enum Intern {
107    /// First time this handle is seen.
108    New,
109    /// Same handle, SAME value — an idempotent re-intern.
110    Seen,
111    /// Same handle, DIFFERENT value — a genuine handle collision (both values kept).
112    Collision,
113}
114
115/// Collision-aware string interner — the lexicon collision backstop (task #22).
116///
117/// The plain build-time map (`HashMap<u64, String>` in `external_sort`) silently
118/// OVERWRITES on a handle collision (two distinct strings hashing to the same 60-bit
119/// token), losing data with no signal. This interner DETECTS the collision at intern
120/// time — O(1), off the resolution hot path — and KEEPS BOTH values, so a collision
121/// becomes loud + recoverable rather than silent corruption. Resolution stays
122/// single-value and fast for the (overwhelmingly common) collision-free handles; only
123/// a flagged handle pays a comparison over its small bucket (length-gated by `str` eq),
124/// from memory. This is a host-side structure (`HashMap`/`String`), separate from the
125/// 42 MB `SlgArena` — its allocations are one-time intern cost, not per-resolution.
126#[derive(Default)]
127pub struct LexiconInterner {
128    /// handle -> first-interned value (the fast, common path).
129    map: HashMap<u64, String>,
130    /// handle -> all distinct values, populated ONLY when a collision occurs.
131    buckets: HashMap<u64, Vec<String>>,
132}
133
134impl LexiconInterner {
135    /// Creates `LexiconInterner` with the given state.
136    pub fn new() -> Self {
137        Self::default()
138    }
139
140    /// Intern `(generate_60bit_token(value), value)`. Returns the collision outcome.
141    pub fn intern_str(&mut self, value: &str) -> Intern {
142        self.intern(generate_60bit_token(value.as_bytes()), value)
143    }
144
145    /// Intern an explicit `(handle, value)` pair — the handle need not be
146    /// `generate_60bit_token(value)` (e.g. did:q42 / Webizen handles). Detects a
147    /// same-handle / different-value collision and preserves both values.
148    pub fn intern(&mut self, handle: u64, value: &str) -> Intern {
149        match self.map.get(&handle) {
150            None => {
151                self.map.insert(handle, value.to_string());
152                Intern::New
153            }
154            Some(existing) if existing == value => Intern::Seen,
155            Some(existing) => {
156                // Genuine collision: preserve BOTH values; never silently overwrite.
157                let bucket = self
158                    .buckets
159                    .entry(handle)
160                    .or_insert_with(|| vec![existing.clone()]);
161                if !bucket.iter().any(|v| v == value) {
162                    bucket.push(value.to_string());
163                }
164                Intern::Collision
165            }
166        }
167    }
168
169    /// Whether `handle` has more than one distinct interned value (a collision).
170    #[inline]
171    pub fn is_collision(&self, handle: u64) -> bool {
172        self.buckets.contains_key(&handle)
173    }
174
175    /// Resolve `handle` -> value with collision awareness. Collision-free (common): the
176    /// single value, no comparison. Collided (rare): `None` — there is no single answer,
177    /// so the caller disambiguates with the query value via [`Self::resolve_value`]
178    /// rather than receiving a silently-wrong one.
179    pub fn resolve(&self, handle: u64) -> Option<&str> {
180        if self.buckets.contains_key(&handle) {
181            return None;
182        }
183        self.map.get(&handle).map(String::as_str)
184    }
185
186    /// Disambiguating resolve — the backstop. Returns the stored value equal to `query`
187    /// iff `handle` interns it. `str` equality length-gates before the byte compare, so
188    /// the rare collision path stays cheap.
189    pub fn resolve_value(&self, handle: u64, query: &str) -> Option<&str> {
190        if let Some(bucket) = self.buckets.get(&handle) {
191            return bucket.iter().map(String::as_str).find(|&s| s == query);
192        }
193        self.map
194            .get(&handle)
195            .map(String::as_str)
196            .filter(|&s| s == query)
197    }
198}
199
200#[cfg(test)]
201mod tests {
202    use super::*;
203
204    #[test]
205    fn test_60bit_truncation() {
206        let uri = "https://mediaprophet.github.io/qualiaDB/user/123";
207        let token = generate_60bit_token(uri.as_bytes());
208
209        // Ensure the top 4 bits are strictly 0 (Lexicon ID datatype tag)
210        assert_eq!(token >> 60, 0, "Token spilled over 60 bits");
211    }
212
213    /// Regression guard for the hash-space unification (task #14): the compile-time
214    /// `q_hash` and the runtime `generate_60bit_token` MUST be one identity space,
215    /// or compile-time-baked URIs (deontic/values/MCP) won't join runtime-parsed/
216    /// ingested URIs (turtle/SPARQL/SHACL/corpus). If this ever fails, the two
217    /// drifted apart again — fix the divergence, don't relax the test.
218    #[test]
219    fn q_hash_and_generate_60bit_token_are_one_space() {
220        for s in [
221            "",
222            "Bob",
223            "http://www.w3.org/ns/shacl#maxInclusive",
224            "https://ns.webcivics.net/values/NaturalPerson",
225            "q42:Principal",
226            "naïve—prière—母語", // non-ASCII must agree too
227        ] {
228            assert_eq!(
229                crate::q_hash(s),
230                generate_60bit_token(s.as_bytes()),
231                "q_hash and generate_60bit_token diverged for {s:?}",
232            );
233            // Both are pure 60-bit identifiers: top 4 bits reserved for the tag overlay.
234            assert_eq!(
235                crate::q_hash(s) >> 60,
236                0,
237                "q_hash spilled past 60 bits for {s:?}"
238            );
239        }
240    }
241
242    #[test]
243    fn interner_detects_collision_and_keeps_both_values() {
244        let mut lx = LexiconInterner::new();
245        // Force a handle collision (same handle, different values) by interning explicit
246        // handles — a natural 60-bit FNV collision is infeasible to find in a test.
247        let h = 0x0ABC_DEF0_1234_5678;
248        assert_eq!(lx.intern(h, "alpha"), Intern::New);
249        assert_eq!(lx.intern(h, "alpha"), Intern::Seen); // idempotent
250        assert_eq!(lx.intern(h, "beta"), Intern::Collision); // genuine collision
251        assert!(lx.is_collision(h));
252
253        // Both values survive and are recoverable via the disambiguating resolve —
254        // unlike a plain HashMap, where "alpha" would have been silently overwritten.
255        assert_eq!(lx.resolve_value(h, "alpha"), Some("alpha"));
256        assert_eq!(lx.resolve_value(h, "beta"), Some("beta"));
257        assert_eq!(lx.resolve_value(h, "gamma"), None);
258
259        // Bare resolve refuses to guess on an ambiguous handle (no silent wrong answer).
260        assert_eq!(lx.resolve(h), None);
261    }
262
263    #[test]
264    fn interner_collision_free_is_the_fast_single_value_path() {
265        let mut lx = LexiconInterner::new();
266        let iri = "https://ns.webcivics.net/values/State";
267        assert_eq!(lx.intern_str(iri), Intern::New);
268        assert_eq!(lx.intern_str(iri), Intern::Seen);
269
270        // q_hash == generate_60bit_token (post-#14), so this is the interned handle.
271        let h = crate::q_hash(iri);
272        assert!(!lx.is_collision(h));
273        // Collision-free: bare resolve returns the single value directly, no comparison.
274        assert_eq!(lx.resolve(h), Some(iri));
275    }
276
277    #[test]
278    fn test_determinism() {
279        let uri = "qualia:guardian";
280        let t1 = generate_60bit_token(uri.as_bytes());
281        let t2 = generate_60bit_token(uri.as_bytes());
282        assert_eq!(t1, t2, "Tokens are not deterministic");
283    }
284
285    #[test]
286    fn test_linguistic_plurality() {
287        let lexicon = LexiconManager::new();
288
289        // Simulating a written concept
290        let written = SemanticModality::Text("peace_infrastructure");
291        let t1 = lexicon.tokenize_modal(&written);
292
293        // Simulating the exact same concept represented as a cryptographic audio hash of a spoken prayer
294        let audio_hash = vec![0x1a, 0x2b, 0x3c, 0x4d, 0x5e];
295        let oral = SemanticModality::AudioHash(&audio_hash);
296        let t2 = lexicon.tokenize_modal(&oral);
297
298        // Simulating a ceremonial SVG file representation
299        let svg_bytes = b"<svg>Heraldry</svg>";
300        let visual = SemanticModality::CeremonialVisual(svg_bytes);
301        let t3 = lexicon.tokenize_modal(&visual);
302
303        // Prove that the database treats all modalities as valid 60-bit structural Quins
304        assert!(t1 <= 0x0FFF_FFFF_FFFF_FFFF);
305        assert!(t2 <= 0x0FFF_FFFF_FFFF_FFFF);
306        assert!(t3 <= 0x0FFF_FFFF_FFFF_FFFF);
307
308        assert_ne!(t1, t2);
309        assert_ne!(t1, t3);
310
311        // Test non-identical bytes produce different hashes
312        let altered_audio = vec![0x1a, 0x2b, 0x3c, 0x4d, 0x5f];
313        let altered_oral = SemanticModality::AudioHash(&altered_audio);
314        let t4 = lexicon.tokenize_modal(&altered_oral);
315
316        assert_ne!(t2, t4);
317    }
318
319    #[test]
320    fn test_generalized_multimodal_plurality() {
321        let lexicon = LexiconManager::new();
322
323        let spatial = SemanticModality::Spatial3D(b"stl_mesh_data_3d");
324        let bio = SemanticModality::Biosignal(b"rppg_waveform_data");
325        let chem = SemanticModality::Molecular(b"CCO_ethanol_smiles");
326        let qphys = SemanticModality::QuantumPhysical(b"state_vector_psi");
327        let trace = SemanticModality::SpatioTemporalTrace(b"ltl_trace_ltl");
328        let custom = SemanticModality::Custom {
329            tag: 0x42,
330            bytes: b"custom_payload",
331        };
332
333        for m in [&spatial, &bio, &chem, &qphys, &trace, &custom] {
334            let tok = lexicon.tokenize_modal(m);
335            assert!(tok <= 0x0FFF_FFFF_FFFF_FFFF, "token within 60-bit bound");
336            assert_ne!(tok, 0);
337        }
338    }
339}