qualia_core_db/query/lexicon.rs
1//! Lexicon Dictionary Manager
2//! Maps multi-modal semantic concepts (Text, Audio, Visual) into the deterministic 60-bit integers
3//! required by the NQuin data structure.
4
5// Re-export the embedded triple tag from resolver for SPARQL-Star support
6pub use crate::resolver::{TAG_EMBEDDED, TAG_WEBIZEN};
7
8/// Represents the pluralistic forms that a semantic concept can take.
9/// We explicitly reject the assumption that knowledge is exclusively bound to Unicode strings.
10pub enum SemanticModality<'a> {
11 Text(&'a str),
12 AudioHash(&'a [u8]), // For mother tongues / oral traditions
13 CeremonialVisual(&'a [u8]), // For heraldry / visual concepts
14 PhoneticSchema(&'a [u8]), // For non-western phonetics
15 Visual(&'a [u8]), // Generalized RGB/RGBA images & video frames
16 Spatial3D(&'a [u8]), // 3D MeshIR, STL/3MF, point clouds, volumetric grids
17 Biosignal(&'a [u8]), // PPG/rPPG, ECG, EEG, respiratory telemetry
18 Molecular(&'a [u8]), // SMILES, InChI, protein sequences, chemical structures
19 QuantumPhysical(&'a [u8]), // Quantum state vectors, density functional grids, ODE fields
20 SpatioTemporalTrace(&'a [u8]), // LTL temporal traces, geospatial paths, motion kinematics
21 Custom { tag: u32, bytes: &'a [u8] }, // Arbitrary domain modality with custom tag
22}
23
24/// Generates a deterministic, collision-resistant 60-bit token from a raw byte stream.
25/// Uses a custom FNV-1a inspired hash restricted to 60 bits.
26#[inline(always)]
27pub fn generate_60bit_token(bytes: &[u8]) -> u64 {
28 let mut hash: u64 = 0xcbf29ce484222325; // FNV offset basis
29 for &byte in bytes {
30 hash ^= byte as u64;
31 hash = hash.wrapping_mul(0x100000001b3); // FNV prime
32 }
33 // Truncate to 60 bits (the top 4 bits are reserved for datatype tags in the O vector)
34 hash & 0x0FFF_FFFF_FFFF_FFFF
35}
36
37/// Generates a Virtual ID for a SPARQL-Star embedded triple `<<s p o>>`.
38///
39/// This function serializes the three u64 component IDs into a 24-byte array
40/// and hashes them using FNV-1a, then tags the result with TAG_EMBEDDED.
41///
42/// # Arguments
43/// * subject - The subject u64 ID
44/// * predicate - The predicate u64 ID
45/// * object - The object u64 ID
46///
47/// # Returns
48/// A 64-bit Virtual ID with the TAG_EMBEDDED bit set, suitable for
49/// storage in the Subject or Object position of a NQuin.
50#[inline(always)]
51pub fn generate_embedded_triple_id(subject: u64, predicate: u64, object: u64) -> u64 {
52 // Serialize the three u64 IDs into a 24-byte array
53 let mut bytes = [0u8; 24];
54 bytes[0..8].copy_from_slice(&subject.to_le_bytes());
55 bytes[8..16].copy_from_slice(&predicate.to_le_bytes());
56 bytes[16..24].copy_from_slice(&object.to_le_bytes());
57
58 // Hash the bytes and tag with EMBEDDED marker
59 generate_60bit_token(&bytes) | TAG_EMBEDDED
60}
61
62/// In-memory Lexicon manager to handle reverse lookups in the future.
63/// For now, ingestion purely maps forward (Bytes -> u64) via the hash.
64pub struct LexiconManager {
65 // A production database would memory-map a reverse lookup file here (u64 -> Modality)
66}
67
68impl LexiconManager {
69 /// Creates a default `LexiconManager`.
70 pub fn new() -> Self {
71 Self {}
72 }
73
74 /// Converts a multi-modal semantic concept into its 60-bit hardware representation.
75 pub fn tokenize_modal(&self, modality: &SemanticModality) -> u64 {
76 match modality {
77 SemanticModality::Text(text) => generate_60bit_token(text.as_bytes()),
78 SemanticModality::AudioHash(bytes) => generate_60bit_token(bytes),
79 SemanticModality::CeremonialVisual(bytes) => generate_60bit_token(bytes),
80 SemanticModality::PhoneticSchema(bytes) => generate_60bit_token(bytes),
81 SemanticModality::Visual(bytes) => generate_60bit_token(bytes),
82 SemanticModality::Spatial3D(bytes) => generate_60bit_token(bytes),
83 SemanticModality::Biosignal(bytes) => generate_60bit_token(bytes),
84 SemanticModality::Molecular(bytes) => generate_60bit_token(bytes),
85 SemanticModality::QuantumPhysical(bytes) => generate_60bit_token(bytes),
86 SemanticModality::SpatioTemporalTrace(bytes) => generate_60bit_token(bytes),
87 SemanticModality::Custom { tag, bytes } => {
88 let mut combined = Vec::with_capacity(4 + bytes.len());
89 combined.extend_from_slice(&tag.to_le_bytes());
90 combined.extend_from_slice(bytes);
91 generate_60bit_token(&combined)
92 }
93 }
94 }
95
96 /// Legacy support for text strings
97 pub fn tokenize(&self, literal: &str) -> u64 {
98 self.tokenize_modal(&SemanticModality::Text(literal))
99 }
100}
101
102use std::collections::HashMap;
103
104/// Outcome of interning a `(handle, value)` pair into a collision-aware lexicon.
105#[derive(Debug, PartialEq, Eq, Clone, Copy)]
106pub enum Intern {
107 /// First time this handle is seen.
108 New,
109 /// Same handle, SAME value — an idempotent re-intern.
110 Seen,
111 /// Same handle, DIFFERENT value — a genuine handle collision (both values kept).
112 Collision,
113}
114
115/// Collision-aware string interner — the lexicon collision backstop (task #22).
116///
117/// The plain build-time map (`HashMap<u64, String>` in `external_sort`) silently
118/// OVERWRITES on a handle collision (two distinct strings hashing to the same 60-bit
119/// token), losing data with no signal. This interner DETECTS the collision at intern
120/// time — O(1), off the resolution hot path — and KEEPS BOTH values, so a collision
121/// becomes loud + recoverable rather than silent corruption. Resolution stays
122/// single-value and fast for the (overwhelmingly common) collision-free handles; only
123/// a flagged handle pays a comparison over its small bucket (length-gated by `str` eq),
124/// from memory. This is a host-side structure (`HashMap`/`String`), separate from the
125/// 42 MB `SlgArena` — its allocations are one-time intern cost, not per-resolution.
126#[derive(Default)]
127pub struct LexiconInterner {
128 /// handle -> first-interned value (the fast, common path).
129 map: HashMap<u64, String>,
130 /// handle -> all distinct values, populated ONLY when a collision occurs.
131 buckets: HashMap<u64, Vec<String>>,
132}
133
134impl LexiconInterner {
135 /// Creates `LexiconInterner` with the given state.
136 pub fn new() -> Self {
137 Self::default()
138 }
139
140 /// Intern `(generate_60bit_token(value), value)`. Returns the collision outcome.
141 pub fn intern_str(&mut self, value: &str) -> Intern {
142 self.intern(generate_60bit_token(value.as_bytes()), value)
143 }
144
145 /// Intern an explicit `(handle, value)` pair — the handle need not be
146 /// `generate_60bit_token(value)` (e.g. did:q42 / Webizen handles). Detects a
147 /// same-handle / different-value collision and preserves both values.
148 pub fn intern(&mut self, handle: u64, value: &str) -> Intern {
149 match self.map.get(&handle) {
150 None => {
151 self.map.insert(handle, value.to_string());
152 Intern::New
153 }
154 Some(existing) if existing == value => Intern::Seen,
155 Some(existing) => {
156 // Genuine collision: preserve BOTH values; never silently overwrite.
157 let bucket = self
158 .buckets
159 .entry(handle)
160 .or_insert_with(|| vec![existing.clone()]);
161 if !bucket.iter().any(|v| v == value) {
162 bucket.push(value.to_string());
163 }
164 Intern::Collision
165 }
166 }
167 }
168
169 /// Whether `handle` has more than one distinct interned value (a collision).
170 #[inline]
171 pub fn is_collision(&self, handle: u64) -> bool {
172 self.buckets.contains_key(&handle)
173 }
174
175 /// Resolve `handle` -> value with collision awareness. Collision-free (common): the
176 /// single value, no comparison. Collided (rare): `None` — there is no single answer,
177 /// so the caller disambiguates with the query value via [`Self::resolve_value`]
178 /// rather than receiving a silently-wrong one.
179 pub fn resolve(&self, handle: u64) -> Option<&str> {
180 if self.buckets.contains_key(&handle) {
181 return None;
182 }
183 self.map.get(&handle).map(String::as_str)
184 }
185
186 /// Disambiguating resolve — the backstop. Returns the stored value equal to `query`
187 /// iff `handle` interns it. `str` equality length-gates before the byte compare, so
188 /// the rare collision path stays cheap.
189 pub fn resolve_value(&self, handle: u64, query: &str) -> Option<&str> {
190 if let Some(bucket) = self.buckets.get(&handle) {
191 return bucket.iter().map(String::as_str).find(|&s| s == query);
192 }
193 self.map
194 .get(&handle)
195 .map(String::as_str)
196 .filter(|&s| s == query)
197 }
198}
199
200#[cfg(test)]
201mod tests {
202 use super::*;
203
204 #[test]
205 fn test_60bit_truncation() {
206 let uri = "https://mediaprophet.github.io/qualiaDB/user/123";
207 let token = generate_60bit_token(uri.as_bytes());
208
209 // Ensure the top 4 bits are strictly 0 (Lexicon ID datatype tag)
210 assert_eq!(token >> 60, 0, "Token spilled over 60 bits");
211 }
212
213 /// Regression guard for the hash-space unification (task #14): the compile-time
214 /// `q_hash` and the runtime `generate_60bit_token` MUST be one identity space,
215 /// or compile-time-baked URIs (deontic/values/MCP) won't join runtime-parsed/
216 /// ingested URIs (turtle/SPARQL/SHACL/corpus). If this ever fails, the two
217 /// drifted apart again — fix the divergence, don't relax the test.
218 #[test]
219 fn q_hash_and_generate_60bit_token_are_one_space() {
220 for s in [
221 "",
222 "Bob",
223 "http://www.w3.org/ns/shacl#maxInclusive",
224 "https://ns.webcivics.net/values/NaturalPerson",
225 "q42:Principal",
226 "naïve—prière—母語", // non-ASCII must agree too
227 ] {
228 assert_eq!(
229 crate::q_hash(s),
230 generate_60bit_token(s.as_bytes()),
231 "q_hash and generate_60bit_token diverged for {s:?}",
232 );
233 // Both are pure 60-bit identifiers: top 4 bits reserved for the tag overlay.
234 assert_eq!(
235 crate::q_hash(s) >> 60,
236 0,
237 "q_hash spilled past 60 bits for {s:?}"
238 );
239 }
240 }
241
242 #[test]
243 fn interner_detects_collision_and_keeps_both_values() {
244 let mut lx = LexiconInterner::new();
245 // Force a handle collision (same handle, different values) by interning explicit
246 // handles — a natural 60-bit FNV collision is infeasible to find in a test.
247 let h = 0x0ABC_DEF0_1234_5678;
248 assert_eq!(lx.intern(h, "alpha"), Intern::New);
249 assert_eq!(lx.intern(h, "alpha"), Intern::Seen); // idempotent
250 assert_eq!(lx.intern(h, "beta"), Intern::Collision); // genuine collision
251 assert!(lx.is_collision(h));
252
253 // Both values survive and are recoverable via the disambiguating resolve —
254 // unlike a plain HashMap, where "alpha" would have been silently overwritten.
255 assert_eq!(lx.resolve_value(h, "alpha"), Some("alpha"));
256 assert_eq!(lx.resolve_value(h, "beta"), Some("beta"));
257 assert_eq!(lx.resolve_value(h, "gamma"), None);
258
259 // Bare resolve refuses to guess on an ambiguous handle (no silent wrong answer).
260 assert_eq!(lx.resolve(h), None);
261 }
262
263 #[test]
264 fn interner_collision_free_is_the_fast_single_value_path() {
265 let mut lx = LexiconInterner::new();
266 let iri = "https://ns.webcivics.net/values/State";
267 assert_eq!(lx.intern_str(iri), Intern::New);
268 assert_eq!(lx.intern_str(iri), Intern::Seen);
269
270 // q_hash == generate_60bit_token (post-#14), so this is the interned handle.
271 let h = crate::q_hash(iri);
272 assert!(!lx.is_collision(h));
273 // Collision-free: bare resolve returns the single value directly, no comparison.
274 assert_eq!(lx.resolve(h), Some(iri));
275 }
276
277 #[test]
278 fn test_determinism() {
279 let uri = "qualia:guardian";
280 let t1 = generate_60bit_token(uri.as_bytes());
281 let t2 = generate_60bit_token(uri.as_bytes());
282 assert_eq!(t1, t2, "Tokens are not deterministic");
283 }
284
285 #[test]
286 fn test_linguistic_plurality() {
287 let lexicon = LexiconManager::new();
288
289 // Simulating a written concept
290 let written = SemanticModality::Text("peace_infrastructure");
291 let t1 = lexicon.tokenize_modal(&written);
292
293 // Simulating the exact same concept represented as a cryptographic audio hash of a spoken prayer
294 let audio_hash = vec![0x1a, 0x2b, 0x3c, 0x4d, 0x5e];
295 let oral = SemanticModality::AudioHash(&audio_hash);
296 let t2 = lexicon.tokenize_modal(&oral);
297
298 // Simulating a ceremonial SVG file representation
299 let svg_bytes = b"<svg>Heraldry</svg>";
300 let visual = SemanticModality::CeremonialVisual(svg_bytes);
301 let t3 = lexicon.tokenize_modal(&visual);
302
303 // Prove that the database treats all modalities as valid 60-bit structural Quins
304 assert!(t1 <= 0x0FFF_FFFF_FFFF_FFFF);
305 assert!(t2 <= 0x0FFF_FFFF_FFFF_FFFF);
306 assert!(t3 <= 0x0FFF_FFFF_FFFF_FFFF);
307
308 assert_ne!(t1, t2);
309 assert_ne!(t1, t3);
310
311 // Test non-identical bytes produce different hashes
312 let altered_audio = vec![0x1a, 0x2b, 0x3c, 0x4d, 0x5f];
313 let altered_oral = SemanticModality::AudioHash(&altered_audio);
314 let t4 = lexicon.tokenize_modal(&altered_oral);
315
316 assert_ne!(t2, t4);
317 }
318
319 #[test]
320 fn test_generalized_multimodal_plurality() {
321 let lexicon = LexiconManager::new();
322
323 let spatial = SemanticModality::Spatial3D(b"stl_mesh_data_3d");
324 let bio = SemanticModality::Biosignal(b"rppg_waveform_data");
325 let chem = SemanticModality::Molecular(b"CCO_ethanol_smiles");
326 let qphys = SemanticModality::QuantumPhysical(b"state_vector_psi");
327 let trace = SemanticModality::SpatioTemporalTrace(b"ltl_trace_ltl");
328 let custom = SemanticModality::Custom {
329 tag: 0x42,
330 bytes: b"custom_payload",
331 };
332
333 for m in [&spatial, &bio, &chem, &qphys, &trace, &custom] {
334 let tok = lexicon.tokenize_modal(m);
335 assert!(tok <= 0x0FFF_FFFF_FFFF_FFFF, "token within 60-bit bound");
336 assert_ne!(tok, 0);
337 }
338 }
339}