1use std::collections::HashMap;
4#[cfg(not(target_arch = "wasm32"))]
5use std::fs::File;
6#[cfg(not(target_arch = "wasm32"))]
7use std::io::Read;
8#[cfg(not(target_arch = "wasm32"))]
9use std::path::Path;
10
11#[cfg(not(target_arch = "wasm32"))]
12use memmap2::Mmap;
13
14pub const LEX_MAGIC: [u8; 8] = *b"Q42LEX\0\0";
15const MAGIC: &[u8; 8] = &LEX_MAGIC;
16pub const LEX_HEADER_SIZE: usize = 32;
17const HEADER_SIZE: usize = LEX_HEADER_SIZE;
18pub const INDEX_ENTRY_SIZE: usize = 16;
19pub const LEX_VERSION_PAGED: u64 = 2;
20pub const LEX_VERSION_V4: u64 = 4;
24pub const PAGED_DIRECTORY_HEADER_SIZE: usize = 8;
25pub const PAGED_DIRECTORY_ENTRY_SIZE: usize = 32;
26pub const PAGED_PAGE_HEADER_SIZE: usize = 16;
27pub const DEFAULT_LEX_PAGE_ENTRIES: usize = 4_096;
31
32pub const LEX_TAG_STRING: u8 = 0x01; const LEX_TAG_EMBEDDED: u8 = 0x02; const LEX_TAG_WEBIZEN: u8 = 0x03; pub const LEX_TAG_NAMESPACED: u8 = 0x04;
39
40pub fn serialize_string_lexicon(entries: &HashMap<u64, String>) -> Result<Vec<u8>, LexError> {
56 let mut sorted: Vec<(&u64, &String)> = entries.iter().collect();
57 sorted.sort_unstable_by_key(|(h, _)| **h);
58 let entry_count = sorted.len() as u64;
59 let strings_offset = HEADER_SIZE as u64 + entry_count * INDEX_ENTRY_SIZE as u64;
60
61 let mut index: Vec<u8> = Vec::with_capacity(sorted.len() * INDEX_ENTRY_SIZE);
62 let mut blob: Vec<u8> = Vec::new();
63 for (hash, text) in &sorted {
64 let str_off = blob.len() as u64;
65 if text.len() > u16::MAX as usize {
66 return Err(LexError::TermTooLong);
67 }
68 blob.push(LEX_TAG_STRING);
69 blob.extend_from_slice(&(text.len() as u16).to_le_bytes());
70 blob.extend_from_slice(text.as_bytes());
71 index.extend_from_slice(&hash.to_le_bytes());
72 index.extend_from_slice(&str_off.to_le_bytes());
73 }
74
75 let mut out = Vec::with_capacity(HEADER_SIZE + index.len() + blob.len());
76 out.extend_from_slice(MAGIC);
77 out.extend_from_slice(&entry_count.to_le_bytes());
78 out.extend_from_slice(&strings_offset.to_le_bytes());
79 out.extend_from_slice(&1u64.to_le_bytes()); out.extend_from_slice(&index);
81 out.extend_from_slice(&blob);
82 Ok(out)
83}
84
85pub fn serialize_paged_string_lexicon(
94 entries: &HashMap<u64, String>,
95 page_entries: usize,
96) -> Result<Vec<u8>, LexError> {
97 if page_entries == 0 {
98 return Err(LexError::BadIndex);
99 }
100 let mut sorted: Vec<(&u64, &String)> = entries.iter().collect();
101 sorted.sort_unstable_by_key(|(hash, _)| **hash);
102 let page_count = (sorted.len() + page_entries - 1) / page_entries;
103 let directory_len = PAGED_DIRECTORY_HEADER_SIZE
104 .checked_add(
105 page_count
106 .checked_mul(PAGED_DIRECTORY_ENTRY_SIZE)
107 .ok_or(LexError::Truncated)?,
108 )
109 .ok_or(LexError::Truncated)?;
110 let mut out = Vec::with_capacity(HEADER_SIZE + directory_len + entries.len() * 24);
111 out.extend_from_slice(MAGIC);
112 out.extend_from_slice(&(sorted.len() as u64).to_le_bytes());
113 out.extend_from_slice(&(HEADER_SIZE as u64).to_le_bytes());
114 out.extend_from_slice(&LEX_VERSION_PAGED.to_le_bytes());
115 out.extend_from_slice(&(page_count as u64).to_le_bytes());
116 out.resize(HEADER_SIZE + directory_len, 0);
117
118 for (page_index, chunk) in sorted.chunks(page_entries).enumerate() {
119 let page_offset = out.len() as u64;
120 let page_start = out.len();
121 out.extend_from_slice(&(chunk.len() as u32).to_le_bytes());
122 out.extend_from_slice(&0u32.to_le_bytes());
123 let blob_offset = PAGED_PAGE_HEADER_SIZE
124 .checked_add(
125 chunk
126 .len()
127 .checked_mul(INDEX_ENTRY_SIZE)
128 .ok_or(LexError::Truncated)?,
129 )
130 .ok_or(LexError::Truncated)?;
131 out.extend_from_slice(&(blob_offset as u64).to_le_bytes());
132 let index_start = out.len();
133 out.resize(index_start + chunk.len() * INDEX_ENTRY_SIZE, 0);
134 for (entry_index, (hash, text)) in chunk.iter().enumerate() {
135 if text.len() > u16::MAX as usize {
136 return Err(LexError::TermTooLong);
137 }
138 let relative = (out.len() - page_start - blob_offset) as u64;
139 out.push(LEX_TAG_STRING);
140 out.extend_from_slice(&(text.len() as u16).to_le_bytes());
141 out.extend_from_slice(text.as_bytes());
142 let index_offset = index_start + entry_index * INDEX_ENTRY_SIZE;
143 out[index_offset..index_offset + 8].copy_from_slice(&hash.to_le_bytes());
144 out[index_offset + 8..index_offset + 16].copy_from_slice(&relative.to_le_bytes());
145 }
146 let page_length = (out.len() - page_start) as u64;
147 let directory =
148 HEADER_SIZE + PAGED_DIRECTORY_HEADER_SIZE + page_index * PAGED_DIRECTORY_ENTRY_SIZE;
149 out[directory..directory + 8].copy_from_slice(&chunk[0].0.to_le_bytes());
150 out[directory + 8..directory + 16].copy_from_slice(&page_offset.to_le_bytes());
151 out[directory + 16..directory + 24].copy_from_slice(&page_length.to_le_bytes());
152 out[directory + 24..directory + 28].copy_from_slice(&(chunk.len() as u32).to_le_bytes());
153 }
154 Ok(out)
155}
156
157pub enum LexiconKey<'a> {
160 Str(&'a str),
162 Triple(&'a [u64; 3]),
164}
165
166#[derive(Debug, Clone)]
168pub enum LexiconEntry {
169 String(String),
171 EmbeddedTriple([u64; 3]),
173 Webizen(String),
175}
176
177#[derive(Debug, Default)]
179pub struct Q42Lexicon {
180 pub entries: HashMap<u64, String>,
181}
182
183#[derive(Debug, Clone, Copy, PartialEq, Eq)]
184pub enum LexError {
185 InvalidMagic,
186 Truncated,
187 BadStringOffset,
188 BadIndex,
189 BadEntry,
190 InvalidUtf8,
191 TermTooLong,
192}
193
194#[derive(Debug, Clone, Copy)]
196pub struct Q42LexMmap<'a> {
197 data: &'a [u8],
198 entry_count: usize,
199 strings_offset: usize,
200 format_version: u64,
201 page_count: usize,
202}
203
204impl<'a> Q42LexMmap<'a> {
205 pub fn from_bytes(data: &'a [u8]) -> Result<Self, LexError> {
207 if data.len() < HEADER_SIZE {
208 return Err(LexError::Truncated);
209 }
210 if data[0..8] != *MAGIC {
211 return Err(LexError::InvalidMagic);
212 }
213 let format_version = u64::from_le_bytes(data[24..32].try_into().unwrap());
214 let entry_count = usize::try_from(u64::from_le_bytes(data[8..16].try_into().unwrap()))
215 .map_err(|_| LexError::Truncated)?;
216 let strings_offset = usize::try_from(u64::from_le_bytes(data[16..24].try_into().unwrap()))
217 .map_err(|_| LexError::Truncated)?;
218 let index_end = HEADER_SIZE
219 .checked_add(
220 entry_count
221 .checked_mul(INDEX_ENTRY_SIZE)
222 .ok_or(LexError::Truncated)?,
223 )
224 .ok_or(LexError::Truncated)?;
225 let (page_count, flat) = match format_version {
226 1 => (0, true),
227 LEX_VERSION_PAGED | LEX_VERSION_V4 => {
228 let page_count_end = strings_offset
229 .checked_add(PAGED_DIRECTORY_HEADER_SIZE)
230 .ok_or(LexError::Truncated)?;
231 if page_count_end > data.len() {
232 return Err(LexError::Truncated);
233 }
234 let page_count = usize::try_from(u64::from_le_bytes(
235 data[strings_offset..page_count_end].try_into().unwrap(),
236 ))
237 .map_err(|_| LexError::Truncated)?;
238 let directory_end = page_count_end
239 .checked_add(
240 page_count
241 .checked_mul(PAGED_DIRECTORY_ENTRY_SIZE)
242 .ok_or(LexError::Truncated)?,
243 )
244 .ok_or(LexError::Truncated)?;
245 if strings_offset != HEADER_SIZE || directory_end > data.len() {
246 return Err(LexError::Truncated);
247 }
248 (page_count, false)
249 }
250 _ => return Err(LexError::BadIndex),
251 };
252 if flat && (index_end > data.len() || strings_offset != index_end) {
253 return Err(LexError::Truncated);
254 }
255 let view = Self {
256 data,
257 entry_count,
258 strings_offset,
259 format_version,
260 page_count,
261 };
262 view.validate_entries()?;
263 Ok(view)
264 }
265
266 #[inline]
267 pub fn entry_count(&self) -> usize {
269 self.entry_count
270 }
271
272 #[inline]
274 fn is_paged(&self) -> bool {
275 self.format_version == LEX_VERSION_PAGED || self.format_version == LEX_VERSION_V4
276 }
277
278 #[inline]
282 pub fn lex_format_version(&self) -> u64 {
283 self.format_version
284 }
285
286 pub fn hash_at(&self, i: usize) -> Option<u64> {
290 if i >= self.entry_count {
291 return None;
292 }
293 if self.is_paged() {
294 let mut base = 0usize;
295 for page in 0..self.page_count {
296 let (_, offset, _, count) = self.page_directory_entry(page)?;
297 if i < base + count {
298 let entry = offset + PAGED_PAGE_HEADER_SIZE + (i - base) * INDEX_ENTRY_SIZE;
299 return Some(u64::from_le_bytes(
300 self.data.get(entry..entry + 8)?.try_into().ok()?,
301 ));
302 }
303 base += count;
304 }
305 return None;
306 }
307 let off = HEADER_SIZE + i * INDEX_ENTRY_SIZE;
308 Some(u64::from_le_bytes(
309 self.data.get(off..off + 8)?.try_into().ok()?,
310 ))
311 }
312
313 pub fn lookup_hash(&self, hash: u64) -> Option<&'a str> {
319 if self.is_paged() {
320 return self.lookup_hash_paged(hash);
321 }
322 let mut lo = 0usize;
323 let mut hi = self.entry_count;
324 while lo < hi {
325 let mid = lo + (hi - lo) / 2;
326 let off = HEADER_SIZE + mid * INDEX_ENTRY_SIZE;
327 let entry_hash = u64::from_le_bytes(self.data[off..off + 8].try_into().ok()?);
328 match entry_hash.cmp(&hash) {
329 std::cmp::Ordering::Less => lo = mid + 1,
330 std::cmp::Ordering::Greater => hi = mid,
331 std::cmp::Ordering::Equal => {
332 let str_off =
333 u64::from_le_bytes(self.data[off + 8..off + 16].try_into().ok()?) as usize;
334 return Self::read_string_at(self.data, self.strings_offset, str_off);
335 }
336 }
337 }
338 None
339 }
340
341 pub fn string_at(&self, i: usize) -> Option<&'a str> {
346 if i >= self.entry_count {
347 return None;
348 }
349 if self.is_paged() {
350 let mut base = 0usize;
351 for page in 0..self.page_count {
352 let (_, offset, length, count) = self.page_directory_entry(page)?;
353 if i < base + count {
354 return self.page_string_at(offset, length, i - base);
355 }
356 base += count;
357 }
358 return None;
359 }
360 let off = HEADER_SIZE + i * INDEX_ENTRY_SIZE;
361 let str_off = u64::from_le_bytes(self.data[off + 8..off + 16].try_into().ok()?) as usize;
362 Self::read_string_at(self.data, self.strings_offset, str_off)
363 }
364
365 pub fn lookup_parts(&self, hash: u64) -> Option<(&'a str, &'a str)> {
373 if !self.is_paged() {
374 return self.lookup_hash(hash).map(|s| ("", s));
375 }
376 let (offset, length, index) = self.paged_entry_slot(hash)?;
377 self.page_entry_parts(offset, length, index)
378 }
379
380 pub fn string_parts_at(&self, i: usize) -> Option<(&'a str, &'a str)> {
384 if i >= self.entry_count {
385 return None;
386 }
387 if self.is_paged() {
388 let mut base = 0usize;
389 for page in 0..self.page_count {
390 let (_, offset, length, count) = self.page_directory_entry(page)?;
391 if i < base + count {
392 return self.page_entry_parts(offset, length, i - base);
393 }
394 base += count;
395 }
396 return None;
397 }
398 let off = HEADER_SIZE + i * INDEX_ENTRY_SIZE;
399 let str_off = u64::from_le_bytes(self.data[off + 8..off + 16].try_into().ok()?) as usize;
400 Self::read_string_at(self.data, self.strings_offset, str_off).map(|s| ("", s))
401 }
402
403 pub fn lookup_owned(&self, hash: u64) -> Option<String> {
405 let (ns, local) = self.lookup_parts(hash)?;
406 let mut out = String::with_capacity(ns.len() + local.len());
407 out.push_str(ns);
408 out.push_str(local);
409 Some(out)
410 }
411
412 pub fn resolve_term_into(&self, hash: u64, out: &mut String) -> bool {
415 match self.lookup_parts(hash) {
416 Some((ns, local)) => {
417 out.clear();
418 out.push_str(ns);
419 out.push_str(local);
420 true
421 }
422 None => false,
423 }
424 }
425
426 pub fn contains(&self, hash: u64) -> bool {
428 self.lookup_parts(hash).is_some()
429 }
430
431 fn validate_entries(&self) -> Result<(), LexError> {
435 if self.is_paged() {
436 return self.validate_paged_entries();
437 }
438 let mut previous_hash = None;
439 for i in 0..self.entry_count {
440 let off = HEADER_SIZE + i * INDEX_ENTRY_SIZE;
441 let hash = u64::from_le_bytes(
442 self.data[off..off + 8]
443 .try_into()
444 .map_err(|_| LexError::BadIndex)?,
445 );
446 if let Some(previous) = previous_hash {
447 if hash <= previous {
448 return Err(LexError::BadIndex);
449 }
450 }
451 previous_hash = Some(hash);
452 let rel_off = usize::try_from(u64::from_le_bytes(
453 self.data[off + 8..off + 16]
454 .try_into()
455 .map_err(|_| LexError::BadIndex)?,
456 ))
457 .map_err(|_| LexError::BadStringOffset)?;
458 let start = self
459 .strings_offset
460 .checked_add(rel_off)
461 .ok_or(LexError::BadStringOffset)?;
462 let tag = *self.data.get(start).ok_or(LexError::BadStringOffset)?;
463 match tag {
464 LEX_TAG_STRING | LEX_TAG_WEBIZEN => {
465 let length_end = start.checked_add(3).ok_or(LexError::BadEntry)?;
466 let length_bytes = self
467 .data
468 .get(start + 1..length_end)
469 .ok_or(LexError::BadEntry)?;
470 let length = u16::from_le_bytes(
471 length_bytes.try_into().map_err(|_| LexError::BadEntry)?,
472 ) as usize;
473 let text_start = length_end;
474 let text_end = text_start.checked_add(length).ok_or(LexError::BadEntry)?;
475 let text = self
476 .data
477 .get(text_start..text_end)
478 .ok_or(LexError::BadEntry)?;
479 std::str::from_utf8(text).map_err(|_| LexError::InvalidUtf8)?;
480 }
481 LEX_TAG_EMBEDDED => {
482 let end = start.checked_add(25).ok_or(LexError::BadEntry)?;
483 self.data.get(start..end).ok_or(LexError::BadEntry)?;
484 }
485 _ => return Err(LexError::BadEntry),
486 }
487 }
488 Ok(())
489 }
490
491 fn page_directory_entry(&self, index: usize) -> Option<(u64, usize, usize, usize)> {
492 if index >= self.page_count {
493 return None;
494 }
495 let offset =
496 self.strings_offset + PAGED_DIRECTORY_HEADER_SIZE + index * PAGED_DIRECTORY_ENTRY_SIZE;
497 let first_hash = u64::from_le_bytes(self.data.get(offset..offset + 8)?.try_into().ok()?);
498 let page_offset = usize::try_from(u64::from_le_bytes(
499 self.data.get(offset + 8..offset + 16)?.try_into().ok()?,
500 ))
501 .ok()?;
502 let page_length = usize::try_from(u64::from_le_bytes(
503 self.data.get(offset + 16..offset + 24)?.try_into().ok()?,
504 ))
505 .ok()?;
506 let count =
507 u32::from_le_bytes(self.data.get(offset + 24..offset + 28)?.try_into().ok()?) as usize;
508 Some((first_hash, page_offset, page_length, count))
509 }
510
511 fn paged_entry_slot(&self, hash: u64) -> Option<(usize, usize, usize)> {
514 let mut lo = 0usize;
515 let mut hi = self.page_count;
516 while lo < hi {
517 let mid = lo + (hi - lo) / 2;
518 if self.page_directory_entry(mid)?.0 <= hash {
519 lo = mid + 1;
520 } else {
521 hi = mid;
522 }
523 }
524 let page = lo.checked_sub(1)?;
525 let (_, offset, length, count) = self.page_directory_entry(page)?;
526 let mut left = 0usize;
527 let mut right = count;
528 while left < right {
529 let mid = left + (right - left) / 2;
530 let entry = offset + PAGED_PAGE_HEADER_SIZE + mid * INDEX_ENTRY_SIZE;
531 let entry_hash = u64::from_le_bytes(self.data.get(entry..entry + 8)?.try_into().ok()?);
532 match entry_hash.cmp(&hash) {
533 std::cmp::Ordering::Less => left = mid + 1,
534 std::cmp::Ordering::Greater => right = mid,
535 std::cmp::Ordering::Equal => return Some((offset, length, mid)),
536 }
537 }
538 None
539 }
540
541 fn lookup_hash_paged(&self, hash: u64) -> Option<&'a str> {
542 let (offset, length, index) = self.paged_entry_slot(hash)?;
543 self.page_string_at(offset, length, index)
544 }
545
546 fn page_string_at(
547 &self,
548 page_offset: usize,
549 page_length: usize,
550 index: usize,
551 ) -> Option<&'a str> {
552 let page_end = page_offset.checked_add(page_length)?;
553 let count = u32::from_le_bytes(
554 self.data
555 .get(page_offset..page_offset + 4)?
556 .try_into()
557 .ok()?,
558 ) as usize;
559 let blob_offset = usize::try_from(u64::from_le_bytes(
560 self.data
561 .get(page_offset + 8..page_offset + 16)?
562 .try_into()
563 .ok()?,
564 ))
565 .ok()?;
566 if index >= count {
567 return None;
568 }
569 let entry = page_offset + PAGED_PAGE_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
570 let relative = usize::try_from(u64::from_le_bytes(
571 self.data.get(entry + 8..entry + 16)?.try_into().ok()?,
572 ))
573 .ok()?;
574 let start = page_offset
575 .checked_add(blob_offset)?
576 .checked_add(relative)?;
577 if start + 3 > page_end || self.data[start] != LEX_TAG_STRING {
578 return None;
579 }
580 let length = u16::from_le_bytes(self.data[start + 1..start + 3].try_into().ok()?) as usize;
581 let end = start.checked_add(3)?.checked_add(length)?;
582 if end > page_end {
583 return None;
584 }
585 std::str::from_utf8(&self.data[start + 3..end]).ok()
586 }
587
588 fn page_ns_count(&self, page_offset: usize) -> Option<usize> {
591 let bytes = self.data.get(page_offset + 4..page_offset + 8)?;
592 Some(u32::from_le_bytes(bytes.try_into().ok()?) as usize)
593 }
594
595 fn page_ns_at(&self, page_offset: usize, page_length: usize, ns_id: usize) -> Option<&'a str> {
599 if self.format_version != LEX_VERSION_V4 {
600 return None;
601 }
602 let ns_count = self.page_ns_count(page_offset)?;
603 if ns_id >= ns_count {
604 return None;
605 }
606 let page_end = page_offset.checked_add(page_length)?;
607 let blob_offset = usize::try_from(u64::from_le_bytes(
608 self.data
609 .get(page_offset + 8..page_offset + 16)?
610 .try_into()
611 .ok()?,
612 ))
613 .ok()?;
614 let mut cursor = page_offset.checked_add(blob_offset)?;
616 for _ in 0..ns_id {
617 let len =
618 u16::from_le_bytes(self.data.get(cursor..cursor + 2)?.try_into().ok()?) as usize;
619 let end = cursor.checked_add(2)?.checked_add(len)?;
620 if end > page_end {
621 return None;
622 }
623 cursor = end;
624 }
625 let len = u16::from_le_bytes(self.data.get(cursor..cursor + 2)?.try_into().ok()?) as usize;
626 let start = cursor.checked_add(2)?;
627 let end = start.checked_add(len)?;
628 if end > page_end {
629 return None;
630 }
631 std::str::from_utf8(self.data.get(start..end)?).ok()
632 }
633
634 fn page_entry_parts(
638 &self,
639 page_offset: usize,
640 page_length: usize,
641 index: usize,
642 ) -> Option<(&'a str, &'a str)> {
643 let page_end = page_offset.checked_add(page_length)?;
644 let count = u32::from_le_bytes(
645 self.data
646 .get(page_offset..page_offset + 4)?
647 .try_into()
648 .ok()?,
649 ) as usize;
650 let blob_offset = usize::try_from(u64::from_le_bytes(
651 self.data
652 .get(page_offset + 8..page_offset + 16)?
653 .try_into()
654 .ok()?,
655 ))
656 .ok()?;
657 if index >= count {
658 return None;
659 }
660 let entry = page_offset + PAGED_PAGE_HEADER_SIZE + index * INDEX_ENTRY_SIZE;
661 let relative = usize::try_from(u64::from_le_bytes(
662 self.data.get(entry + 8..entry + 16)?.try_into().ok()?,
663 ))
664 .ok()?;
665 let start = page_offset
666 .checked_add(blob_offset)?
667 .checked_add(relative)?;
668 if start >= page_end {
669 return None;
670 }
671 match self.data[start] {
672 LEX_TAG_STRING => {
673 let len = u16::from_le_bytes(self.data.get(start + 1..start + 3)?.try_into().ok()?)
674 as usize;
675 let text_start = start + 3;
676 let end = text_start.checked_add(len)?;
677 if end > page_end {
678 return None;
679 }
680 Some((
681 "",
682 std::str::from_utf8(self.data.get(text_start..end)?).ok()?,
683 ))
684 }
685 LEX_TAG_NAMESPACED => {
686 if self.format_version != LEX_VERSION_V4 {
687 return None;
688 }
689 let ns_id =
690 u16::from_le_bytes(self.data.get(start + 1..start + 3)?.try_into().ok()?)
691 as usize;
692 let len = u16::from_le_bytes(self.data.get(start + 3..start + 5)?.try_into().ok()?)
693 as usize;
694 let text_start = start + 5;
695 let end = text_start.checked_add(len)?;
696 if end > page_end {
697 return None;
698 }
699 let local = std::str::from_utf8(self.data.get(text_start..end)?).ok()?;
700 let ns = self.page_ns_at(page_offset, page_length, ns_id)?;
701 Some((ns, local))
702 }
703 _ => None,
704 }
705 }
706
707 fn validate_paged_entries(&self) -> Result<(), LexError> {
708 let mut total = 0usize;
709 let mut previous = None;
710 for page in 0..self.page_count {
711 let (first, offset, length, count) =
712 self.page_directory_entry(page).ok_or(LexError::BadIndex)?;
713 if count == 0
714 || offset < HEADER_SIZE
715 || offset
716 .checked_add(length)
717 .is_none_or(|end| end > self.data.len())
718 {
719 return Err(LexError::BadIndex);
720 }
721 if previous.is_some_and(|value| first <= value) {
722 return Err(LexError::BadIndex);
723 }
724 let actual_count =
725 u32::from_le_bytes(self.data[offset..offset + 4].try_into().unwrap()) as usize;
726 let blob = usize::try_from(u64::from_le_bytes(
727 self.data[offset + 8..offset + 16].try_into().unwrap(),
728 ))
729 .map_err(|_| LexError::BadIndex)?;
730 if actual_count != count
731 || blob != PAGED_PAGE_HEADER_SIZE + count * INDEX_ENTRY_SIZE
732 || blob > length
733 {
734 return Err(LexError::BadIndex);
735 }
736 for item in 0..count {
737 let entry = offset + PAGED_PAGE_HEADER_SIZE + item * INDEX_ENTRY_SIZE;
738 let hash = u64::from_le_bytes(self.data[entry..entry + 8].try_into().unwrap());
739 if item == 0 && hash != first {
740 return Err(LexError::BadIndex);
741 }
742 if previous.is_some_and(|value| hash <= value) {
743 return Err(LexError::BadIndex);
744 }
745 previous = Some(hash);
746 if self.page_entry_parts(offset, length, item).is_none() {
750 return Err(LexError::BadEntry);
751 }
752 }
753 total = total.checked_add(count).ok_or(LexError::BadIndex)?;
754 }
755 if total != self.entry_count {
756 return Err(LexError::BadIndex);
757 }
758 Ok(())
759 }
760
761 fn read_string_at(data: &[u8], blob_base: usize, rel_off: usize) -> Option<&str> {
762 let start = blob_base.checked_add(rel_off)?;
763 if start.checked_add(3)? > data.len() {
764 return None;
765 }
766 if data[start] != LEX_TAG_STRING {
768 return None;
769 }
770 let len = u16::from_le_bytes(data[start + 1..start + 3].try_into().ok()?) as usize;
771 let text_start = start + 3;
772 let text_end = text_start.checked_add(len)?;
773 if text_end > data.len() {
774 return None;
775 }
776 std::str::from_utf8(&data[text_start..text_end]).ok()
777 }
778
779 pub fn lookup_embedded_triple(&self, hash: u64) -> Option<[u64; 3]> {
784 if self.is_paged() {
785 return None;
786 }
787 let mut lo = 0usize;
788 let mut hi = self.entry_count;
789 while lo < hi {
790 let mid = lo + (hi - lo) / 2;
791 let off = HEADER_SIZE + mid * INDEX_ENTRY_SIZE;
792 let entry_hash = u64::from_le_bytes(self.data[off..off + 8].try_into().ok()?);
793 match entry_hash.cmp(&hash) {
794 std::cmp::Ordering::Less => lo = mid + 1,
795 std::cmp::Ordering::Greater => hi = mid,
796 std::cmp::Ordering::Equal => {
797 let str_off =
798 u64::from_le_bytes(self.data[off + 8..off + 16].try_into().ok()?) as usize;
799 return Self::read_embedded_triple_at(self.data, self.strings_offset, str_off)
800 .copied();
801 }
802 }
803 }
804 None
805 }
806
807 pub fn lookup_webizen_identity(&self, hash: u64) -> Option<&'a str> {
809 if self.is_paged() {
810 return None;
811 }
812 let mut lo = 0usize;
813 let mut hi = self.entry_count;
814 while lo < hi {
815 let mid = lo + (hi - lo) / 2;
816 let off = HEADER_SIZE + mid * INDEX_ENTRY_SIZE;
817 let entry_hash = u64::from_le_bytes(self.data[off..off + 8].try_into().ok()?);
818 match entry_hash.cmp(&hash) {
819 std::cmp::Ordering::Less => lo = mid + 1,
820 std::cmp::Ordering::Greater => hi = mid,
821 std::cmp::Ordering::Equal => {
822 let str_off =
823 u64::from_le_bytes(self.data[off + 8..off + 16].try_into().ok()?) as usize;
824 return Self::read_webizen_at(self.data, self.strings_offset, str_off);
825 }
826 }
827 }
828 None
829 }
830
831 fn read_webizen_at(data: &[u8], blob_base: usize, rel_off: usize) -> Option<&str> {
832 let start = blob_base.checked_add(rel_off)?;
833 if start.checked_add(3)? > data.len() || data[start] != LEX_TAG_WEBIZEN {
834 return None;
835 }
836 let len = u16::from_le_bytes(data[start + 1..start + 3].try_into().ok()?) as usize;
837 let text_start = start + 3;
838 let text_end = text_start.checked_add(len)?;
839 if text_end > data.len() {
840 return None;
841 }
842 std::str::from_utf8(&data[text_start..text_end]).ok()
843 }
844
845 fn read_embedded_triple_at(data: &[u8], blob_base: usize, rel_off: usize) -> Option<&[u64; 3]> {
849 let start = blob_base.saturating_add(rel_off);
850 if start + 1 + 24 > data.len() {
851 return None;
852 }
853 if data[start] != LEX_TAG_EMBEDDED {
855 return None;
856 }
857 let triple_start = start + 1;
859 let bytes = &data[triple_start..triple_start + 24];
860 let ptr = bytes.as_ptr() as *const [u64; 3];
861 unsafe { Some(&*ptr) }
862 }
863}
864
865#[cfg(not(target_arch = "wasm32"))]
867pub struct Q42LexFile {
868 mmap: Mmap,
869}
870
871#[cfg(not(target_arch = "wasm32"))]
872impl Q42LexFile {
873 pub fn open(path: &Path) -> std::io::Result<Self> {
875 let file = File::open(path)?;
876 let mmap = unsafe { Mmap::map(&file)? };
877 Q42LexMmap::from_bytes(&mmap)
878 .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{e:?}")))?;
879 Ok(Self { mmap })
880 }
881
882 #[inline]
883 pub fn view(&self) -> Q42LexMmap<'_> {
885 Q42LexMmap::from_bytes(&self.mmap).expect("validated at open")
886 }
887}
888
889#[cfg(target_arch = "wasm32")]
893pub struct Q42LexFile;
894
895#[cfg(target_arch = "wasm32")]
896impl Q42LexFile {
897 pub fn open(_path: &std::path::Path) -> std::io::Result<Self> {
899 Err(std::io::Error::new(
900 std::io::ErrorKind::Unsupported,
901 "Q42LexFile is not supported on wasm32; read Q42LEX bytes with Q42LexMmap",
902 ))
903 }
904
905 #[inline]
906 pub fn view(&self) -> Q42LexMmap<'_> {
908 Q42LexMmap {
909 data: &[],
910 entry_count: 0,
911 strings_offset: 0,
912 format_version: 0,
913 page_count: 0,
914 }
915 }
916}
917
918impl Q42Lexicon {
919 #[cfg(not(target_arch = "wasm32"))]
921 pub fn load_for_q42(q42_path: &Path) -> std::io::Result<Self> {
922 if crate::q42_volume::is_unified_volume(q42_path)? {
923 let vol = crate::q42_volume::Q42Volume::open(q42_path)?;
924 if vol.volume_manifest()?.is_some() {
925 let set = crate::q42_volume::Q42VolumeSet::open_root(q42_path)?;
926 let mut lexicon = Self::load_from_lex_bytes(set.root().lex_bytes())?;
927 for shard in set.lexicon_segments() {
928 let entries = Self::load_from_lex_bytes(shard.lex_bytes())?;
929 lexicon.entries.extend(entries.entries);
930 }
931 return Ok(lexicon);
932 }
933 return Self::load_from_lex_bytes(vol.lex_bytes());
934 }
935 let sidecar = q42_path.with_extension("q42.lex");
936 if sidecar.is_file() {
937 return Self::load(&sidecar);
938 }
939 Err(std::io::Error::new(
940 std::io::ErrorKind::NotFound,
941 format!(
942 "no lexicon in {} or sidecar {}",
943 q42_path.display(),
944 sidecar.display()
945 ),
946 ))
947 }
948
949 pub fn load_from_lex_bytes(data: &[u8]) -> std::io::Result<Self> {
951 let view = Q42LexMmap::from_bytes(data)
952 .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{e:?}")))?;
953 let mut entries = HashMap::with_capacity(view.entry_count());
954 for i in 0..view.entry_count() {
955 let Some(hash) = view.hash_at(i) else {
956 break;
957 };
958 if let Some((ns, local)) = view.string_parts_at(i) {
959 let mut text = String::with_capacity(ns.len() + local.len());
960 text.push_str(ns);
961 text.push_str(local);
962 entries.insert(hash, text);
963 }
964 }
965 Ok(Self { entries })
966 }
967
968 #[cfg(not(target_arch = "wasm32"))]
969 pub fn load(path: &Path) -> std::io::Result<Self> {
971 let mut file = File::open(path)?;
972 let mut header = [0u8; 32];
973 file.read_exact(&mut header)?;
974 if header[0..8] != *MAGIC {
975 return Err(std::io::Error::new(
976 std::io::ErrorKind::InvalidData,
977 "invalid Q42LEX magic",
978 ));
979 }
980 let entry_count = u64::from_le_bytes(header[8..16].try_into().unwrap()) as usize;
981 let strings_offset = u64::from_le_bytes(header[16..24].try_into().unwrap()) as usize;
982
983 let mut index_buf = vec![0u8; entry_count * 16];
984 file.read_exact(&mut index_buf)?;
985
986 let mut blob = Vec::new();
987 file.seek(std::io::SeekFrom::Start(strings_offset as u64))?;
988 file.read_to_end(&mut blob)?;
989
990 let mut entries = HashMap::with_capacity(entry_count);
991 for i in 0..entry_count {
992 let off = i * 16;
993 let hash = u64::from_le_bytes(index_buf[off..off + 8].try_into().unwrap());
994 let str_off =
995 u64::from_le_bytes(index_buf[off + 8..off + 16].try_into().unwrap()) as usize;
996 if str_off + 2 > blob.len() {
997 continue;
998 }
999 let len = u16::from_le_bytes(blob[str_off..str_off + 2].try_into().unwrap()) as usize;
1000 let start = str_off + 2;
1001 let end = start.saturating_add(len).min(blob.len());
1002 if let Ok(text) = std::str::from_utf8(&blob[start..end]) {
1003 entries.insert(hash, text.to_string());
1004 }
1005 }
1006
1007 Ok(Self { entries })
1008 }
1009
1010 pub fn lookup(&self, hash: u64) -> Option<&str> {
1012 self.entries.get(&hash).map(|s| s.as_str())
1013 }
1014
1015 pub fn find_literal(&self, needle: &str) -> Option<u64> {
1017 let needle = needle.to_lowercase();
1018 self.entries
1019 .iter()
1020 .find(|(_, v)| v.to_lowercase() == needle)
1021 .map(|(h, _)| *h)
1022 }
1023
1024 pub fn search_contains(&self, sub: &str, limit: usize) -> Vec<(u64, String)> {
1026 let sub = sub.to_lowercase();
1027 let mut out = Vec::new();
1028 for (h, v) in &self.entries {
1029 if v.to_lowercase().contains(&sub) {
1030 out.push((*h, v.clone()));
1031 if out.len() >= limit {
1032 break;
1033 }
1034 }
1035 }
1036 out
1037 }
1038}
1039
1040#[cfg(not(target_arch = "wasm32"))]
1041use std::io::Seek;
1042
1043#[cfg(all(test, not(target_arch = "wasm32")))]
1044mod tests {
1045 use super::*;
1046 use std::io::Write;
1047 use tempfile::NamedTempFile;
1048
1049 fn write_lex_bytes(entries: &[(u64, &str)]) -> Vec<u8> {
1050 let mut sorted: Vec<(u64, &str)> = entries.to_vec();
1051 sorted.sort_unstable_by_key(|(h, _)| *h);
1052 let entry_count = sorted.len() as u64;
1053 let strings_offset = 32 + entry_count * 16;
1054 let mut blob = Vec::new();
1055 let mut index = Vec::new();
1056 for (hash, text) in &sorted {
1057 let str_off = blob.len() as u64;
1058 blob.push(LEX_TAG_STRING);
1060 let b = text.as_bytes();
1061 let len = b.len().min(65535) as u16;
1062 blob.extend_from_slice(&len.to_le_bytes());
1063 blob.extend_from_slice(&b[..len as usize]);
1064 index.extend_from_slice(&hash.to_le_bytes());
1065 index.extend_from_slice(&str_off.to_le_bytes());
1066 }
1067 let mut out = Vec::new();
1068 out.extend_from_slice(MAGIC);
1069 out.extend_from_slice(&entry_count.to_le_bytes());
1070 out.extend_from_slice(&strings_offset.to_le_bytes());
1071 out.extend_from_slice(&1u64.to_le_bytes());
1072 out.extend_from_slice(&index);
1073 out.extend_from_slice(&blob);
1074 out
1075 }
1076
1077 #[test]
1078 fn mmap_lex_binary_search() {
1079 let h1 = crate::q_hash("Patient");
1080 let h2 = crate::q_hash("fever");
1081 let bytes = write_lex_bytes(&[(h1, "Patient"), (h2, "fever")]);
1082 let lex = Q42LexMmap::from_bytes(&bytes).unwrap();
1083 assert_eq!(lex.lookup_hash(h1), Some("Patient"));
1084 assert_eq!(lex.lookup_hash(h2), Some("fever"));
1085 assert_eq!(lex.lookup_hash(0xDEAD), None);
1086 }
1087
1088 #[test]
1093 fn serialize_lexicon_round_trips_unicode() {
1094 let samples = [
1095 "carefully", "välinpitämättömästi", "อย่างสะเพร่า", "不注意に", "بلا مبالاة", "невнимательно", "a definition — with em-dash & 😀", ];
1103 let mut map = HashMap::new();
1104 for s in samples {
1105 map.insert(crate::q_hash(s), s.to_string());
1106 }
1107 let bytes = serialize_string_lexicon(&map).unwrap();
1108 let lex = Q42LexMmap::from_bytes(&bytes).unwrap();
1109 assert_eq!(lex.entry_count(), samples.len());
1110 for s in samples {
1111 assert_eq!(
1112 lex.lookup_hash(crate::q_hash(s)),
1113 Some(s),
1114 "multilingual lexeme must round-trip byte-identical: {s:?}"
1115 );
1116 }
1117 }
1118
1119 #[test]
1122 fn serialize_lexicon_rejects_overlong_term() {
1123 let long: String = "あ".repeat(22_000);
1126 let h = crate::q_hash(&long);
1127 let mut map = HashMap::new();
1128 map.insert(h, long);
1129 assert_eq!(serialize_string_lexicon(&map), Err(LexError::TermTooLong));
1130 }
1131
1132 #[test]
1133 fn paged_lexicon_binary_search_crosses_page_boundaries() {
1134 let mut map = HashMap::new();
1135 for value in 0u64..9 {
1136 map.insert(value * 10 + 3, format!("urn:q42:paged:{value}"));
1137 }
1138 let bytes = serialize_paged_string_lexicon(&map, 2).unwrap();
1139 assert_eq!(
1140 u64::from_le_bytes(bytes[24..32].try_into().unwrap()),
1141 LEX_VERSION_PAGED
1142 );
1143 let view = Q42LexMmap::from_bytes(&bytes).unwrap();
1144 assert_eq!(view.entry_count(), 9);
1145 assert_eq!(view.lookup_hash(3), Some("urn:q42:paged:0"));
1146 assert_eq!(view.lookup_hash(83), Some("urn:q42:paged:8"));
1147 assert_eq!(view.string_at(4), Some("urn:q42:paged:4"));
1148 assert_eq!(view.lookup_hash(4), None);
1149 let cold = Q42Lexicon::load_from_lex_bytes(&bytes).unwrap();
1150 assert_eq!(cold.lookup(43), Some("urn:q42:paged:4"));
1151 }
1152
1153 #[test]
1154 fn mmap_lex_file_roundtrip() {
1155 let h = crate::q_hash("Entity");
1156 let bytes = write_lex_bytes(&[(h, "Entity")]);
1157 let mut tmp = NamedTempFile::new().unwrap();
1158 tmp.write_all(&bytes).unwrap();
1159 let file = Q42LexFile::open(tmp.path()).unwrap();
1160 assert_eq!(file.view().lookup_hash(h), Some("Entity"));
1161 }
1162}