1use core::cell::UnsafeCell;
13#[path = "compute_universe_epoch.rs"]
15mod epoch;
16pub use epoch::{begin_request_epoch, ComputeRequestEpoch};
17#[allow(unused_imports)]
18pub(crate) use epoch::{
19 rebuild_kv as rebuild_context_provenance, record_kv as record_context_provenance,
20};
21#[path = "compute_universe_context.rs"]
22mod context;
23#[allow(unused_imports)]
24pub(crate) use context::{
25 context_input_digest, context_live, context_node_count, context_tensor_at,
26};
27#[cfg(not(target_arch = "wasm32"))]
28use std::sync::atomic::AtomicBool;
29use std::sync::atomic::{AtomicU32, Ordering};
30
31#[cfg(not(target_arch = "wasm32"))]
32use std::thread;
33#[cfg(not(target_arch = "wasm32"))]
34use std::time::Duration;
35
36use crate::gpu_context::{
37 ComputeUniverse, OperationalMode, QueueLane, UniverseOrchestrator, VramLedger, VramLedgerSlot,
38};
39use crate::tensor::buffer_export::{TensorBufferHeader, TENSOR_STRIDE};
40use crate::tensor::resident_substrate::{global_resident_substrate, MAX_KNN_HITS};
41use crate::tensor::Tensor10D;
42
43#[repr(u8)]
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum QualiaPrimitive {
47 Phase8LlmForward = 0,
49 GraphTensorSubstrate = 1,
51 Phase8Sentinel = 2,
53 VramLedgerPin = 3,
55}
56
57impl ComputeUniverse {
58 #[inline]
60 pub fn qualia_primitives(self) -> &'static [QualiaPrimitive] {
62 match self {
63 ComputeUniverse::LlmInference => &[
64 QualiaPrimitive::Phase8LlmForward,
65 QualiaPrimitive::Phase8Sentinel,
66 QualiaPrimitive::VramLedgerPin,
67 ],
68 ComputeUniverse::Tensor10D => &[
69 QualiaPrimitive::GraphTensorSubstrate,
70 QualiaPrimitive::VramLedgerPin,
71 ],
72 ComputeUniverse::Viewport => &[
73 QualiaPrimitive::GraphTensorSubstrate,
74 QualiaPrimitive::VramLedgerPin,
75 ],
76 ComputeUniverse::AcousticPlane => &[
77 QualiaPrimitive::GraphTensorSubstrate,
78 QualiaPrimitive::VramLedgerPin,
79 ],
80 }
81 }
82
83 #[inline]
84 pub fn phase8_channels(self) -> &'static [Phase8Channel] {
86 match self {
87 ComputeUniverse::LlmInference => &[
88 Phase8Channel::LogitUpstream,
89 Phase8Channel::ControlDownstream,
90 Phase8Channel::ContextInject,
91 ],
92 ComputeUniverse::Tensor10D => &[Phase8Channel::ContextInject],
93 ComputeUniverse::Viewport => &[],
94 ComputeUniverse::AcousticPlane => &[],
95 }
96 }
97}
98
99#[repr(u8)]
101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
102pub enum Phase8Channel {
103 LogitUpstream = 0,
105 ControlDownstream = 1,
107 ContextInject = 2,
109}
110
111pub const CONTEXT_INJECT_RING_CAP: usize = 64;
112pub const ATTENTION_MASK_WORDS: usize = 256;
113pub const KV_ATTENTION_MASK_WORDS: usize = 32;
115pub const MAX_DRAFT_LEN: usize = 8;
116pub const TOPOLOGY_DRAFT_RING_CAP: usize = 4;
117
118#[repr(C, align(8))]
120#[derive(Clone, Copy, Debug, PartialEq)]
121pub struct ContextInjectToken {
122 pub tensor_index: u32,
123 pub subject_hash: u64,
124 pub distance: f32,
125 pub manifold_w: f32,
126}
127
128impl ContextInjectToken {
129 pub const fn empty() -> Self {
131 Self {
132 tensor_index: 0,
133 subject_hash: 0,
134 distance: 0.0,
135 manifold_w: 0.0,
136 }
137 }
138}
139
140impl Default for ContextInjectToken {
141 fn default() -> Self {
143 Self::empty()
144 }
145}
146
147pub struct ContextInjectRing {
149 epochs: UnsafeCell<[u64; CONTEXT_INJECT_RING_CAP]>,
150 slots: UnsafeCell<[ContextInjectToken; CONTEXT_INJECT_RING_CAP]>,
151 write_seq: AtomicU32,
152 read_seq: AtomicU32,
153}
154
155unsafe impl Sync for ContextInjectRing {}
157
158impl ContextInjectRing {
159 pub const fn new() -> Self {
161 Self {
162 slots: UnsafeCell::new([ContextInjectToken::empty(); CONTEXT_INJECT_RING_CAP]),
163 epochs: UnsafeCell::new([0; CONTEXT_INJECT_RING_CAP]),
164 write_seq: AtomicU32::new(0),
165 read_seq: AtomicU32::new(0),
166 }
167 }
168
169 #[inline]
170 pub fn len(&self) -> usize {
172 let w = self.write_seq.load(Ordering::Acquire);
173 let r = self.read_seq.load(Ordering::Acquire);
174 (w.wrapping_sub(r)) as usize
175 }
176
177 #[inline]
178 pub fn is_empty(&self) -> bool {
180 self.len() == 0
181 }
182
183 pub fn try_push(&self, token: ContextInjectToken) -> bool {
185 if !epoch::can_publish() {
186 return false;
187 }
188 let epoch = epoch::current();
189 if !epoch::accepts(epoch) {
190 return false;
191 }
192 let w = self.write_seq.load(Ordering::Relaxed);
193 let r = self.read_seq.load(Ordering::Acquire);
194 if w.wrapping_sub(r) >= CONTEXT_INJECT_RING_CAP as u32 {
195 return false;
196 }
197 let slot = (w % CONTEXT_INJECT_RING_CAP as u32) as usize;
198 unsafe {
199 (*self.slots.get())[slot] = token;
200 (*self.epochs.get())[slot] = epoch;
201 }
202 self.write_seq.store(w.wrapping_add(1), Ordering::Release);
203 true
204 }
205
206 pub fn try_pop(&self) -> Option<ContextInjectToken> {
208 loop {
209 let r = self.read_seq.load(Ordering::Relaxed);
210 let w = self.write_seq.load(Ordering::Acquire);
211 if r == w {
212 return None;
213 }
214 let slot = (r % CONTEXT_INJECT_RING_CAP as u32) as usize;
215 let token = unsafe { (*self.slots.get())[slot] };
216 let epoch = unsafe { (*self.epochs.get())[slot] };
217 self.read_seq.store(r.wrapping_add(1), Ordering::Release);
218 if epoch::accepts(epoch) {
219 return Some(token);
220 }
221 }
222 }
223}
224
225#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
227pub struct GraphTensorSubstrate {
228 pub byte_len: u64,
229 pub node_count: u32,
230 pub stride_bytes: u32,
231 pub header_bytes: u32,
232}
233
234impl GraphTensorSubstrate {
235 #[inline]
236 pub fn from_tensor_bytes(resident_bytes: u64) -> Self {
238 let header_bytes = std::mem::size_of::<TensorBufferHeader>() as u32;
239 let stride = TENSOR_STRIDE as u32;
240 if resident_bytes <= header_bytes as u64 {
241 return Self {
242 byte_len: resident_bytes,
243 header_bytes,
244 stride_bytes: stride,
245 node_count: 0,
246 };
247 }
248 let payload = resident_bytes - header_bytes as u64;
249 let node_count = (payload / stride as u64).min(u32::MAX as u64) as u32;
250 Self {
251 byte_len: resident_bytes,
252 node_count,
253 stride_bytes: stride,
254 header_bytes,
255 }
256 }
257
258 #[inline]
259 pub fn is_resident(&self) -> bool {
261 self.node_count > 0
262 }
263}
264
265#[derive(Debug, Clone, Copy)]
267pub struct AttentionRouteMask {
268 pub words: [u64; ATTENTION_MASK_WORDS],
269 pub active_bits: u32,
270}
271
272impl Default for AttentionRouteMask {
273 fn default() -> Self {
275 Self {
276 words: [0u64; ATTENTION_MASK_WORDS],
277 active_bits: 0,
278 }
279 }
280}
281
282impl AttentionRouteMask {
283 #[inline]
284 pub fn set_index(&mut self, index: u32) {
286 let bit = index as usize;
287 let word = bit / 64;
288 let offset = bit % 64;
289 if word < ATTENTION_MASK_WORDS {
290 self.words[word] |= 1u64 << offset;
291 self.active_bits = self.active_bits.saturating_add(1);
292 }
293 }
294
295 #[inline]
296 pub fn is_set(&self, index: u32) -> bool {
298 let bit = index as usize;
299 let word = bit / 64;
300 let offset = bit % 64;
301 word < ATTENTION_MASK_WORDS && (self.words[word] & (1u64 << offset)) != 0
302 }
303}
304
305pub struct UniverseFabric {
307 pub orchestrator: UniverseOrchestrator,
308 pub substrate: GraphTensorSubstrate,
309}
310
311impl UniverseFabric {
312 #[inline]
313 pub fn current(ledger: &VramLedger) -> Self {
315 let tensor_bytes = ledger.used_in_slot(VramLedgerSlot::Tensor10D);
316 Self {
317 orchestrator: crate::gpu_context::universe_orchestrator(),
318 substrate: GraphTensorSubstrate::from_tensor_bytes(tensor_bytes),
319 }
320 }
321
322 #[inline]
324 pub fn reserve_protects_llm_kv(&self, global: OperationalMode) -> bool {
327 global == OperationalMode::Reserve
328 && self
329 .orchestrator
330 .effective_mode(ComputeUniverse::LlmInference, global)
331 == OperationalMode::Full
332 && self
333 .orchestrator
334 .effective_mode(ComputeUniverse::Viewport, global)
335 == OperationalMode::Reserve
336 }
337
338 #[inline]
339 pub fn queue_for(&self, universe: ComputeUniverse) -> QueueLane {
341 self.orchestrator.partition(universe).queue_lane
342 }
343
344 #[inline]
345 pub fn can_pin_tensor(&self, ledger: &VramLedger, extra_bytes: u64) -> bool {
348 ledger.can_allocate_in_universe(&self.orchestrator, ComputeUniverse::Tensor10D, extra_bytes)
349 }
350}
351
352static CONTEXT_INJECT_RING: ContextInjectRing = ContextInjectRing::new();
353
354#[inline]
355pub fn context_inject_ring() -> &'static ContextInjectRing {
357 &CONTEXT_INJECT_RING
358}
359
360#[inline]
362pub fn push_tensor_context(token: ContextInjectToken) -> bool {
364 let ok = context_inject_ring().try_push(token);
365 if ok {
366 crate::gpu_context::record_logic_flash();
367 }
368 ok
369}
370
371#[inline]
373pub fn pop_tensor_context() -> Option<ContextInjectToken> {
375 context_inject_ring().try_pop()
376}
377
378static DECODE_TOKEN_ID: AtomicU32 = AtomicU32::new(0);
381static DECODE_STEP: AtomicU32 = AtomicU32::new(0);
382static QUERY_SUBJECT_HASH: AtomicU32 = AtomicU32::new(0); static QUERY_SUBJECT_HI: AtomicU32 = AtomicU32::new(0);
384
385const ZERO_TENSOR10D: Tensor10D = Tensor10D {
386 q: 0.0,
387 v: 0.0,
388 w: 0.0,
389 x: 0.0,
390 y: 0.0,
391 z: 0.0,
392 t: 0.0,
393 alpha: 0.0,
394 mu: 0.0,
395 sigma: 0.0,
396};
397
398static QUERY_TENSOR_CELL: std::sync::Mutex<Tensor10D> = std::sync::Mutex::new(ZERO_TENSOR10D);
399static QUERY_SEQ: AtomicU32 = AtomicU32::new(0);
400
401static ATTENTION_MASK_CELL: std::sync::Mutex<AttentionRouteMask> =
402 std::sync::Mutex::new(AttentionRouteMask {
403 words: [0u64; ATTENTION_MASK_WORDS],
404 active_bits: 0,
405 });
406static ATTENTION_MASK_SEQ: AtomicU32 = AtomicU32::new(0);
407
408#[cfg(not(target_arch = "wasm32"))]
409static PRODUCER_STARTED: AtomicBool = AtomicBool::new(false);
410#[cfg(not(target_arch = "wasm32"))]
411static PRODUCER_STOP: AtomicBool = AtomicBool::new(false);
412
413#[inline]
415pub fn publish_decode_hint(token_id: u32, step: u32) {
418 if !epoch::can_publish() {
419 return;
420 }
421 DECODE_TOKEN_ID.store(token_id, Ordering::Relaxed);
422 DECODE_STEP.store(step, Ordering::Release);
423}
424
425#[inline]
427pub fn publish_query_tensor(tensor: Tensor10D, subject_hash: u64) {
430 if !epoch::can_publish() {
431 return;
432 }
433 *QUERY_TENSOR_CELL.lock().unwrap_or_else(|e| e.into_inner()) = tensor;
434 QUERY_SUBJECT_HASH.store(subject_hash as u32, Ordering::Relaxed);
435 QUERY_SUBJECT_HI.store((subject_hash >> 32) as u32, Ordering::Relaxed);
436 QUERY_SEQ.fetch_add(1, Ordering::Release);
437}
438
439#[inline]
440pub fn query_subject_hash() -> u64 {
442 let lo = QUERY_SUBJECT_HASH.load(Ordering::Relaxed) as u64;
443 let hi = QUERY_SUBJECT_HI.load(Ordering::Relaxed) as u64;
444 (hi << 32) | lo
445}
446
447#[inline]
448pub fn load_query_tensor() -> Tensor10D {
450 *QUERY_TENSOR_CELL.lock().unwrap_or_else(|e| e.into_inner())
451}
452
453#[inline]
454pub fn attention_route_mask() -> AttentionRouteMask {
456 *ATTENTION_MASK_CELL
457 .lock()
458 .unwrap_or_else(|e| e.into_inner())
459}
460
461#[inline]
462pub fn attention_mask_seq() -> u32 {
464 ATTENTION_MASK_SEQ.load(Ordering::Acquire)
465}
466
467#[inline]
469pub fn publish_attention_route_mask(mask: AttentionRouteMask) {
472 if !epoch::can_publish() {
473 return;
474 }
475 *ATTENTION_MASK_CELL
476 .lock()
477 .unwrap_or_else(|e| e.into_inner()) = mask;
478 ATTENTION_MASK_SEQ.fetch_add(1, Ordering::Release);
479}
480
481#[inline]
482fn set_kv_mask_bit(words: &mut [u32; KV_ATTENTION_MASK_WORDS], slot: u32) {
484 let bit = slot as usize;
485 let word = bit / 32;
486 let offset = bit % 32;
487 if word < KV_ATTENTION_MASK_WORDS {
488 words[word] |= 1u32 << offset;
489 }
490}
491
492#[inline]
494pub fn attention_kv_mask_u32(
496 token_idx: u32,
497 max_context: u32,
498) -> ([u32; KV_ATTENTION_MASK_WORDS], u32) {
499 let route = attention_route_mask();
500 let mut words = [0u32; KV_ATTENTION_MASK_WORDS];
501 let cap = max_context.min((KV_ATTENTION_MASK_WORDS as u32) * 32);
502 set_kv_mask_bit(&mut words, token_idx.min(cap.saturating_sub(1)));
503
504 let provenance = crate::tensor::kv_provenance::global_kv_provenance();
505 let mut mapped = 1u32;
506 for (word_idx, w) in route.words.iter().enumerate() {
507 for bit in 0..64 {
508 if (*w & (1u64 << bit)) == 0 {
509 continue;
510 }
511 let tensor_idx = (word_idx * 64 + bit) as u32;
512 let kv_slot = epoch::scoped_kv_slot(tensor_idx)
513 .unwrap_or_else(|| provenance.kv_slot_for_tensor(tensor_idx))
514 .unwrap_or(tensor_idx);
515 if kv_slot < cap {
516 set_kv_mask_bit(&mut words, kv_slot);
517 mapped += 1;
518 }
519 }
520 }
521
522 let mask_active = if mapped > 1 && route.active_bits > 0 {
523 1u32
524 } else {
525 0u32
526 };
527 (words, mask_active)
528}
529
530#[inline]
532pub fn build_attention_route_mask(
535 query: &Tensor10D,
536 max_distance: f32,
537 hits: &[usize],
538 hit_count: usize,
539) -> AttentionRouteMask {
540 let mut mask = AttentionRouteMask::default();
541 for i in 0..hit_count.min(hits.len()) {
542 mask.set_index(hits[i] as u32);
543 }
544 let _ = (query, max_distance);
545 mask
546}
547
548pub fn run_tensor_search_producer_cycle(max_distance: f32, vocab_len: u32) -> usize {
550 epoch::produce(|| run_tensor_search_producer_cycle_in_epoch(max_distance, vocab_len))
551}
552fn run_tensor_search_producer_cycle_in_epoch(max_distance: f32, vocab_len: u32) -> usize {
553 if let Some(source) = epoch::source() {
554 return context::produce_owned(&source, max_distance, vocab_len);
555 }
556 let substrate = global_resident_substrate();
557 if substrate.node_count() == 0 {
558 return 0;
559 }
560
561 let query = load_query_tensor();
562 let subject_hash = query_subject_hash();
563 let mut hits = [0usize; MAX_KNN_HITS];
564 let hit_count = {
571 #[cfg(feature = "gpu-runtime")]
572 {
573 crate::tensor::volume_gpu::try_gpu_tensor_search_into(&query, max_distance, &mut hits)
574 .unwrap_or_else(|| {
575 substrate
576 .tensor_search_into(&query, max_distance, &mut hits)
577 .unwrap_or(0)
578 })
579 }
580 #[cfg(not(feature = "gpu-runtime"))]
581 {
582 substrate
583 .tensor_search_into(&query, max_distance, &mut hits)
584 .unwrap_or(0)
585 }
586 };
587
588 let mask = build_attention_route_mask(&query, max_distance, &hits, hit_count);
589 publish_attention_route_mask(mask);
590
591 let pushed = push_tensor_search_hits(&hits[..hit_count], subject_hash, query.w, max_distance);
592 let _ = extrapolate_topology_draft(&query, subject_hash, 4, vocab_len);
593 pushed
594}
595
596#[cfg(not(target_arch = "wasm32"))]
597fn tensor_search_producer_loop() {
600 crate::platform_scheduler::bind_background_thread();
601 let mut idle_spins = 0u32;
602 while !PRODUCER_STOP.load(Ordering::Relaxed) {
603 let pushed = run_tensor_search_producer_cycle(3.0, 32_000);
604 if pushed > 0 {
605 crate::gpu_context::record_producer_cycle(pushed as u32);
606 idle_spins = 0;
607 } else if global_resident_substrate().node_count() == 0 {
608 thread::sleep(Duration::from_millis(4));
609 } else {
610 idle_spins = idle_spins.saturating_add(1);
611 if idle_spins > 32 {
612 thread::sleep(Duration::from_millis(1));
613 idle_spins = 0;
614 } else {
615 thread::yield_now();
616 }
617 }
618 }
619}
620
621#[cfg(not(target_arch = "wasm32"))]
623pub fn start_tensor_search_producer() -> bool {
624 if PRODUCER_STARTED.swap(true, Ordering::AcqRel) {
625 return false;
626 }
627 PRODUCER_STOP.store(false, Ordering::Release);
628 thread::Builder::new()
629 .name("qualia-u1-tensor-producer".into())
630 .spawn(tensor_search_producer_loop)
631 .expect("U1 tensor search producer thread");
632 true
633}
634
635#[cfg(not(target_arch = "wasm32"))]
637pub fn stop_tensor_search_producer() {
638 PRODUCER_STOP.store(true, Ordering::Release);
639}
640
641#[cfg(target_arch = "wasm32")]
643pub fn start_tensor_search_producer() -> bool {
644 false
645}
646
647#[cfg(target_arch = "wasm32")]
649pub fn stop_tensor_search_producer() {}
650
651#[inline]
653pub fn push_tensor_search_hits(
656 indices: &[usize],
657 subject_hash: u64,
658 manifold_w: f32,
659 max_distance: f32,
660) -> usize {
661 let mut pushed = 0usize;
662 for (rank, &index) in indices.iter().enumerate() {
663 if !push_tensor_context(ContextInjectToken {
664 tensor_index: index as u32,
665 subject_hash,
666 distance: max_distance * (1.0 + rank as f32 * 0.01),
667 manifold_w,
668 }) {
669 crate::gpu_context::record_context_ring_drop();
670 break;
671 }
672 pushed += 1;
673 }
674 pushed
675}
676
677#[repr(C, align(8))]
681#[derive(Clone, Copy, Debug, PartialEq)]
682pub struct TopologyDraftBatch {
683 pub draft_len: u8,
684 pub _pad: [u8; 7],
685 pub draft_ids: [u32; MAX_DRAFT_LEN],
686 pub concept_hashes: [u64; MAX_DRAFT_LEN],
687}
688
689impl TopologyDraftBatch {
690 pub const fn empty() -> Self {
692 Self {
693 draft_len: 0,
694 _pad: [0; 7],
695 draft_ids: [0; MAX_DRAFT_LEN],
696 concept_hashes: [0; MAX_DRAFT_LEN],
697 }
698 }
699}
700
701pub struct TopologyDraftRing {
702 epochs: UnsafeCell<[u64; TOPOLOGY_DRAFT_RING_CAP]>,
703 slots: UnsafeCell<[TopologyDraftBatch; TOPOLOGY_DRAFT_RING_CAP]>,
704 write_seq: AtomicU32,
705 read_seq: AtomicU32,
706}
707
708unsafe impl Sync for TopologyDraftRing {}
709
710impl TopologyDraftRing {
711 pub const fn new() -> Self {
713 Self {
714 slots: UnsafeCell::new([TopologyDraftBatch::empty(); TOPOLOGY_DRAFT_RING_CAP]),
715 epochs: UnsafeCell::new([0; TOPOLOGY_DRAFT_RING_CAP]),
716 write_seq: AtomicU32::new(0),
717 read_seq: AtomicU32::new(0),
718 }
719 }
720
721 pub fn try_push(&self, batch: TopologyDraftBatch) -> bool {
723 if !epoch::can_publish() {
724 return false;
725 }
726 let epoch = epoch::current();
727 if !epoch::accepts(epoch) {
728 return false;
729 }
730 if batch.draft_len == 0 {
731 return false;
732 }
733 let w = self.write_seq.load(Ordering::Relaxed);
734 let r = self.read_seq.load(Ordering::Acquire);
735 if w.wrapping_sub(r) >= TOPOLOGY_DRAFT_RING_CAP as u32 {
736 return false;
737 }
738 let slot = (w % TOPOLOGY_DRAFT_RING_CAP as u32) as usize;
739 unsafe {
740 (*self.slots.get())[slot] = batch;
741 (*self.epochs.get())[slot] = epoch;
742 }
743 self.write_seq.store(w.wrapping_add(1), Ordering::Release);
744 true
745 }
746
747 pub fn try_pop(&self) -> Option<TopologyDraftBatch> {
749 loop {
750 let r = self.read_seq.load(Ordering::Relaxed);
751 let w = self.write_seq.load(Ordering::Acquire);
752 if r == w {
753 return None;
754 }
755 let slot = (r % TOPOLOGY_DRAFT_RING_CAP as u32) as usize;
756 let batch = unsafe { (*self.slots.get())[slot] };
757 let epoch = unsafe { (*self.epochs.get())[slot] };
758 self.read_seq.store(r.wrapping_add(1), Ordering::Release);
759 if epoch::accepts(epoch) {
760 return Some(batch);
761 }
762 }
763 }
764}
765
766static TOPOLOGY_DRAFT_RING: TopologyDraftRing = TopologyDraftRing::new();
767
768#[inline]
769pub fn topology_draft_ring() -> &'static TopologyDraftRing {
771 &TOPOLOGY_DRAFT_RING
772}
773
774#[inline]
775pub fn pop_topology_draft() -> Option<TopologyDraftBatch> {
777 topology_draft_ring().try_pop()
778}
779
780#[inline]
786pub fn sentinel_allows_topology_draft(batch: &TopologyDraftBatch) -> bool {
789 batch.draft_len > 0
790}
791
792#[derive(Clone, Copy, Debug, PartialEq)]
794pub struct SpeculativeVerifyResult {
795 pub accepted_count: usize,
797 pub emitted_token: u32,
800 pub fully_accepted: bool,
802}
803
804pub fn verify_topology_draft(
814 draft: &TopologyDraftBatch,
815 target_predictions: &[u32],
816) -> SpeculativeVerifyResult {
817 let len = (draft.draft_len as usize)
818 .min(target_predictions.len())
819 .min(MAX_DRAFT_LEN);
820 let mut accepted = 0;
821 while accepted < len {
822 if draft.draft_ids[accepted] == target_predictions[accepted] {
823 accepted += 1;
824 } else {
825 break;
826 }
827 }
828
829 let fully_accepted = accepted == draft.draft_len as usize;
830 let emitted_token = if accepted < target_predictions.len() {
831 target_predictions[accepted]
832 } else if accepted > 0 {
833 draft.draft_ids[accepted - 1]
834 } else {
835 0
836 };
837
838 SpeculativeVerifyResult {
839 accepted_count: accepted,
840 emitted_token,
841 fully_accepted,
842 }
843}
844
845pub fn extrapolate_topology_draft(
847 query: &Tensor10D,
848 subject_hash: u64,
849 gamma: usize,
850 vocab_len: u32,
851) -> Option<TopologyDraftBatch> {
852 #[cfg(any(not(target_arch = "wasm32"), feature = "wasm-llm"))]
853 {
854 extrapolate_topology_draft_mapped(query, subject_hash, gamma, vocab_len, None)
855 }
856 #[cfg(all(target_arch = "wasm32", not(feature = "wasm-llm")))]
857 {
858 extrapolate_topology_draft_mapped(query, subject_hash, gamma, vocab_len)
859 }
860}
861
862pub fn extrapolate_topology_draft_mapped(
868 query: &Tensor10D,
869 subject_hash: u64,
870 gamma: usize,
871 vocab_len: u32,
872 #[cfg(any(not(target_arch = "wasm32"), feature = "wasm-llm"))] mapper: Option<
873 &crate::topology_draft::TopologyDraftMapper<'_>,
874 >,
875) -> Option<TopologyDraftBatch> {
876 let gamma = gamma.clamp(1, MAX_DRAFT_LEN);
877 let substrate = global_resident_substrate();
878 if substrate.node_count() == 0 || vocab_len == 0 {
879 return None;
880 }
881 let mut hits = [0usize; MAX_KNN_HITS];
882 let hit_count = {
883 #[cfg(feature = "gpu-runtime")]
884 {
885 crate::tensor::volume_gpu::try_gpu_tensor_search_into(query, 4.0, &mut hits)
886 .unwrap_or_else(|| {
887 substrate
888 .tensor_search_into(query, 4.0, &mut hits)
889 .unwrap_or(0)
890 })
891 }
892 #[cfg(not(feature = "gpu-runtime"))]
893 {
894 substrate
895 .tensor_search_into(query, 4.0, &mut hits)
896 .unwrap_or(0)
897 }
898 };
899 if hit_count == 0 {
900 return None;
901 }
902
903 let mut concepts = [0u64; MAX_DRAFT_LEN];
904 for i in 0..gamma.min(hit_count) {
905 concepts[i] = substrate.subject_hash_at(hits[i] as u32) ^ subject_hash;
906 }
907 let batch = {
908 #[cfg(any(not(target_arch = "wasm32"), feature = "wasm-llm"))]
909 if let Some(m) = mapper {
910 m.fill_draft_batch(&concepts[..gamma.min(hit_count)], gamma.min(hit_count))
911 } else {
912 let mut batch = TopologyDraftBatch::empty();
913 for i in 0..gamma.min(hit_count) {
914 batch.concept_hashes[i] = concepts[i];
915 batch.draft_ids[i] = (concepts[i] as u32) % vocab_len.max(1);
916 }
917 batch.draft_len = gamma.min(hit_count) as u8;
918 batch
919 }
920 #[cfg(all(target_arch = "wasm32", not(feature = "wasm-llm")))]
921 {
922 let mut batch = TopologyDraftBatch::empty();
923 for i in 0..gamma.min(hit_count) {
924 batch.concept_hashes[i] = concepts[i];
925 batch.draft_ids[i] = (concepts[i] as u32) % vocab_len.max(1);
926 }
927 batch.draft_len = gamma.min(hit_count) as u8;
928 batch
929 }
930 };
931 let _ = topology_draft_ring().try_push(batch);
932 Some(batch)
933}
934
935#[cfg(test)]
936#[path = "compute_universe_test_support.rs"]
937mod test_support;
938
939#[cfg(test)]
940#[path = "compute_universe_tests.rs"]
941mod tests;