Skip to main content

qualia_core_db/inference/
compute_universe.rs

1//! Qualia-native **compute universe fabric** (Track B2).
2//!
3//! This is not a generic GPU scheduler. Universes U0/U1/U2 map to QualiaDB primitives:
4//!
5//! 1. **Graph–tensor duality** — `NQuin` graph and `Tensor10D` SOA share one resident substrate (U1 pin).
6//! 2. **Phase-8 bifurcation** — lock-free SPSC rings between U0 (LLM), Sentinel, and U1 (tensor).
7//! 3. **Sentinel governance** — `DenyRollback` on the control ring before the next token matmul.
8//! 4. **VramLedger pins** — Full / Eco / Reserve per universe; U0 KV protected in Reserve.
9//!
10//! One physical `etalon::Device` (`gpu_context::shared_gpu`); logical parallelism via queues + rings.
11
12use core::cell::UnsafeCell;
13// Request lifetime and result-envelope ownership are separated from fabric math.
14#[path = "compute_universe_epoch.rs"]
15mod epoch;
16pub use epoch::{begin_request_epoch, ComputeRequestEpoch};
17#[allow(unused_imports)]
18pub(crate) use epoch::{
19    rebuild_kv as rebuild_context_provenance, record_kv as record_context_provenance,
20};
21#[path = "compute_universe_context.rs"]
22mod context;
23#[allow(unused_imports)]
24pub(crate) use context::{
25    context_input_digest, context_live, context_node_count, context_tensor_at,
26};
27#[cfg(not(target_arch = "wasm32"))]
28use std::sync::atomic::AtomicBool;
29use std::sync::atomic::{AtomicU32, Ordering};
30
31#[cfg(not(target_arch = "wasm32"))]
32use std::thread;
33#[cfg(not(target_arch = "wasm32"))]
34use std::time::Duration;
35
36use crate::gpu_context::{
37    ComputeUniverse, OperationalMode, QueueLane, UniverseOrchestrator, VramLedger, VramLedgerSlot,
38};
39use crate::tensor::buffer_export::{TensorBufferHeader, TENSOR_STRIDE};
40use crate::tensor::resident_substrate::{global_resident_substrate, MAX_KNN_HITS};
41use crate::tensor::Tensor10D;
42
43/// Qualia engine primitive exploited by a compute universe (not portable DB semantics).
44#[repr(u8)]
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum QualiaPrimitive {
47    /// U0 autoregressive forward + KV (Phase-8 LLM thread).
48    Phase8LlmForward = 0,
49    /// U1/U2 shared SOA: `NQuin` ↔ `Tensor10D` zero-copy substrate.
50    GraphTensorSubstrate = 1,
51    /// Sentinel on logit stream; `DenyRollback` on control ring.
52    Phase8Sentinel = 2,
53    /// Pinned ledger slots per universe (Full / Eco / Reserve).
54    VramLedgerPin = 3,
55}
56
57impl ComputeUniverse {
58    /// Native primitives this universe must use (documentation + static dispatch hooks).
59    #[inline]
60    /// Qualia primitives resident in this compute universe.
61    pub fn qualia_primitives(self) -> &'static [QualiaPrimitive] {
62        match self {
63            ComputeUniverse::LlmInference => &[
64                QualiaPrimitive::Phase8LlmForward,
65                QualiaPrimitive::Phase8Sentinel,
66                QualiaPrimitive::VramLedgerPin,
67            ],
68            ComputeUniverse::Tensor10D => &[
69                QualiaPrimitive::GraphTensorSubstrate,
70                QualiaPrimitive::VramLedgerPin,
71            ],
72            ComputeUniverse::Viewport => &[
73                QualiaPrimitive::GraphTensorSubstrate,
74                QualiaPrimitive::VramLedgerPin,
75            ],
76            ComputeUniverse::AcousticPlane => &[
77                QualiaPrimitive::GraphTensorSubstrate,
78                QualiaPrimitive::VramLedgerPin,
79            ],
80        }
81    }
82
83    #[inline]
84    /// Phase-8 channels owned by this universe.
85    pub fn phase8_channels(self) -> &'static [Phase8Channel] {
86        match self {
87            ComputeUniverse::LlmInference => &[
88                Phase8Channel::LogitUpstream,
89                Phase8Channel::ControlDownstream,
90                Phase8Channel::ContextInject,
91            ],
92            ComputeUniverse::Tensor10D => &[Phase8Channel::ContextInject],
93            ComputeUniverse::Viewport => &[],
94            ComputeUniverse::AcousticPlane => &[],
95        }
96    }
97}
98
99/// Phase-8 bifurcation ring (see `llm_agent.rs` LogitStream / ControlStream + U1 inject).
100#[repr(u8)]
101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
102pub enum Phase8Channel {
103    /// U0 → Sentinel: per-step logit summary (anomaly / governance).
104    LogitUpstream = 0,
105    /// Sentinel → U0: `DenyRollback` before next matmul.
106    ControlDownstream = 1,
107    /// U1 → U0: continuous context from `visit_tensor_search_into` (no stop-and-RAG).
108    ContextInject = 2,
109}
110
111pub const CONTEXT_INJECT_RING_CAP: usize = 64;
112pub const ATTENTION_MASK_WORDS: usize = 256;
113/// KV cache bitmask for `fused_attention.wgsl` (1024 context slots).
114pub const KV_ATTENTION_MASK_WORDS: usize = 32;
115pub const MAX_DRAFT_LEN: usize = 8;
116pub const TOPOLOGY_DRAFT_RING_CAP: usize = 4;
117
118/// Lightweight pointer pushed U1→U0 (48 B `NQuin` stays in SOA; ring carries index + hash).
119#[repr(C, align(8))]
120#[derive(Clone, Copy, Debug, PartialEq)]
121pub struct ContextInjectToken {
122    pub tensor_index: u32,
123    pub subject_hash: u64,
124    pub distance: f32,
125    pub manifold_w: f32,
126}
127
128impl ContextInjectToken {
129    /// Zeroed token (no tensor, no subject).
130    pub const fn empty() -> Self {
131        Self {
132            tensor_index: 0,
133            subject_hash: 0,
134            distance: 0.0,
135            manifold_w: 0.0,
136        }
137    }
138}
139
140impl Default for ContextInjectToken {
141    /// Default is the zeroed `empty()` token.
142    fn default() -> Self {
143        Self::empty()
144    }
145}
146
147/// Fixed-capacity SPSC context ring (producer U1, consumer U0).
148pub struct ContextInjectRing {
149    epochs: UnsafeCell<[u64; CONTEXT_INJECT_RING_CAP]>,
150    slots: UnsafeCell<[ContextInjectToken; CONTEXT_INJECT_RING_CAP]>,
151    write_seq: AtomicU32,
152    read_seq: AtomicU32,
153}
154
155// SAFETY: SPSC — one producer (U1), one consumer (U0); slot published via write_seq Release.
156unsafe impl Sync for ContextInjectRing {}
157
158impl ContextInjectRing {
159    /// Empty fixed-capacity ring (all slots zeroed).
160    pub const fn new() -> Self {
161        Self {
162            slots: UnsafeCell::new([ContextInjectToken::empty(); CONTEXT_INJECT_RING_CAP]),
163            epochs: UnsafeCell::new([0; CONTEXT_INJECT_RING_CAP]),
164            write_seq: AtomicU32::new(0),
165            read_seq: AtomicU32::new(0),
166        }
167    }
168
169    #[inline]
170    /// Number of live ledger entries.
171    pub fn len(&self) -> usize {
172        let w = self.write_seq.load(Ordering::Acquire);
173        let r = self.read_seq.load(Ordering::Acquire);
174        (w.wrapping_sub(r)) as usize
175    }
176
177    #[inline]
178    /// `true` when no entries are tracked.
179    pub fn is_empty(&self) -> bool {
180        self.len() == 0
181    }
182
183    /// Producer (U1 tensor search). Returns false when full.
184    pub fn try_push(&self, token: ContextInjectToken) -> bool {
185        if !epoch::can_publish() {
186            return false;
187        }
188        let epoch = epoch::current();
189        if !epoch::accepts(epoch) {
190            return false;
191        }
192        let w = self.write_seq.load(Ordering::Relaxed);
193        let r = self.read_seq.load(Ordering::Acquire);
194        if w.wrapping_sub(r) >= CONTEXT_INJECT_RING_CAP as u32 {
195            return false;
196        }
197        let slot = (w % CONTEXT_INJECT_RING_CAP as u32) as usize;
198        unsafe {
199            (*self.slots.get())[slot] = token;
200            (*self.epochs.get())[slot] = epoch;
201        }
202        self.write_seq.store(w.wrapping_add(1), Ordering::Release);
203        true
204    }
205
206    /// Consumer (U0 decode loop). Returns None when empty.
207    pub fn try_pop(&self) -> Option<ContextInjectToken> {
208        loop {
209            let r = self.read_seq.load(Ordering::Relaxed);
210            let w = self.write_seq.load(Ordering::Acquire);
211            if r == w {
212                return None;
213            }
214            let slot = (r % CONTEXT_INJECT_RING_CAP as u32) as usize;
215            let token = unsafe { (*self.slots.get())[slot] };
216            let epoch = unsafe { (*self.epochs.get())[slot] };
217            self.read_seq.store(r.wrapping_add(1), Ordering::Release);
218            if epoch::accepts(epoch) {
219                return Some(token);
220            }
221        }
222    }
223}
224
225/// Resident graph↔tensor SOA visible to U1 (write/search), U0 (read/inject), U2 (render).
226#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
227pub struct GraphTensorSubstrate {
228    pub byte_len: u64,
229    pub node_count: u32,
230    pub stride_bytes: u32,
231    pub header_bytes: u32,
232}
233
234impl GraphTensorSubstrate {
235    #[inline]
236    /// Residency set with `resident_bytes` already occupied.
237    pub fn from_tensor_bytes(resident_bytes: u64) -> Self {
238        let header_bytes = std::mem::size_of::<TensorBufferHeader>() as u32;
239        let stride = TENSOR_STRIDE as u32;
240        if resident_bytes <= header_bytes as u64 {
241            return Self {
242                byte_len: resident_bytes,
243                header_bytes,
244                stride_bytes: stride,
245                node_count: 0,
246            };
247        }
248        let payload = resident_bytes - header_bytes as u64;
249        let node_count = (payload / stride as u64).min(u32::MAX as u64) as u32;
250        Self {
251            byte_len: resident_bytes,
252            node_count,
253            stride_bytes: stride,
254            header_bytes,
255        }
256    }
257
258    #[inline]
259    /// `true` when any tensor bytes are resident.
260    pub fn is_resident(&self) -> bool {
261        self.node_count > 0
262    }
263}
264
265/// U1 output: sparse attention routing bitmask for U0 matmul (10D kNN filter).
266#[derive(Debug, Clone, Copy)]
267pub struct AttentionRouteMask {
268    pub words: [u64; ATTENTION_MASK_WORDS],
269    pub active_bits: u32,
270}
271
272impl Default for AttentionRouteMask {
273    /// Empty mask — all words zero, `active_bits` 0.
274    fn default() -> Self {
275        Self {
276            words: [0u64; ATTENTION_MASK_WORDS],
277            active_bits: 0,
278        }
279    }
280}
281
282impl AttentionRouteMask {
283    #[inline]
284    /// Set residency bit `index` in the bitmap.
285    pub fn set_index(&mut self, index: u32) {
286        let bit = index as usize;
287        let word = bit / 64;
288        let offset = bit % 64;
289        if word < ATTENTION_MASK_WORDS {
290            self.words[word] |= 1u64 << offset;
291            self.active_bits = self.active_bits.saturating_add(1);
292        }
293    }
294
295    #[inline]
296    /// `true` when residency bit `index` is set.
297    pub fn is_set(&self, index: u32) -> bool {
298        let bit = index as usize;
299        let word = bit / 64;
300        let offset = bit % 64;
301        word < ATTENTION_MASK_WORDS && (self.words[word] & (1u64 << offset)) != 0
302    }
303}
304
305/// Qualia fabric: ledger orchestration + shared substrate + Phase-8 context ring.
306pub struct UniverseFabric {
307    pub orchestrator: UniverseOrchestrator,
308    pub substrate: GraphTensorSubstrate,
309}
310
311impl UniverseFabric {
312    #[inline]
313    /// Snapshot of the ledger's current residency state.
314    pub fn current(ledger: &VramLedger) -> Self {
315        let tensor_bytes = ledger.used_in_slot(VramLedgerSlot::Tensor10D);
316        Self {
317            orchestrator: crate::gpu_context::universe_orchestrator(),
318            substrate: GraphTensorSubstrate::from_tensor_bytes(tensor_bytes),
319        }
320    }
321
322    /// Reserve mode: U2 capped, U0 KV stays Full (LLM wins scheduling).
323    #[inline]
324    /// `true` when this reserve guarantees headroom for the LLM KV
325    /// cache under `global` mode.
326    pub fn reserve_protects_llm_kv(&self, global: OperationalMode) -> bool {
327        global == OperationalMode::Reserve
328            && self
329                .orchestrator
330                .effective_mode(ComputeUniverse::LlmInference, global)
331                == OperationalMode::Full
332            && self
333                .orchestrator
334                .effective_mode(ComputeUniverse::Viewport, global)
335                == OperationalMode::Reserve
336    }
337
338    #[inline]
339    /// Queue lane for `universe` under this policy.
340    pub fn queue_for(&self, universe: ComputeUniverse) -> QueueLane {
341        self.orchestrator.partition(universe).queue_lane
342    }
343
344    #[inline]
345    /// `true` when `extra_bytes` can be pinned without exceeding the
346    /// ledger's VRAM budget.
347    pub fn can_pin_tensor(&self, ledger: &VramLedger, extra_bytes: u64) -> bool {
348        ledger.can_allocate_in_universe(&self.orchestrator, ComputeUniverse::Tensor10D, extra_bytes)
349    }
350}
351
352static CONTEXT_INJECT_RING: ContextInjectRing = ContextInjectRing::new();
353
354#[inline]
355/// Global context-injection ring.
356pub fn context_inject_ring() -> &'static ContextInjectRing {
357    &CONTEXT_INJECT_RING
358}
359
360/// Record a tensor kNN hit for async U0 consumption (continuous RAG path).
361#[inline]
362/// Push a context token onto the inject ring; `false` when full.
363pub fn push_tensor_context(token: ContextInjectToken) -> bool {
364    let ok = context_inject_ring().try_push(token);
365    if ok {
366        crate::gpu_context::record_logic_flash();
367    }
368    ok
369}
370
371/// U0 decode loop pulls injected graph context (non-blocking).
372#[inline]
373/// Pop the next pending context token.
374pub fn pop_tensor_context() -> Option<ContextInjectToken> {
375    context_inject_ring().try_pop()
376}
377
378// ── U0 decode hints (atomics) ───────────────────────────────────────────────
379
380static DECODE_TOKEN_ID: AtomicU32 = AtomicU32::new(0);
381static DECODE_STEP: AtomicU32 = AtomicU32::new(0);
382static QUERY_SUBJECT_HASH: AtomicU32 = AtomicU32::new(0); // lower 32 bits; upper in QUERY_SUBJECT_HI
383static QUERY_SUBJECT_HI: AtomicU32 = AtomicU32::new(0);
384
385const ZERO_TENSOR10D: Tensor10D = Tensor10D {
386    q: 0.0,
387    v: 0.0,
388    w: 0.0,
389    x: 0.0,
390    y: 0.0,
391    z: 0.0,
392    t: 0.0,
393    alpha: 0.0,
394    mu: 0.0,
395    sigma: 0.0,
396};
397
398static QUERY_TENSOR_CELL: std::sync::Mutex<Tensor10D> = std::sync::Mutex::new(ZERO_TENSOR10D);
399static QUERY_SEQ: AtomicU32 = AtomicU32::new(0);
400
401static ATTENTION_MASK_CELL: std::sync::Mutex<AttentionRouteMask> =
402    std::sync::Mutex::new(AttentionRouteMask {
403        words: [0u64; ATTENTION_MASK_WORDS],
404        active_bits: 0,
405    });
406static ATTENTION_MASK_SEQ: AtomicU32 = AtomicU32::new(0);
407
408#[cfg(not(target_arch = "wasm32"))]
409static PRODUCER_STARTED: AtomicBool = AtomicBool::new(false);
410#[cfg(not(target_arch = "wasm32"))]
411static PRODUCER_STOP: AtomicBool = AtomicBool::new(false);
412
413/// U0 publishes the active decode position for the U1 producer (non-blocking).
414#[inline]
415/// Publish a decode-step hint (token id + step) for the
416/// speculative-decode consumer.
417pub fn publish_decode_hint(token_id: u32, step: u32) {
418    if !epoch::can_publish() {
419        return;
420    }
421    DECODE_TOKEN_ID.store(token_id, Ordering::Relaxed);
422    DECODE_STEP.store(step, Ordering::Release);
423}
424
425/// U0 publishes the 10D query anchor U1 should search around.
426#[inline]
427/// Publish the active query tensor and its subject hash for
428/// cross-universe consumers.
429pub fn publish_query_tensor(tensor: Tensor10D, subject_hash: u64) {
430    if !epoch::can_publish() {
431        return;
432    }
433    *QUERY_TENSOR_CELL.lock().unwrap_or_else(|e| e.into_inner()) = tensor;
434    QUERY_SUBJECT_HASH.store(subject_hash as u32, Ordering::Relaxed);
435    QUERY_SUBJECT_HI.store((subject_hash >> 32) as u32, Ordering::Relaxed);
436    QUERY_SEQ.fetch_add(1, Ordering::Release);
437}
438
439#[inline]
440/// Subject hash of the last published query tensor.
441pub fn query_subject_hash() -> u64 {
442    let lo = QUERY_SUBJECT_HASH.load(Ordering::Relaxed) as u64;
443    let hi = QUERY_SUBJECT_HI.load(Ordering::Relaxed) as u64;
444    (hi << 32) | lo
445}
446
447#[inline]
448/// Load the last published query tensor.
449pub fn load_query_tensor() -> Tensor10D {
450    *QUERY_TENSOR_CELL.lock().unwrap_or_else(|e| e.into_inner())
451}
452
453#[inline]
454/// Current published attention route mask.
455pub fn attention_route_mask() -> AttentionRouteMask {
456    *ATTENTION_MASK_CELL
457        .lock()
458        .unwrap_or_else(|e| e.into_inner())
459}
460
461#[inline]
462/// Sequence counter for the attention-mask publication channel.
463pub fn attention_mask_seq() -> u32 {
464    ATTENTION_MASK_SEQ.load(Ordering::Acquire)
465}
466
467/// Publish U1 route mask snapshot (producer / tests).
468#[inline]
469/// Publish a new attention route mask and bump the sequence
470/// counter.
471pub fn publish_attention_route_mask(mask: AttentionRouteMask) {
472    if !epoch::can_publish() {
473        return;
474    }
475    *ATTENTION_MASK_CELL
476        .lock()
477        .unwrap_or_else(|e| e.into_inner()) = mask;
478    ATTENTION_MASK_SEQ.fetch_add(1, Ordering::Release);
479}
480
481#[inline]
482/// Set bit `slot` in the KV attention mask word array.
483fn set_kv_mask_bit(words: &mut [u32; KV_ATTENTION_MASK_WORDS], slot: u32) {
484    let bit = slot as usize;
485    let word = bit / 32;
486    let offset = bit % 32;
487    if word < KV_ATTENTION_MASK_WORDS {
488        words[word] |= 1u32 << offset;
489    }
490}
491
492/// Map U1 route mask (tensor indices) → KV slot mask for attention (B3.2b light).
493#[inline]
494/// Read KV attention-mask word `index` of the published mask.
495pub fn attention_kv_mask_u32(
496    token_idx: u32,
497    max_context: u32,
498) -> ([u32; KV_ATTENTION_MASK_WORDS], u32) {
499    let route = attention_route_mask();
500    let mut words = [0u32; KV_ATTENTION_MASK_WORDS];
501    let cap = max_context.min((KV_ATTENTION_MASK_WORDS as u32) * 32);
502    set_kv_mask_bit(&mut words, token_idx.min(cap.saturating_sub(1)));
503
504    let provenance = crate::tensor::kv_provenance::global_kv_provenance();
505    let mut mapped = 1u32;
506    for (word_idx, w) in route.words.iter().enumerate() {
507        for bit in 0..64 {
508            if (*w & (1u64 << bit)) == 0 {
509                continue;
510            }
511            let tensor_idx = (word_idx * 64 + bit) as u32;
512            let kv_slot = epoch::scoped_kv_slot(tensor_idx)
513                .unwrap_or_else(|| provenance.kv_slot_for_tensor(tensor_idx))
514                .unwrap_or(tensor_idx);
515            if kv_slot < cap {
516                set_kv_mask_bit(&mut words, kv_slot);
517                mapped += 1;
518            }
519        }
520    }
521
522    let mask_active = if mapped > 1 && route.active_bits > 0 {
523        1u32
524    } else {
525        0u32
526    };
527    (words, mask_active)
528}
529
530/// B3.2a — kNN hits → sparse attention bitmask (zero-heap).
531#[inline]
532/// Build a route mask from tensor-search hits — sets one KV bit
533/// per hit slot.
534pub fn build_attention_route_mask(
535    query: &Tensor10D,
536    max_distance: f32,
537    hits: &[usize],
538    hit_count: usize,
539) -> AttentionRouteMask {
540    let mut mask = AttentionRouteMask::default();
541    for i in 0..hit_count.min(hits.len()) {
542        mask.set_index(hits[i] as u32);
543    }
544    let _ = (query, max_distance);
545    mask
546}
547
548/// One U1 producer cycle: search resident SOA, push ring, refresh mask, draft topology.
549pub fn run_tensor_search_producer_cycle(max_distance: f32, vocab_len: u32) -> usize {
550    epoch::produce(|| run_tensor_search_producer_cycle_in_epoch(max_distance, vocab_len))
551}
552fn run_tensor_search_producer_cycle_in_epoch(max_distance: f32, vocab_len: u32) -> usize {
553    if let Some(source) = epoch::source() {
554        return context::produce_owned(&source, max_distance, vocab_len);
555    }
556    let substrate = global_resident_substrate();
557    if substrate.node_count() == 0 {
558        return 0;
559    }
560
561    let query = load_query_tensor();
562    let subject_hash = query_subject_hash();
563    let mut hits = [0usize; MAX_KNN_HITS];
564    // METRIC PARITY (ALGEBRA_MANIFOLD_PLAN.md Phase 4.1, now unified): the GPU shader
565    // (shaders/tensor_volume.wgsl) ports `Tensor10D::full_distance` exactly — the metric is
566    // chosen by the QUERY's `v` topology class (euclidean / cyclic / hyperbolic / boundary).
567    // So this GPU path and the CPU fallback below (which also uses `full_distance`) agree
568    // for ALL `v`, not only `v == 0`. `volume_gpu::cpu_tensor_search_into` is the shared,
569    // GPU-independent reference for the same metric.
570    let hit_count = {
571        #[cfg(feature = "gpu-runtime")]
572        {
573            crate::tensor::volume_gpu::try_gpu_tensor_search_into(&query, max_distance, &mut hits)
574                .unwrap_or_else(|| {
575                    substrate
576                        .tensor_search_into(&query, max_distance, &mut hits)
577                        .unwrap_or(0)
578                })
579        }
580        #[cfg(not(feature = "gpu-runtime"))]
581        {
582            substrate
583                .tensor_search_into(&query, max_distance, &mut hits)
584                .unwrap_or(0)
585        }
586    };
587
588    let mask = build_attention_route_mask(&query, max_distance, &hits, hit_count);
589    publish_attention_route_mask(mask);
590
591    let pushed = push_tensor_search_hits(&hits[..hit_count], subject_hash, query.w, max_distance);
592    let _ = extrapolate_topology_draft(&query, subject_hash, 4, vocab_len);
593    pushed
594}
595
596#[cfg(not(target_arch = "wasm32"))]
597/// Background U1 producer loop — bounded producer cycles until
598/// `PRODUCER_STOP`, adaptive idle backoff.
599fn tensor_search_producer_loop() {
600    crate::platform_scheduler::bind_background_thread();
601    let mut idle_spins = 0u32;
602    while !PRODUCER_STOP.load(Ordering::Relaxed) {
603        let pushed = run_tensor_search_producer_cycle(3.0, 32_000);
604        if pushed > 0 {
605            crate::gpu_context::record_producer_cycle(pushed as u32);
606            idle_spins = 0;
607        } else if global_resident_substrate().node_count() == 0 {
608            thread::sleep(Duration::from_millis(4));
609        } else {
610            idle_spins = idle_spins.saturating_add(1);
611            if idle_spins > 32 {
612                thread::sleep(Duration::from_millis(1));
613                idle_spins = 0;
614            } else {
615                thread::yield_now();
616            }
617        }
618    }
619}
620
621/// B3.3a — spawn background U1 tensor search producer (idempotent).
622#[cfg(not(target_arch = "wasm32"))]
623pub fn start_tensor_search_producer() -> bool {
624    if PRODUCER_STARTED.swap(true, Ordering::AcqRel) {
625        return false;
626    }
627    PRODUCER_STOP.store(false, Ordering::Release);
628    thread::Builder::new()
629        .name("qualia-u1-tensor-producer".into())
630        .spawn(tensor_search_producer_loop)
631        .expect("U1 tensor search producer thread");
632    true
633}
634
635/// Signal the producer loop to exit (native only).
636#[cfg(not(target_arch = "wasm32"))]
637pub fn stop_tensor_search_producer() {
638    PRODUCER_STOP.store(true, Ordering::Release);
639}
640
641/// WASM variant — no producer thread; always reports not started.
642#[cfg(target_arch = "wasm32")]
643pub fn start_tensor_search_producer() -> bool {
644    false
645}
646
647/// WASM variant — no-op (no producer thread exists).
648#[cfg(target_arch = "wasm32")]
649pub fn stop_tensor_search_producer() {}
650
651/// B2.4 producer hook: push kNN indices from `visit_tensor_search` / `tensor_search_into`.
652#[inline]
653/// Push kNN hit indices into the search-hits channel (drains to
654/// the attention-mask ring).
655pub fn push_tensor_search_hits(
656    indices: &[usize],
657    subject_hash: u64,
658    manifold_w: f32,
659    max_distance: f32,
660) -> usize {
661    let mut pushed = 0usize;
662    for (rank, &index) in indices.iter().enumerate() {
663        if !push_tensor_context(ContextInjectToken {
664            tensor_index: index as u32,
665            subject_hash,
666            distance: max_distance * (1.0 + rank as f32 * 0.01),
667            manifold_w,
668        }) {
669            crate::gpu_context::record_context_ring_drop();
670            break;
671        }
672        pushed += 1;
673    }
674    pushed
675}
676
677// ── B3.1 topological speculative decoding (draft ring scaffold) ───────────────
678
679/// Draft batch pushed U1→U0 for parallel verify (no draft-model weights).
680#[repr(C, align(8))]
681#[derive(Clone, Copy, Debug, PartialEq)]
682pub struct TopologyDraftBatch {
683    pub draft_len: u8,
684    pub _pad: [u8; 7],
685    pub draft_ids: [u32; MAX_DRAFT_LEN],
686    pub concept_hashes: [u64; MAX_DRAFT_LEN],
687}
688
689impl TopologyDraftBatch {
690    /// Empty draft batch (`draft_len` 0).
691    pub const fn empty() -> Self {
692        Self {
693            draft_len: 0,
694            _pad: [0; 7],
695            draft_ids: [0; MAX_DRAFT_LEN],
696            concept_hashes: [0; MAX_DRAFT_LEN],
697        }
698    }
699}
700
701pub struct TopologyDraftRing {
702    epochs: UnsafeCell<[u64; TOPOLOGY_DRAFT_RING_CAP]>,
703    slots: UnsafeCell<[TopologyDraftBatch; TOPOLOGY_DRAFT_RING_CAP]>,
704    write_seq: AtomicU32,
705    read_seq: AtomicU32,
706}
707
708unsafe impl Sync for TopologyDraftRing {}
709
710impl TopologyDraftRing {
711    /// Empty fixed-capacity draft ring.
712    pub const fn new() -> Self {
713        Self {
714            slots: UnsafeCell::new([TopologyDraftBatch::empty(); TOPOLOGY_DRAFT_RING_CAP]),
715            epochs: UnsafeCell::new([0; TOPOLOGY_DRAFT_RING_CAP]),
716            write_seq: AtomicU32::new(0),
717            read_seq: AtomicU32::new(0),
718        }
719    }
720
721    /// Push a draft batch; `false` when the ring is full.
722    pub fn try_push(&self, batch: TopologyDraftBatch) -> bool {
723        if !epoch::can_publish() {
724            return false;
725        }
726        let epoch = epoch::current();
727        if !epoch::accepts(epoch) {
728            return false;
729        }
730        if batch.draft_len == 0 {
731            return false;
732        }
733        let w = self.write_seq.load(Ordering::Relaxed);
734        let r = self.read_seq.load(Ordering::Acquire);
735        if w.wrapping_sub(r) >= TOPOLOGY_DRAFT_RING_CAP as u32 {
736            return false;
737        }
738        let slot = (w % TOPOLOGY_DRAFT_RING_CAP as u32) as usize;
739        unsafe {
740            (*self.slots.get())[slot] = batch;
741            (*self.epochs.get())[slot] = epoch;
742        }
743        self.write_seq.store(w.wrapping_add(1), Ordering::Release);
744        true
745    }
746
747    /// Pop the next pending draft batch.
748    pub fn try_pop(&self) -> Option<TopologyDraftBatch> {
749        loop {
750            let r = self.read_seq.load(Ordering::Relaxed);
751            let w = self.write_seq.load(Ordering::Acquire);
752            if r == w {
753                return None;
754            }
755            let slot = (r % TOPOLOGY_DRAFT_RING_CAP as u32) as usize;
756            let batch = unsafe { (*self.slots.get())[slot] };
757            let epoch = unsafe { (*self.epochs.get())[slot] };
758            self.read_seq.store(r.wrapping_add(1), Ordering::Release);
759            if epoch::accepts(epoch) {
760                return Some(batch);
761            }
762        }
763    }
764}
765
766static TOPOLOGY_DRAFT_RING: TopologyDraftRing = TopologyDraftRing::new();
767
768#[inline]
769/// Global topology-draft ring.
770pub fn topology_draft_ring() -> &'static TopologyDraftRing {
771    &TOPOLOGY_DRAFT_RING
772}
773
774#[inline]
775/// Pop the next topology draft batch from the global ring.
776pub fn pop_topology_draft() -> Option<TopologyDraftBatch> {
777    topology_draft_ring().try_pop()
778}
779
780/// Phase-8 Sentinel gate for U1→U0 topology drafts (B3.1d polish).
781///
782/// Empty drafts are rejected. The previous `0x99` first-byte filter on token ids /
783/// concept hashes was a false positive (~1/256 legitimate tokens) and is removed.
784/// Real governance signals must use an explicit flag / quin rule, not a random byte.
785#[inline]
786/// Sentinel gate: admit the draft batch when it carries at least
787/// one concept.
788pub fn sentinel_allows_topology_draft(batch: &TopologyDraftBatch) -> bool {
789    batch.draft_len > 0
790}
791
792/// Result of speculative verification of a `TopologyDraftBatch`.
793#[derive(Clone, Copy, Debug, PartialEq)]
794pub struct SpeculativeVerifyResult {
795    /// Number of draft tokens accepted in prefix sequence (0..=draft_len).
796    pub accepted_count: usize,
797    /// Next token to emit: either draft token accepted + 1 bonus token,
798    /// or the target model's correction token at the first rejected position.
799    pub emitted_token: u32,
800    /// Whether all draft tokens were accepted without rejection.
801    pub fully_accepted: bool,
802}
803
804/// Verify a speculative `TopologyDraftBatch` against target model greedy predictions
805/// or logit argmax tokens.
806///
807/// For each draft token $i \in [0, \text{draft\_len})$, target model predicts $t^*_i$.
808/// If $t_i == t^*_i$, draft token is accepted.
809/// Upon first divergence at index $j$, acceptance stops; $j$ tokens are accepted,
810/// and $t^*_j$ is emitted as the correction token.
811/// If all draft tokens are accepted, all $\text{draft\_len}$ tokens are accepted,
812/// and bonus token $t^*_{\text{draft\_len}}$ is emitted.
813pub fn verify_topology_draft(
814    draft: &TopologyDraftBatch,
815    target_predictions: &[u32],
816) -> SpeculativeVerifyResult {
817    let len = (draft.draft_len as usize)
818        .min(target_predictions.len())
819        .min(MAX_DRAFT_LEN);
820    let mut accepted = 0;
821    while accepted < len {
822        if draft.draft_ids[accepted] == target_predictions[accepted] {
823            accepted += 1;
824        } else {
825            break;
826        }
827    }
828
829    let fully_accepted = accepted == draft.draft_len as usize;
830    let emitted_token = if accepted < target_predictions.len() {
831        target_predictions[accepted]
832    } else if accepted > 0 {
833        draft.draft_ids[accepted - 1]
834    } else {
835        0
836    };
837
838    SpeculativeVerifyResult {
839        accepted_count: accepted,
840        emitted_token,
841        fully_accepted,
842    }
843}
844
845/// Extrapolate γ concept hashes from kNN trajectory (B3.1b); optional `TopologyDraftMapper` (B3.1c).
846pub fn extrapolate_topology_draft(
847    query: &Tensor10D,
848    subject_hash: u64,
849    gamma: usize,
850    vocab_len: u32,
851) -> Option<TopologyDraftBatch> {
852    #[cfg(any(not(target_arch = "wasm32"), feature = "wasm-llm"))]
853    {
854        extrapolate_topology_draft_mapped(query, subject_hash, gamma, vocab_len, None)
855    }
856    #[cfg(all(target_arch = "wasm32", not(feature = "wasm-llm")))]
857    {
858        extrapolate_topology_draft_mapped(query, subject_hash, gamma, vocab_len)
859    }
860}
861
862/// Core draft extrapolation: kNN search over the resident
863/// substrate (GPU path when `gpu-runtime`), XOR concept hashes with
864/// `subject_hash`, optional `TopologyDraftMapper` for token ids,
865/// then push onto the draft ring. `None` when the substrate is
866/// empty or no hits.
867pub fn extrapolate_topology_draft_mapped(
868    query: &Tensor10D,
869    subject_hash: u64,
870    gamma: usize,
871    vocab_len: u32,
872    #[cfg(any(not(target_arch = "wasm32"), feature = "wasm-llm"))] mapper: Option<
873        &crate::topology_draft::TopologyDraftMapper<'_>,
874    >,
875) -> Option<TopologyDraftBatch> {
876    let gamma = gamma.clamp(1, MAX_DRAFT_LEN);
877    let substrate = global_resident_substrate();
878    if substrate.node_count() == 0 || vocab_len == 0 {
879        return None;
880    }
881    let mut hits = [0usize; MAX_KNN_HITS];
882    let hit_count = {
883        #[cfg(feature = "gpu-runtime")]
884        {
885            crate::tensor::volume_gpu::try_gpu_tensor_search_into(query, 4.0, &mut hits)
886                .unwrap_or_else(|| {
887                    substrate
888                        .tensor_search_into(query, 4.0, &mut hits)
889                        .unwrap_or(0)
890                })
891        }
892        #[cfg(not(feature = "gpu-runtime"))]
893        {
894            substrate
895                .tensor_search_into(query, 4.0, &mut hits)
896                .unwrap_or(0)
897        }
898    };
899    if hit_count == 0 {
900        return None;
901    }
902
903    let mut concepts = [0u64; MAX_DRAFT_LEN];
904    for i in 0..gamma.min(hit_count) {
905        concepts[i] = substrate.subject_hash_at(hits[i] as u32) ^ subject_hash;
906    }
907    let batch = {
908        #[cfg(any(not(target_arch = "wasm32"), feature = "wasm-llm"))]
909        if let Some(m) = mapper {
910            m.fill_draft_batch(&concepts[..gamma.min(hit_count)], gamma.min(hit_count))
911        } else {
912            let mut batch = TopologyDraftBatch::empty();
913            for i in 0..gamma.min(hit_count) {
914                batch.concept_hashes[i] = concepts[i];
915                batch.draft_ids[i] = (concepts[i] as u32) % vocab_len.max(1);
916            }
917            batch.draft_len = gamma.min(hit_count) as u8;
918            batch
919        }
920        #[cfg(all(target_arch = "wasm32", not(feature = "wasm-llm")))]
921        {
922            let mut batch = TopologyDraftBatch::empty();
923            for i in 0..gamma.min(hit_count) {
924                batch.concept_hashes[i] = concepts[i];
925                batch.draft_ids[i] = (concepts[i] as u32) % vocab_len.max(1);
926            }
927            batch.draft_len = gamma.min(hit_count) as u8;
928            batch
929        }
930    };
931    let _ = topology_draft_ring().try_push(batch);
932    Some(batch)
933}
934
935#[cfg(test)]
936#[path = "compute_universe_test_support.rs"]
937mod test_support;
938
939#[cfg(test)]
940#[path = "compute_universe_tests.rs"]
941mod tests;