qualia_core_db/inference/inference_agent/decode/
mod.rs1#[cfg(not(target_arch = "wasm32"))]
10use super::config::effective_inference_timeout_ms;
11#[cfg(any(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
12use super::config::{DECODE_TOKEN_BUDGET, TEST_TRANSFORMER_LAYER_CAP, TEST_VOCAB_CHUNK_CAP};
13use super::control::DecodeControl;
14#[cfg(not(target_arch = "wasm32"))]
15use super::decode_helpers::get_prefix_cache;
16#[cfg(any(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
17#[allow(unused_imports)]
18use super::decode_helpers::{
19 apply_model_helper_stops, build_sieve, drain_tensor_context_inject, embedding_fallback_logits,
20 try_accept_topology_draft, TopologyDraftStep,
21};
22use super::local_agent::LocalLlmAgent;
23#[cfg(not(target_arch = "wasm32"))]
24use super::sticky_infer;
25#[allow(unused_imports)]
26use super::types::AgentBackend;
27#[cfg(not(target_arch = "wasm32"))]
28use crate::inference::runtime::stage::{
29 InferenceStageKind, StageStatus, StageTrace, ThinkingEvent, ThinkingTracker,
30};
31#[allow(unused_imports)]
32use crate::{q_hash, NQuin};
33
34impl LocalLlmAgent {
35 pub fn infer_local_model_streaming<F: FnMut(String) + Send + 'static>(
47 &self,
48 prompt: &str,
49 graph_context: &str,
50 on_token: Option<F>,
51 ) -> (String, Vec<u64>, u32, Option<NQuin>) {
52 self.infer_local_model_inner(prompt, graph_context, None, on_token, None)
53 }
54
55 pub fn infer_local_model_controlled<F: FnMut(String) + Send + 'static>(
57 &self,
58 prompt: &str,
59 graph_context: &str,
60 control: DecodeControl,
61 on_token: Option<F>,
62 ) -> (String, Vec<u64>, u32, Option<NQuin>) {
63 self.infer_local_model_inner(prompt, graph_context, Some(control), on_token, None)
64 }
65
66 pub(super) fn infer_local_model(
68 &self,
69 prompt: &str,
70 graph_context: &str,
71 ) -> (String, Vec<u64>, u32, Option<NQuin>) {
72 self.infer_local_model_inner::<fn(String)>(prompt, graph_context, None, None, None)
73 }
74
75 #[cfg(not(target_arch = "wasm32"))]
78 pub fn infer_local_model_with_execution<F: FnMut(String) + Send + 'static>(
79 &self,
80 prompt: &str,
81 graph_context: &str,
82 execution: super::native_request::NativeDecodeRequest,
83 control: Option<DecodeControl>,
84 on_token: Option<F>,
85 ) -> (String, Vec<u64>, u32, Option<NQuin>) {
86 self.infer_local_model_inner(prompt, graph_context, control, on_token, Some(execution))
87 }
88
89 #[cfg_attr(target_arch = "wasm32", allow(unused_variables, unused_mut))]
90 fn infer_local_model_inner<F: FnMut(String) + Send + 'static>(
91 &self,
92 prompt: &str,
93 graph_context: &str,
94 control: Option<DecodeControl>,
95 on_token: Option<F>,
96 #[cfg(not(target_arch = "wasm32"))] execution: Option<
97 super::native_request::NativeDecodeRequest,
98 >,
99 #[cfg(target_arch = "wasm32")] _execution: Option<()>,
100 ) -> (String, Vec<u64>, u32, Option<NQuin>) {
101 let prov_hash = crate::q_hash(graph_context);
102 let use_sieve = self
103 .use_sieve_output
104 .load(std::sync::atomic::Ordering::Relaxed);
105 let sieve_spec = if use_sieve {
106 Some(*self.sieve_spec.lock().unwrap_or_else(|e| e.into_inner()))
107 } else {
108 None
109 };
110 let sieve_lex_path = if use_sieve {
111 self.sieve_lex_path
112 .lock()
113 .unwrap_or_else(|e| e.into_inner())
114 .clone()
115 } else {
116 None
117 };
118
119 #[cfg(not(target_arch = "wasm32"))]
120 {
121 self.infer_native_partition(
122 prompt,
123 graph_context,
124 control,
125 on_token,
126 prov_hash,
127 use_sieve,
128 sieve_spec,
129 sieve_lex_path,
130 execution,
131 )
132 }
133 #[cfg(target_arch = "wasm32")]
134 {
135 self.infer_wasm_partition(
136 prompt,
137 graph_context,
138 control,
139 on_token,
140 prov_hash,
141 use_sieve,
142 sieve_spec,
143 sieve_lex_path,
144 )
145 }
146 }
147}
148
149#[cfg(target_arch = "wasm32")]
150mod browser;
151#[cfg(not(target_arch = "wasm32"))]
152mod native;
153
154#[cfg(not(target_arch = "wasm32"))]
155type WorkerResult = (String, u32, Option<NQuin>, bool, StageTrace);
156
157#[cfg(not(target_arch = "wasm32"))]
158mod messages {
159 #[derive(Clone, Copy)]
161 pub(super) struct LogitSummary {
162 pub(super) _top_id: u32,
163 pub(super) anomaly: u8,
164 }
165 #[derive(Clone)]
166 pub(super) enum LlmMsg {
167 Logit(LogitSummary),
168 Eos,
169 }
170 #[derive(Clone)]
171 pub(super) enum SentMsg {
172 DenyRollback,
173 }
174}
175#[cfg(not(target_arch = "wasm32"))]
176use messages::{LlmMsg, LogitSummary, SentMsg};
177#[cfg(not(target_arch = "wasm32"))]
178mod admission;
179#[cfg(all(target_arch = "wasm32", feature = "gpu-runtime"))]
180mod browser_select;
181#[cfg(all(test, not(target_arch = "wasm32")))]
182mod cancellation_tests;
183#[cfg(not(target_arch = "wasm32"))]
184mod cold;
185#[cfg(not(target_arch = "wasm32"))]
186mod diagnostic;
187#[cfg(not(target_arch = "wasm32"))]
188mod forced;
189#[cfg(not(target_arch = "wasm32"))]
190mod model;
191#[cfg(not(target_arch = "wasm32"))]
192mod prefill;
193#[cfg(not(target_arch = "wasm32"))]
194mod publication;
195#[cfg(not(target_arch = "wasm32"))]
196mod qwen35;
197#[cfg(not(target_arch = "wasm32"))]
198mod render;
199#[cfg(not(target_arch = "wasm32"))]
200mod select;
201#[cfg(not(target_arch = "wasm32"))]
202mod speculative;
203#[cfg(not(target_arch = "wasm32"))]
204mod worker;