1use super::dense_host::DenseHost;
4use super::{weights::DenseSource, Plan};
5use etalon_inference::ordinary::qwen35::{self, Host};
6use etalon_model::inference::{qwen_attention::QkNorm, TensorInfo};
7use std::sync::Arc;
8
9#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10pub enum SessionError {
11 Budget,
12 Shape,
13 Invalidated,
14 Position,
15 ContextFull,
16 Provider,
17 Numerical,
18 Canceled,
19}
20impl From<qwen35::Error> for SessionError {
21 fn from(_: qwen35::Error) -> Self {
22 Self::Provider
23 }
24}
25
26#[derive(Default)]
27struct LayerState {
28 history: Vec<f32>,
29 recurrent: Vec<f32>,
30 keys: Vec<f32>,
31 values: Vec<f32>,
32}
33struct Buffers {
34 layers: Vec<LayerState>,
35 workspace: Vec<f32>,
36 normalized: Vec<f32>,
37 norm: Vec<f32>,
38 router: Vec<f32>,
39 gate_up: Vec<f32>,
40 gate: Vec<f32>,
41 up: Vec<f32>,
42 swiglu: Vec<f32>,
43 expert: Vec<f32>,
44 combined: Vec<f32>,
45 logits: Vec<f32>,
46}
47#[derive(Default)]
48struct Sequence {
49 cursor: usize,
50 valid: bool,
51}
52struct Token<'a> {
53 sequence: &'a mut Sequence,
54 committed: bool,
55}
56impl Token<'_> {
57 fn commit(mut self) {
58 self.sequence.cursor += 1;
59 self.committed = true;
60 }
61 fn commit_partial(mut self) {
62 self.committed = true;
63 }
64}
65impl Drop for Token<'_> {
66 fn drop(&mut self) {
67 if !self.committed {
68 self.sequence.valid = false;
69 }
70 }
71}
72
73pub struct Session {
74 plan: Arc<Plan>,
75 buffers: Buffers,
76 sequence: Sequence,
77 reserved_bytes: usize,
78}
79fn add(a: usize, b: usize) -> Result<usize, SessionError> {
80 a.checked_add(b).ok_or(SessionError::Budget)
81}
82fn mul(a: usize, b: usize) -> Result<usize, SessionError> {
83 a.checked_mul(b).ok_or(SessionError::Budget)
84}
85fn zeros(n: usize) -> Result<Vec<f32>, SessionError> {
86 let mut result = Vec::new();
87 result
88 .try_reserve_exact(n)
89 .map_err(|_| SessionError::Budget)?;
90 result.resize(n, 0.0);
91 Ok(result)
92}
93pub(super) fn finite(values: &[f32]) -> Result<(), SessionError> {
94 if values.iter().all(|v| v.is_finite()) {
95 Ok(())
96 } else {
97 Err(SessionError::Numerical)
98 }
99}
100
101impl Session {
102 pub fn plan(&self) -> &Arc<Plan> {
103 &self.plan
104 }
105 pub fn is_valid(&self) -> bool {
106 self.sequence.valid
107 }
108 pub fn new(plan: Arc<Plan>, byte_budget: usize) -> Result<Self, SessionError> {
111 let p = &plan.profile;
112 if p.hidden == 0
113 || p.max_context == 0
114 || p.layers != plan.layers.len()
115 || p.layers == 0
116 || p.intermediate == 0
117 || p.vocab == 0
118 {
119 return Err(SessionError::Shape);
120 }
121 let mut floats = 0;
122 let mut workspace = 0;
123 for layer in &plan.layers {
124 let r = layer.attention.requirements(p.max_context)?;
125 floats = add(floats, add(add(r.history, r.recurrent)?, mul(r.kv, 2)?)?)?;
126 workspace = workspace.max(r.workspace);
127 }
128 floats = add(
130 floats,
131 add(
132 add(mul(p.hidden, 4)?, mul(p.intermediate, 5)?)?,
133 add(add(p.experts, p.vocab)?, workspace)?,
134 )?,
135 )?;
136 let reserved_bytes = add(
137 add(
138 mul(floats, 4)?,
139 mul(p.layers, std::mem::size_of::<LayerState>())?,
140 )?,
141 std::mem::size_of::<Self>(),
142 )?;
143 if reserved_bytes > byte_budget {
144 return Err(SessionError::Budget);
145 }
146 let mut layers = Vec::new();
147 layers
148 .try_reserve_exact(p.layers)
149 .map_err(|_| SessionError::Budget)?;
150 for layer in &plan.layers {
151 let r = layer.attention.requirements(p.max_context)?;
152 layers.push(LayerState {
153 history: zeros(r.history)?,
154 recurrent: zeros(r.recurrent)?,
155 keys: zeros(r.kv)?,
156 values: zeros(r.kv)?,
157 });
158 }
159 let buffers = Buffers {
160 layers,
161 workspace: zeros(workspace)?,
162 normalized: zeros(p.hidden)?,
163 norm: zeros(p.hidden)?,
164 router: zeros(p.experts)?,
165 gate_up: zeros(mul(p.intermediate, 2)?)?,
166 gate: zeros(p.intermediate)?,
167 up: zeros(p.intermediate)?,
168 swiglu: zeros(p.intermediate)?,
169 expert: zeros(p.hidden)?,
170 combined: zeros(p.hidden)?,
171 logits: zeros(p.vocab)?,
172 };
173 Ok(Self {
174 plan,
175 buffers,
176 sequence: Sequence {
177 cursor: 0,
178 valid: true,
179 },
180 reserved_bytes,
181 })
182 }
183 pub fn reserved_bytes(&self) -> usize {
184 self.reserved_bytes
185 }
186 pub fn kv_cache_bytes(&self) -> usize {
188 self.buffers
191 .layers
192 .iter()
193 .map(|state| (state.keys.len() + state.values.len()) * std::mem::size_of::<f32>())
194 .sum()
195 }
196 pub fn position(&self) -> Result<usize, SessionError> {
197 if self.sequence.valid {
198 Ok(self.sequence.cursor)
199 } else {
200 Err(SessionError::Invalidated)
201 }
202 }
203 pub fn logits(&self) -> Result<&[f32], SessionError> {
204 self.position()?;
205 Ok(&self.buffers.logits)
206 }
207 pub fn reset(&mut self) {
208 for state in &mut self.buffers.layers {
209 state.history.fill(0.0);
210 state.recurrent.fill(0.0);
211 state.keys.fill(0.0);
212 state.values.fill(0.0);
213 }
214 let b = &mut self.buffers;
215 for values in [
216 &mut b.workspace,
217 &mut b.normalized,
218 &mut b.norm,
219 &mut b.router,
220 &mut b.gate_up,
221 &mut b.gate,
222 &mut b.up,
223 &mut b.swiglu,
224 &mut b.expert,
225 &mut b.combined,
226 &mut b.logits,
227 ] {
228 values.fill(0.0);
229 }
230 self.sequence = Sequence {
231 cursor: 0,
232 valid: true,
233 };
234 }
235 pub fn embed(&self, token: u32, hidden: &mut [f32]) -> Result<(), SessionError> {
236 self.position()?;
237 let p = &self.plan.profile;
238 if token as usize >= p.vocab || hidden.len() != p.hidden {
239 return Err(SessionError::Shape);
240 }
241 let mut row = self.plan.embedding;
242 let width = match row.encoding() {
243 etalon_model::inference::WeightEncoding::IeeeF32 => 4,
244 etalon_model::inference::WeightEncoding::IeeeF16
245 | etalon_model::inference::WeightEncoding::BrainF16 => 2,
246 _ => return Err(SessionError::Shape),
247 };
248 let offset = mul(mul(token as usize, p.hidden)?, width)?;
249 row.byte_offset = row
250 .byte_offset
251 .checked_add(offset as u64)
252 .ok_or(SessionError::Shape)?;
253 row.n_dims = 1;
254 row.dims = [p.hidden as u64, 0, 0, 0];
255 let source = DenseSource(&self.plan.pkg);
256 hidden.fill(f32::NAN);
257 DenseHost { source: &source }.read(&row, hidden)?;
258 finite(hidden)
259 }
260 pub fn step(
263 &mut self,
264 hidden: &mut [f32],
265 position: usize,
266 ) -> Result<(u32, f32), SessionError> {
267 self.step_controlled(hidden, position, &|| false)
268 }
269 pub fn step_controlled(
270 &mut self,
271 hidden: &mut [f32],
272 position: usize,
273 canceled: &dyn Fn() -> bool,
274 ) -> Result<(u32, f32), SessionError> {
275 if !self.sequence.valid {
276 return Err(SessionError::Invalidated);
277 }
278 if position != self.sequence.cursor {
279 return Err(SessionError::Position);
280 }
281 if position >= self.plan.profile.max_context {
282 return Err(SessionError::ContextFull);
283 }
284 if hidden.len() != self.plan.profile.hidden {
285 return Err(SessionError::Shape);
286 }
287 let token = Token {
288 sequence: &mut self.sequence,
289 committed: false,
290 };
291 let result = execute(&self.plan, &mut self.buffers, hidden, position, canceled)?;
292 token.commit();
293 Ok(result)
294 }
295 pub fn prefill_controlled(
297 &mut self,
298 hidden: &mut [f32],
299 position: usize,
300 canceled: &dyn Fn() -> bool,
301 ) -> Result<(), SessionError> {
302 if !self.sequence.valid {
303 return Err(SessionError::Invalidated);
304 }
305 if position != self.sequence.cursor {
306 return Err(SessionError::Position);
307 }
308 if position >= self.plan.profile.max_context {
309 return Err(SessionError::ContextFull);
310 }
311 if hidden.len() != self.plan.profile.hidden {
312 return Err(SessionError::Shape);
313 }
314 let token = Token {
315 sequence: &mut self.sequence,
316 committed: false,
317 };
318 execute_layers(&self.plan, &mut self.buffers, hidden, position, canceled)?;
319 token.commit();
320 Ok(())
321 }
322}
323
324fn normalize(
325 host: &impl Host,
326 info: &TensorInfo,
327 hidden: &[f32],
328 norm: &mut [f32],
329 normalized: &mut [f32],
330 convention: QkNorm,
331 epsilon: f32,
332) -> Result<(), SessionError> {
333 norm.fill(f32::NAN);
334 host.read(info, norm)?;
335 qwen35::ordinary_rms_norm(
336 hidden,
337 norm,
338 convention == QkNorm::RawZeroCentered,
339 epsilon,
340 normalized,
341 )?;
342 Ok(())
343}
344fn execute(
345 plan: &Plan,
346 b: &mut Buffers,
347 hidden: &mut [f32],
348 position: usize,
349 canceled: &dyn Fn() -> bool,
350) -> Result<(u32, f32), SessionError> {
351 let p = &plan.profile;
352 let source = DenseSource(&plan.pkg);
353 let host = DenseHost { source: &source };
354 finite(hidden)?;
355 execute_layers(plan, b, hidden, position, canceled)?;
356 normalize(
357 &host,
358 &plan.final_norm,
359 hidden,
360 &mut b.norm,
361 &mut b.normalized,
362 p.ordinary_norm,
363 p.epsilon,
364 )?;
365 let result = project_output(plan, &mut b.logits, &b.normalized)?;
366 if canceled() {
367 return Err(SessionError::Canceled);
368 }
369 Ok(result)
370}
371fn execute_layers(
372 plan: &Plan,
373 b: &mut Buffers,
374 hidden: &mut [f32],
375 position: usize,
376 canceled: &dyn Fn() -> bool,
377) -> Result<(), SessionError> {
378 finite(hidden)?;
379 for index in 0..plan.layers.len() {
380 if canceled() {
381 return Err(SessionError::Canceled);
382 }
383 execute_attention(plan, b, index, hidden, position)?;
384 if canceled() {
385 return Err(SessionError::Canceled);
386 }
387 execute_ffn(plan, b, index, hidden)?;
388 }
389 if canceled() {
390 return Err(SessionError::Canceled);
391 }
392 Ok(())
393}
394fn execute_attention(
395 plan: &Plan,
396 b: &mut Buffers,
397 index: usize,
398 hidden: &mut [f32],
399 position: usize,
400) -> Result<(), SessionError> {
401 let source = DenseSource(&plan.pkg);
402 let host = DenseHost { source: &source };
403 let layer = &plan.layers[index];
404 let state = &mut b.layers[index];
405 let p = &plan.profile;
406 normalize(
407 &host,
408 &layer.input_norm,
409 hidden,
410 &mut b.norm,
411 &mut b.normalized,
412 p.ordinary_norm,
413 p.epsilon,
414 )?;
415 layer.attention.execute(
416 &host,
417 &b.normalized,
418 hidden,
419 &mut state.history,
420 &mut state.recurrent,
421 &mut state.keys,
422 &mut state.values,
423 position,
424 &p.rotary_inv_freq,
425 p.epsilon,
426 &mut b.workspace,
427 )?;
428 Ok(())
429}
430fn execute_ffn(
431 plan: &Plan,
432 b: &mut Buffers,
433 index: usize,
434 hidden: &mut [f32],
435) -> Result<(), SessionError> {
436 let source = DenseSource(&plan.pkg);
437 let host = DenseHost { source: &source };
438 let p = &plan.profile;
439 normalize(
440 &host,
441 &plan.layers[index].post_norm,
442 hidden,
443 &mut b.norm,
444 &mut b.normalized,
445 p.ordinary_norm,
446 p.epsilon,
447 )?;
448 super::session_ffn::execute(
449 plan,
450 index,
451 &host,
452 &b.normalized,
453 &mut b.router,
454 &mut b.gate_up,
455 &mut b.gate,
456 &mut b.up,
457 &mut b.swiglu,
458 &mut b.expert,
459 &mut b.combined,
460 )?;
461 for (value, delta) in hidden.iter_mut().zip(&b.combined) {
462 *value += delta;
463 }
464 finite(hidden)
465}
466fn project_output(
467 plan: &Plan,
468 logits: &mut [f32],
469 hidden: &[f32],
470) -> Result<(u32, f32), SessionError> {
471 plan.output
472 .gemv(&plan.pkg, hidden, logits)
473 .map_err(|_| SessionError::Provider)?;
474 finite(logits)?;
475 let mut best = 0;
476 for i in 1..logits.len() {
477 if logits[i] > logits[best] {
478 best = i;
479 }
480 }
481 Ok((best as u32, logits[best]))
482}
483
484#[cfg(test)]
485#[path = "session_tests.rs"]
486mod tests;
487
488#[path = "session_lab.rs"]
489mod lab;