Skip to main content

qualia_core_db/gguf_bridge/qwen35/ftw/
session.rs

1//! Caller-budgeted cold state and allocation-free mixed-attention token execution.
2//! A token commits only after every layer, final norm, and output projection.
3use super::dense_host::DenseHost;
4use super::{weights::DenseSource, Plan};
5use etalon_inference::ordinary::qwen35::{self, Host};
6use etalon_model::inference::{qwen_attention::QkNorm, TensorInfo};
7use std::sync::Arc;
8
9#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10pub enum SessionError {
11    Budget,
12    Shape,
13    Invalidated,
14    Position,
15    ContextFull,
16    Provider,
17    Numerical,
18    Canceled,
19}
20impl From<qwen35::Error> for SessionError {
21    fn from(_: qwen35::Error) -> Self {
22        Self::Provider
23    }
24}
25
26#[derive(Default)]
27struct LayerState {
28    history: Vec<f32>,
29    recurrent: Vec<f32>,
30    keys: Vec<f32>,
31    values: Vec<f32>,
32}
33struct Buffers {
34    layers: Vec<LayerState>,
35    workspace: Vec<f32>,
36    normalized: Vec<f32>,
37    norm: Vec<f32>,
38    router: Vec<f32>,
39    gate_up: Vec<f32>,
40    gate: Vec<f32>,
41    up: Vec<f32>,
42    swiglu: Vec<f32>,
43    expert: Vec<f32>,
44    combined: Vec<f32>,
45    logits: Vec<f32>,
46}
47#[derive(Default)]
48struct Sequence {
49    cursor: usize,
50    valid: bool,
51}
52struct Token<'a> {
53    sequence: &'a mut Sequence,
54    committed: bool,
55}
56impl Token<'_> {
57    fn commit(mut self) {
58        self.sequence.cursor += 1;
59        self.committed = true;
60    }
61    fn commit_partial(mut self) {
62        self.committed = true;
63    }
64}
65impl Drop for Token<'_> {
66    fn drop(&mut self) {
67        if !self.committed {
68            self.sequence.valid = false;
69        }
70    }
71}
72
73pub struct Session {
74    plan: Arc<Plan>,
75    buffers: Buffers,
76    sequence: Sequence,
77    reserved_bytes: usize,
78}
79fn add(a: usize, b: usize) -> Result<usize, SessionError> {
80    a.checked_add(b).ok_or(SessionError::Budget)
81}
82fn mul(a: usize, b: usize) -> Result<usize, SessionError> {
83    a.checked_mul(b).ok_or(SessionError::Budget)
84}
85fn zeros(n: usize) -> Result<Vec<f32>, SessionError> {
86    let mut result = Vec::new();
87    result
88        .try_reserve_exact(n)
89        .map_err(|_| SessionError::Budget)?;
90    result.resize(n, 0.0);
91    Ok(result)
92}
93pub(super) fn finite(values: &[f32]) -> Result<(), SessionError> {
94    if values.iter().all(|v| v.is_finite()) {
95        Ok(())
96    } else {
97        Err(SessionError::Numerical)
98    }
99}
100
101impl Session {
102    pub fn plan(&self) -> &Arc<Plan> {
103        &self.plan
104    }
105    pub fn is_valid(&self) -> bool {
106        self.sequence.valid
107    }
108    /// The byte budget covers newly allocated state/scratch and owner metadata.
109    /// Immutable admitted weights remain owned by the separately budgeted plan.
110    pub fn new(plan: Arc<Plan>, byte_budget: usize) -> Result<Self, SessionError> {
111        let p = &plan.profile;
112        if p.hidden == 0
113            || p.max_context == 0
114            || p.layers != plan.layers.len()
115            || p.layers == 0
116            || p.intermediate == 0
117            || p.vocab == 0
118        {
119            return Err(SessionError::Shape);
120        }
121        let mut floats = 0;
122        let mut workspace = 0;
123        for layer in &plan.layers {
124            let r = layer.attention.requirements(p.max_context)?;
125            floats = add(floats, add(add(r.history, r.recurrent)?, mul(r.kv, 2)?)?)?;
126            workspace = workspace.max(r.workspace);
127        }
128        // norm/normalized/expert/combined + gate_up/gate/up/swiglu.
129        floats = add(
130            floats,
131            add(
132                add(mul(p.hidden, 4)?, mul(p.intermediate, 5)?)?,
133                add(add(p.experts, p.vocab)?, workspace)?,
134            )?,
135        )?;
136        let reserved_bytes = add(
137            add(
138                mul(floats, 4)?,
139                mul(p.layers, std::mem::size_of::<LayerState>())?,
140            )?,
141            std::mem::size_of::<Self>(),
142        )?;
143        if reserved_bytes > byte_budget {
144            return Err(SessionError::Budget);
145        }
146        let mut layers = Vec::new();
147        layers
148            .try_reserve_exact(p.layers)
149            .map_err(|_| SessionError::Budget)?;
150        for layer in &plan.layers {
151            let r = layer.attention.requirements(p.max_context)?;
152            layers.push(LayerState {
153                history: zeros(r.history)?,
154                recurrent: zeros(r.recurrent)?,
155                keys: zeros(r.kv)?,
156                values: zeros(r.kv)?,
157            });
158        }
159        let buffers = Buffers {
160            layers,
161            workspace: zeros(workspace)?,
162            normalized: zeros(p.hidden)?,
163            norm: zeros(p.hidden)?,
164            router: zeros(p.experts)?,
165            gate_up: zeros(mul(p.intermediate, 2)?)?,
166            gate: zeros(p.intermediate)?,
167            up: zeros(p.intermediate)?,
168            swiglu: zeros(p.intermediate)?,
169            expert: zeros(p.hidden)?,
170            combined: zeros(p.hidden)?,
171            logits: zeros(p.vocab)?,
172        };
173        Ok(Self {
174            plan,
175            buffers,
176            sequence: Sequence {
177                cursor: 0,
178                valid: true,
179            },
180            reserved_bytes,
181        })
182    }
183    pub fn reserved_bytes(&self) -> usize {
184        self.reserved_bytes
185    }
186    /// Allocated keys and values, excluding recurrent state and scratch.
187    pub fn kv_cache_bytes(&self) -> usize {
188        // Construction checked the aggregate float/byte count against the
189        // selected budget before allocation; these private slices never grow.
190        self.buffers
191            .layers
192            .iter()
193            .map(|state| (state.keys.len() + state.values.len()) * std::mem::size_of::<f32>())
194            .sum()
195    }
196    pub fn position(&self) -> Result<usize, SessionError> {
197        if self.sequence.valid {
198            Ok(self.sequence.cursor)
199        } else {
200            Err(SessionError::Invalidated)
201        }
202    }
203    pub fn logits(&self) -> Result<&[f32], SessionError> {
204        self.position()?;
205        Ok(&self.buffers.logits)
206    }
207    pub fn reset(&mut self) {
208        for state in &mut self.buffers.layers {
209            state.history.fill(0.0);
210            state.recurrent.fill(0.0);
211            state.keys.fill(0.0);
212            state.values.fill(0.0);
213        }
214        let b = &mut self.buffers;
215        for values in [
216            &mut b.workspace,
217            &mut b.normalized,
218            &mut b.norm,
219            &mut b.router,
220            &mut b.gate_up,
221            &mut b.gate,
222            &mut b.up,
223            &mut b.swiglu,
224            &mut b.expert,
225            &mut b.combined,
226            &mut b.logits,
227        ] {
228            values.fill(0.0);
229        }
230        self.sequence = Sequence {
231            cursor: 0,
232            valid: true,
233        };
234    }
235    pub fn embed(&self, token: u32, hidden: &mut [f32]) -> Result<(), SessionError> {
236        self.position()?;
237        let p = &self.plan.profile;
238        if token as usize >= p.vocab || hidden.len() != p.hidden {
239            return Err(SessionError::Shape);
240        }
241        let mut row = self.plan.embedding;
242        let width = match row.encoding() {
243            etalon_model::inference::WeightEncoding::IeeeF32 => 4,
244            etalon_model::inference::WeightEncoding::IeeeF16
245            | etalon_model::inference::WeightEncoding::BrainF16 => 2,
246            _ => return Err(SessionError::Shape),
247        };
248        let offset = mul(mul(token as usize, p.hidden)?, width)?;
249        row.byte_offset = row
250            .byte_offset
251            .checked_add(offset as u64)
252            .ok_or(SessionError::Shape)?;
253        row.n_dims = 1;
254        row.dims = [p.hidden as u64, 0, 0, 0];
255        let source = DenseSource(&self.plan.pkg);
256        hidden.fill(f32::NAN);
257        DenseHost { source: &source }.read(&row, hidden)?;
258        finite(hidden)
259    }
260    /// Hidden initially contains the embedding for this position. A partial
261    /// error or unwind poisons the entire sequence; callers must reset or drop it.
262    pub fn step(
263        &mut self,
264        hidden: &mut [f32],
265        position: usize,
266    ) -> Result<(u32, f32), SessionError> {
267        self.step_controlled(hidden, position, &|| false)
268    }
269    pub fn step_controlled(
270        &mut self,
271        hidden: &mut [f32],
272        position: usize,
273        canceled: &dyn Fn() -> bool,
274    ) -> Result<(u32, f32), SessionError> {
275        if !self.sequence.valid {
276            return Err(SessionError::Invalidated);
277        }
278        if position != self.sequence.cursor {
279            return Err(SessionError::Position);
280        }
281        if position >= self.plan.profile.max_context {
282            return Err(SessionError::ContextFull);
283        }
284        if hidden.len() != self.plan.profile.hidden {
285            return Err(SessionError::Shape);
286        }
287        let token = Token {
288            sequence: &mut self.sequence,
289            committed: false,
290        };
291        let result = execute(&self.plan, &mut self.buffers, hidden, position, canceled)?;
292        token.commit();
293        Ok(result)
294    }
295    /// Advance prompt state without projecting unused vocabulary logits.
296    pub fn prefill_controlled(
297        &mut self,
298        hidden: &mut [f32],
299        position: usize,
300        canceled: &dyn Fn() -> bool,
301    ) -> Result<(), SessionError> {
302        if !self.sequence.valid {
303            return Err(SessionError::Invalidated);
304        }
305        if position != self.sequence.cursor {
306            return Err(SessionError::Position);
307        }
308        if position >= self.plan.profile.max_context {
309            return Err(SessionError::ContextFull);
310        }
311        if hidden.len() != self.plan.profile.hidden {
312            return Err(SessionError::Shape);
313        }
314        let token = Token {
315            sequence: &mut self.sequence,
316            committed: false,
317        };
318        execute_layers(&self.plan, &mut self.buffers, hidden, position, canceled)?;
319        token.commit();
320        Ok(())
321    }
322}
323
324fn normalize(
325    host: &impl Host,
326    info: &TensorInfo,
327    hidden: &[f32],
328    norm: &mut [f32],
329    normalized: &mut [f32],
330    convention: QkNorm,
331    epsilon: f32,
332) -> Result<(), SessionError> {
333    norm.fill(f32::NAN);
334    host.read(info, norm)?;
335    qwen35::ordinary_rms_norm(
336        hidden,
337        norm,
338        convention == QkNorm::RawZeroCentered,
339        epsilon,
340        normalized,
341    )?;
342    Ok(())
343}
344fn execute(
345    plan: &Plan,
346    b: &mut Buffers,
347    hidden: &mut [f32],
348    position: usize,
349    canceled: &dyn Fn() -> bool,
350) -> Result<(u32, f32), SessionError> {
351    let p = &plan.profile;
352    let source = DenseSource(&plan.pkg);
353    let host = DenseHost { source: &source };
354    finite(hidden)?;
355    execute_layers(plan, b, hidden, position, canceled)?;
356    normalize(
357        &host,
358        &plan.final_norm,
359        hidden,
360        &mut b.norm,
361        &mut b.normalized,
362        p.ordinary_norm,
363        p.epsilon,
364    )?;
365    let result = project_output(plan, &mut b.logits, &b.normalized)?;
366    if canceled() {
367        return Err(SessionError::Canceled);
368    }
369    Ok(result)
370}
371fn execute_layers(
372    plan: &Plan,
373    b: &mut Buffers,
374    hidden: &mut [f32],
375    position: usize,
376    canceled: &dyn Fn() -> bool,
377) -> Result<(), SessionError> {
378    finite(hidden)?;
379    for index in 0..plan.layers.len() {
380        if canceled() {
381            return Err(SessionError::Canceled);
382        }
383        execute_attention(plan, b, index, hidden, position)?;
384        if canceled() {
385            return Err(SessionError::Canceled);
386        }
387        execute_ffn(plan, b, index, hidden)?;
388    }
389    if canceled() {
390        return Err(SessionError::Canceled);
391    }
392    Ok(())
393}
394fn execute_attention(
395    plan: &Plan,
396    b: &mut Buffers,
397    index: usize,
398    hidden: &mut [f32],
399    position: usize,
400) -> Result<(), SessionError> {
401    let source = DenseSource(&plan.pkg);
402    let host = DenseHost { source: &source };
403    let layer = &plan.layers[index];
404    let state = &mut b.layers[index];
405    let p = &plan.profile;
406    normalize(
407        &host,
408        &layer.input_norm,
409        hidden,
410        &mut b.norm,
411        &mut b.normalized,
412        p.ordinary_norm,
413        p.epsilon,
414    )?;
415    layer.attention.execute(
416        &host,
417        &b.normalized,
418        hidden,
419        &mut state.history,
420        &mut state.recurrent,
421        &mut state.keys,
422        &mut state.values,
423        position,
424        &p.rotary_inv_freq,
425        p.epsilon,
426        &mut b.workspace,
427    )?;
428    Ok(())
429}
430fn execute_ffn(
431    plan: &Plan,
432    b: &mut Buffers,
433    index: usize,
434    hidden: &mut [f32],
435) -> Result<(), SessionError> {
436    let source = DenseSource(&plan.pkg);
437    let host = DenseHost { source: &source };
438    let p = &plan.profile;
439    normalize(
440        &host,
441        &plan.layers[index].post_norm,
442        hidden,
443        &mut b.norm,
444        &mut b.normalized,
445        p.ordinary_norm,
446        p.epsilon,
447    )?;
448    super::session_ffn::execute(
449        plan,
450        index,
451        &host,
452        &b.normalized,
453        &mut b.router,
454        &mut b.gate_up,
455        &mut b.gate,
456        &mut b.up,
457        &mut b.swiglu,
458        &mut b.expert,
459        &mut b.combined,
460    )?;
461    for (value, delta) in hidden.iter_mut().zip(&b.combined) {
462        *value += delta;
463    }
464    finite(hidden)
465}
466fn project_output(
467    plan: &Plan,
468    logits: &mut [f32],
469    hidden: &[f32],
470) -> Result<(u32, f32), SessionError> {
471    plan.output
472        .gemv(&plan.pkg, hidden, logits)
473        .map_err(|_| SessionError::Provider)?;
474    finite(logits)?;
475    let mut best = 0;
476    for i in 1..logits.len() {
477        if logits[i] > logits[best] {
478            best = i;
479        }
480    }
481    Ok((best as u32, logits[best]))
482}
483
484#[cfg(test)]
485#[path = "session_tests.rs"]
486mod tests;
487
488#[path = "session_lab.rs"]
489mod lab;