Skip to main content

qualia_client_core/
continuous_batch.rs

1//! Client-facing, caller-buffered boundary for one native continuous-decode round.
2//!
3//! This module intentionally does not own a model or KV cache.  The Studio/client
4//! supplies stable session rows and caller-owned scratch; the native scheduler
5//! backend performs the actual token step.  Keeping this boundary flat means a
6//! future chat state machine can use the same path without serialising per-turn
7//! requests through a global decode mutex.
8
9use qualia_core_db::inference::runtime::scheduler::{
10    RaggedBackendError, RaggedBatchItem, RaggedBatchOutput, RaggedBatchReceipt, RaggedDecodeBackend,
11};
12
13/// One client session's input to a native continuous decode round.
14#[repr(C)]
15#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
16pub struct SessionDecodeInput {
17    /// Stable native request identity; must be unique within the round.
18    pub request_id: u64,
19    /// Scheduler slot assigned to this session.
20    pub slot: u32,
21    /// Most recently accepted token for this sequence.
22    pub token_id: u32,
23    /// Position of `token_id` in the sequence.
24    pub position: u32,
25    /// Offset into the shared caller-owned logical KV page table.
26    pub block_table_offset: u32,
27    /// Number of logical pages for this sequence.
28    pub logical_pages: u32,
29}
30
31/// Token returned for one client session, preserving its scheduler identity.
32#[repr(C)]
33#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
34pub struct SessionDecodeOutput {
35    pub request_id: u64,
36    pub slot: u32,
37    pub next_token_id: u32,
38}
39
40/// Receipt returned to the client/Studio telemetry path after a decode round.
41#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
42pub struct ContinuousBatchReceipt {
43    pub sessions_completed: u32,
44    pub backend: RaggedBatchReceipt,
45}
46
47#[derive(Clone, Copy, Debug, PartialEq, Eq)]
48pub enum ContinuousBatchError {
49    ItemScratchTooSmall,
50    BackendOutputScratchTooSmall,
51    SessionOutputTooSmall,
52    DuplicateRequestId,
53    InvalidBlockTableRange,
54    Backend(RaggedBackendError),
55    OutputIdentityMismatch,
56}
57
58impl From<RaggedBackendError> for ContinuousBatchError {
59    fn from(value: RaggedBackendError) -> Self {
60        Self::Backend(value)
61    }
62}
63
64/// Execute one real ragged decode round from client session rows.
65///
66/// No heap state is created here.  All intermediate rows are supplied by the
67/// caller, and every returned output is checked against the session identity
68/// that entered the round before it becomes visible to the client.
69pub fn execute_continuous_batch_round<B: RaggedDecodeBackend>(
70    backend: &mut B,
71    sessions: &[SessionDecodeInput],
72    block_tables: &[u32],
73    item_scratch: &mut [RaggedBatchItem],
74    backend_output_scratch: &mut [RaggedBatchOutput],
75    out: &mut [SessionDecodeOutput],
76) -> Result<ContinuousBatchReceipt, ContinuousBatchError> {
77    let count = sessions.len();
78    if item_scratch.len() < count {
79        return Err(ContinuousBatchError::ItemScratchTooSmall);
80    }
81    if backend_output_scratch.len() < count {
82        return Err(ContinuousBatchError::BackendOutputScratchTooSmall);
83    }
84    if out.len() < count {
85        return Err(ContinuousBatchError::SessionOutputTooSmall);
86    }
87
88    for (index, session) in sessions.iter().enumerate() {
89        let table_start = session.block_table_offset as usize;
90        let table_end = table_start.saturating_add(session.logical_pages as usize);
91        if table_end > block_tables.len() {
92            return Err(ContinuousBatchError::InvalidBlockTableRange);
93        }
94        for prior in &sessions[..index] {
95            if prior.request_id == session.request_id {
96                return Err(ContinuousBatchError::DuplicateRequestId);
97            }
98        }
99        item_scratch[index] = RaggedBatchItem {
100            request_id: session.request_id,
101            slot: session.slot,
102            token_id: session.token_id,
103            position: session.position,
104            block_table_offset: session.block_table_offset,
105            logical_pages: session.logical_pages,
106            _reserved: 0,
107        };
108    }
109
110    let backend_receipt = backend.execute_ragged(
111        &item_scratch[..count],
112        block_tables,
113        &mut backend_output_scratch[..count],
114    )?;
115    if backend_receipt.batch_size != count as u32 || backend_receipt.backend_launches != 1 {
116        return Err(ContinuousBatchError::OutputIdentityMismatch);
117    }
118
119    for index in 0..count {
120        let input = sessions[index];
121        let decoded = backend_output_scratch[index];
122        if decoded.request_id != input.request_id || decoded.slot != input.slot {
123            return Err(ContinuousBatchError::OutputIdentityMismatch);
124        }
125        out[index] = SessionDecodeOutput {
126            request_id: decoded.request_id,
127            slot: decoded.slot,
128            next_token_id: decoded.next_token_id,
129        };
130    }
131
132    Ok(ContinuousBatchReceipt {
133        sessions_completed: count as u32,
134        backend: backend_receipt,
135    })
136}
137
138/// Maximum concurrent active chat sessions in a single continuous batch round.
139pub const MAX_CONCURRENT_CHAT_SESSIONS: usize = 8;
140
141/// State for an admitted chat session advancing through continuous decode rounds.
142#[derive(Clone, Debug, PartialEq, Eq)]
143pub struct ChatBatchSessionState {
144    pub session_id: String,
145    pub request_id: u64,
146    pub slot: u32,
147    pub current_token_id: u32,
148    pub position: u32,
149    pub block_table_offset: u32,
150    pub logical_pages: u32,
151    pub is_active: bool,
152    pub is_finished: bool,
153    pub generated_tokens: Vec<u32>,
154}
155
156/// Zero-heap-hot-path coordinator managing concurrent chat sessions across continuous batch decode rounds.
157#[derive(Debug, Default)]
158pub struct ChatContinuousBatchCoordinator {
159    sessions: Vec<ChatBatchSessionState>,
160    next_slot: u32,
161}
162
163impl ChatContinuousBatchCoordinator {
164    /// Create a new coordinator for multi-session continuous batching.
165    pub fn new() -> Self {
166        Self {
167            sessions: Vec::new(),
168            next_slot: 0,
169        }
170    }
171
172    /// Admit a new chat session to the batch pool.
173    pub fn admit_session(
174        &mut self,
175        session_id: &str,
176        request_id: u64,
177        initial_token_id: u32,
178        initial_position: u32,
179        block_table_offset: u32,
180        logical_pages: u32,
181    ) -> Result<u32, ContinuousBatchError> {
182        for s in &self.sessions {
183            if s.is_active && s.request_id == request_id {
184                return Err(ContinuousBatchError::DuplicateRequestId);
185            }
186        }
187        let slot = self.next_slot;
188        self.next_slot = self.next_slot.wrapping_add(1);
189
190        self.sessions.push(ChatBatchSessionState {
191            session_id: session_id.to_string(),
192            request_id,
193            slot,
194            current_token_id: initial_token_id,
195            position: initial_position,
196            block_table_offset,
197            logical_pages,
198            is_active: true,
199            is_finished: false,
200            generated_tokens: Vec::new(),
201        });
202
203        Ok(slot)
204    }
205
206    /// Number of active, unfinished sessions currently tracked.
207    pub fn active_session_count(&self) -> usize {
208        self.sessions
209            .iter()
210            .filter(|s| s.is_active && !s.is_finished)
211            .count()
212    }
213
214    /// Step all active chat sessions forward by one token step.
215    ///
216    /// The decode step executes via `execute_continuous_batch_round` over fixed stack
217    /// scratch arrays (zero heap allocation in hot step loop).
218    pub fn step_round<B: RaggedDecodeBackend>(
219        &mut self,
220        backend: &mut B,
221        block_tables: &[u32],
222        eos_token_id: u32,
223    ) -> Result<ContinuousBatchReceipt, ContinuousBatchError> {
224        let active_indices: Vec<usize> = self
225            .sessions
226            .iter()
227            .enumerate()
228            .filter(|(_, s)| s.is_active && !s.is_finished)
229            .map(|(i, _)| i)
230            .take(MAX_CONCURRENT_CHAT_SESSIONS)
231            .collect();
232
233        let count = active_indices.len();
234        if count == 0 {
235            return Ok(ContinuousBatchReceipt::default());
236        }
237
238        let mut inputs = [SessionDecodeInput::default(); MAX_CONCURRENT_CHAT_SESSIONS];
239        let mut items = [RaggedBatchItem::default(); MAX_CONCURRENT_CHAT_SESSIONS];
240        let mut backend_outputs = [RaggedBatchOutput::default(); MAX_CONCURRENT_CHAT_SESSIONS];
241        let mut session_outputs = [SessionDecodeOutput::default(); MAX_CONCURRENT_CHAT_SESSIONS];
242
243        for (idx, &session_idx) in active_indices.iter().enumerate() {
244            let session = &self.sessions[session_idx];
245            inputs[idx] = SessionDecodeInput {
246                request_id: session.request_id,
247                slot: session.slot,
248                token_id: session.current_token_id,
249                position: session.position,
250                block_table_offset: session.block_table_offset,
251                logical_pages: session.logical_pages,
252            };
253        }
254
255        let receipt = execute_continuous_batch_round(
256            backend,
257            &inputs[..count],
258            block_tables,
259            &mut items[..count],
260            &mut backend_outputs[..count],
261            &mut session_outputs[..count],
262        )?;
263
264        for (idx, &session_idx) in active_indices.iter().enumerate() {
265            let output = session_outputs[idx];
266            let session = &mut self.sessions[session_idx];
267            session.generated_tokens.push(output.next_token_id);
268            session.current_token_id = output.next_token_id;
269            session.position = session.position.saturating_add(1);
270            if output.next_token_id == eos_token_id {
271                session.is_finished = true;
272            }
273        }
274
275        Ok(receipt)
276    }
277
278    /// Check if a specific session has completed generation.
279    pub fn is_session_finished(&self, request_id: u64) -> bool {
280        self.sessions
281            .iter()
282            .find(|s| s.request_id == request_id)
283            .map(|s| s.is_finished)
284            .unwrap_or(true)
285    }
286
287    /// Retrieve the generated token stream for a session.
288    pub fn get_session_tokens(&self, request_id: u64) -> Option<&[u32]> {
289        self.sessions
290            .iter()
291            .find(|s| s.request_id == request_id)
292            .map(|s| s.generated_tokens.as_slice())
293    }
294
295    /// Mark a session as cancelled/inactive.
296    pub fn cancel_session(&mut self, request_id: u64) {
297        if let Some(session) = self
298            .sessions
299            .iter_mut()
300            .find(|s| s.request_id == request_id)
301        {
302            session.is_active = false;
303        }
304    }
305}
306
307#[cfg(test)]
308mod tests {
309    use super::*;
310    use qualia_core_db::inference::runtime::scheduler::MultiSequenceRaggedBackend;
311
312    #[test]
313    fn routes_multiple_client_sessions_through_one_backend_round() {
314        let mut backend =
315            MultiSequenceRaggedBackend::new(|request, slot, token, position, pages| {
316                request as u32 + slot + token + position + pages.len() as u32
317            });
318        let sessions = [
319            SessionDecodeInput {
320                request_id: 7,
321                slot: 0,
322                token_id: 3,
323                position: 4,
324                block_table_offset: 0,
325                logical_pages: 1,
326            },
327            SessionDecodeInput {
328                request_id: 9,
329                slot: 1,
330                token_id: 5,
331                position: 6,
332                block_table_offset: 1,
333                logical_pages: 2,
334            },
335        ];
336        let block_tables = [4, 8, 12];
337        let mut items = [RaggedBatchItem::default(); 2];
338        let mut backend_out = [RaggedBatchOutput::default(); 2];
339        let mut out = [SessionDecodeOutput::default(); 2];
340
341        let receipt = execute_continuous_batch_round(
342            &mut backend,
343            &sessions,
344            &block_tables,
345            &mut items,
346            &mut backend_out,
347            &mut out,
348        )
349        .unwrap();
350
351        assert_eq!(receipt.sessions_completed, 2);
352        assert_eq!(receipt.backend.backend_launches, 1);
353        assert_eq!(out[0].next_token_id, 15);
354        assert_eq!(out[1].next_token_id, 23);
355    }
356
357    #[test]
358    fn rejects_duplicate_session_request_ids_before_backend_execution() {
359        let mut backend = MultiSequenceRaggedBackend::new(|_, _, token, _, _| token);
360        let sessions = [
361            SessionDecodeInput {
362                request_id: 7,
363                ..SessionDecodeInput::default()
364            },
365            SessionDecodeInput {
366                request_id: 7,
367                ..SessionDecodeInput::default()
368            },
369        ];
370        let mut items = [RaggedBatchItem::default(); 2];
371        let mut backend_out = [RaggedBatchOutput::default(); 2];
372        let mut out = [SessionDecodeOutput::default(); 2];
373
374        let error = execute_continuous_batch_round(
375            &mut backend,
376            &sessions,
377            &[],
378            &mut items,
379            &mut backend_out,
380            &mut out,
381        )
382        .unwrap_err();
383
384        assert_eq!(error, ContinuousBatchError::DuplicateRequestId);
385    }
386
387    #[test]
388    fn coordinator_advances_multiple_active_chat_sessions_to_completion() {
389        let mut backend = MultiSequenceRaggedBackend::new(|_req, _slot, token, _pos, _pages| {
390            if token >= 10 {
391                99 // EOS
392            } else {
393                token + 1
394            }
395        });
396
397        let mut coord = ChatContinuousBatchCoordinator::new();
398        coord
399            .admit_session("session-alpha", 101, 8, 0, 0, 1)
400            .unwrap();
401        coord
402            .admit_session("session-beta", 102, 9, 0, 1, 1)
403            .unwrap();
404
405        assert_eq!(coord.active_session_count(), 2);
406        let block_tables = [10, 20];
407
408        // Round 1: alpha goes 8 -> 9; beta goes 9 -> 10
409        let r1 = coord.step_round(&mut backend, &block_tables, 99).unwrap();
410        assert_eq!(r1.sessions_completed, 2);
411        assert_eq!(coord.active_session_count(), 2);
412        assert!(!coord.is_session_finished(101));
413        assert!(!coord.is_session_finished(102));
414
415        // Round 2: alpha goes 9 -> 10; beta goes 10 -> 99 (EOS -> finishes)
416        let r2 = coord.step_round(&mut backend, &block_tables, 99).unwrap();
417        assert_eq!(r2.sessions_completed, 2);
418        assert_eq!(coord.active_session_count(), 1);
419        assert!(!coord.is_session_finished(101));
420        assert!(coord.is_session_finished(102));
421
422        // Round 3: only alpha runs (goes 10 -> 99 (EOS -> finishes))
423        let r3 = coord.step_round(&mut backend, &block_tables, 99).unwrap();
424        assert_eq!(r3.sessions_completed, 1);
425        assert_eq!(coord.active_session_count(), 0);
426        assert!(coord.is_session_finished(101));
427
428        assert_eq!(coord.get_session_tokens(101), Some(&[9, 10, 99][..]));
429        assert_eq!(coord.get_session_tokens(102), Some(&[10, 99][..]));
430    }
431}