1use qualia_core_db::inference::runtime::scheduler::{
10 RaggedBackendError, RaggedBatchItem, RaggedBatchOutput, RaggedBatchReceipt, RaggedDecodeBackend,
11};
12
13#[repr(C)]
15#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
16pub struct SessionDecodeInput {
17 pub request_id: u64,
19 pub slot: u32,
21 pub token_id: u32,
23 pub position: u32,
25 pub block_table_offset: u32,
27 pub logical_pages: u32,
29}
30
31#[repr(C)]
33#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
34pub struct SessionDecodeOutput {
35 pub request_id: u64,
36 pub slot: u32,
37 pub next_token_id: u32,
38}
39
40#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
42pub struct ContinuousBatchReceipt {
43 pub sessions_completed: u32,
44 pub backend: RaggedBatchReceipt,
45}
46
47#[derive(Clone, Copy, Debug, PartialEq, Eq)]
48pub enum ContinuousBatchError {
49 ItemScratchTooSmall,
50 BackendOutputScratchTooSmall,
51 SessionOutputTooSmall,
52 DuplicateRequestId,
53 InvalidBlockTableRange,
54 Backend(RaggedBackendError),
55 OutputIdentityMismatch,
56}
57
58impl From<RaggedBackendError> for ContinuousBatchError {
59 fn from(value: RaggedBackendError) -> Self {
60 Self::Backend(value)
61 }
62}
63
64pub fn execute_continuous_batch_round<B: RaggedDecodeBackend>(
70 backend: &mut B,
71 sessions: &[SessionDecodeInput],
72 block_tables: &[u32],
73 item_scratch: &mut [RaggedBatchItem],
74 backend_output_scratch: &mut [RaggedBatchOutput],
75 out: &mut [SessionDecodeOutput],
76) -> Result<ContinuousBatchReceipt, ContinuousBatchError> {
77 let count = sessions.len();
78 if item_scratch.len() < count {
79 return Err(ContinuousBatchError::ItemScratchTooSmall);
80 }
81 if backend_output_scratch.len() < count {
82 return Err(ContinuousBatchError::BackendOutputScratchTooSmall);
83 }
84 if out.len() < count {
85 return Err(ContinuousBatchError::SessionOutputTooSmall);
86 }
87
88 for (index, session) in sessions.iter().enumerate() {
89 let table_start = session.block_table_offset as usize;
90 let table_end = table_start.saturating_add(session.logical_pages as usize);
91 if table_end > block_tables.len() {
92 return Err(ContinuousBatchError::InvalidBlockTableRange);
93 }
94 for prior in &sessions[..index] {
95 if prior.request_id == session.request_id {
96 return Err(ContinuousBatchError::DuplicateRequestId);
97 }
98 }
99 item_scratch[index] = RaggedBatchItem {
100 request_id: session.request_id,
101 slot: session.slot,
102 token_id: session.token_id,
103 position: session.position,
104 block_table_offset: session.block_table_offset,
105 logical_pages: session.logical_pages,
106 _reserved: 0,
107 };
108 }
109
110 let backend_receipt = backend.execute_ragged(
111 &item_scratch[..count],
112 block_tables,
113 &mut backend_output_scratch[..count],
114 )?;
115 if backend_receipt.batch_size != count as u32 || backend_receipt.backend_launches != 1 {
116 return Err(ContinuousBatchError::OutputIdentityMismatch);
117 }
118
119 for index in 0..count {
120 let input = sessions[index];
121 let decoded = backend_output_scratch[index];
122 if decoded.request_id != input.request_id || decoded.slot != input.slot {
123 return Err(ContinuousBatchError::OutputIdentityMismatch);
124 }
125 out[index] = SessionDecodeOutput {
126 request_id: decoded.request_id,
127 slot: decoded.slot,
128 next_token_id: decoded.next_token_id,
129 };
130 }
131
132 Ok(ContinuousBatchReceipt {
133 sessions_completed: count as u32,
134 backend: backend_receipt,
135 })
136}
137
138pub const MAX_CONCURRENT_CHAT_SESSIONS: usize = 8;
140
141#[derive(Clone, Debug, PartialEq, Eq)]
143pub struct ChatBatchSessionState {
144 pub session_id: String,
145 pub request_id: u64,
146 pub slot: u32,
147 pub current_token_id: u32,
148 pub position: u32,
149 pub block_table_offset: u32,
150 pub logical_pages: u32,
151 pub is_active: bool,
152 pub is_finished: bool,
153 pub generated_tokens: Vec<u32>,
154}
155
156#[derive(Debug, Default)]
158pub struct ChatContinuousBatchCoordinator {
159 sessions: Vec<ChatBatchSessionState>,
160 next_slot: u32,
161}
162
163impl ChatContinuousBatchCoordinator {
164 pub fn new() -> Self {
166 Self {
167 sessions: Vec::new(),
168 next_slot: 0,
169 }
170 }
171
172 pub fn admit_session(
174 &mut self,
175 session_id: &str,
176 request_id: u64,
177 initial_token_id: u32,
178 initial_position: u32,
179 block_table_offset: u32,
180 logical_pages: u32,
181 ) -> Result<u32, ContinuousBatchError> {
182 for s in &self.sessions {
183 if s.is_active && s.request_id == request_id {
184 return Err(ContinuousBatchError::DuplicateRequestId);
185 }
186 }
187 let slot = self.next_slot;
188 self.next_slot = self.next_slot.wrapping_add(1);
189
190 self.sessions.push(ChatBatchSessionState {
191 session_id: session_id.to_string(),
192 request_id,
193 slot,
194 current_token_id: initial_token_id,
195 position: initial_position,
196 block_table_offset,
197 logical_pages,
198 is_active: true,
199 is_finished: false,
200 generated_tokens: Vec::new(),
201 });
202
203 Ok(slot)
204 }
205
206 pub fn active_session_count(&self) -> usize {
208 self.sessions
209 .iter()
210 .filter(|s| s.is_active && !s.is_finished)
211 .count()
212 }
213
214 pub fn step_round<B: RaggedDecodeBackend>(
219 &mut self,
220 backend: &mut B,
221 block_tables: &[u32],
222 eos_token_id: u32,
223 ) -> Result<ContinuousBatchReceipt, ContinuousBatchError> {
224 let active_indices: Vec<usize> = self
225 .sessions
226 .iter()
227 .enumerate()
228 .filter(|(_, s)| s.is_active && !s.is_finished)
229 .map(|(i, _)| i)
230 .take(MAX_CONCURRENT_CHAT_SESSIONS)
231 .collect();
232
233 let count = active_indices.len();
234 if count == 0 {
235 return Ok(ContinuousBatchReceipt::default());
236 }
237
238 let mut inputs = [SessionDecodeInput::default(); MAX_CONCURRENT_CHAT_SESSIONS];
239 let mut items = [RaggedBatchItem::default(); MAX_CONCURRENT_CHAT_SESSIONS];
240 let mut backend_outputs = [RaggedBatchOutput::default(); MAX_CONCURRENT_CHAT_SESSIONS];
241 let mut session_outputs = [SessionDecodeOutput::default(); MAX_CONCURRENT_CHAT_SESSIONS];
242
243 for (idx, &session_idx) in active_indices.iter().enumerate() {
244 let session = &self.sessions[session_idx];
245 inputs[idx] = SessionDecodeInput {
246 request_id: session.request_id,
247 slot: session.slot,
248 token_id: session.current_token_id,
249 position: session.position,
250 block_table_offset: session.block_table_offset,
251 logical_pages: session.logical_pages,
252 };
253 }
254
255 let receipt = execute_continuous_batch_round(
256 backend,
257 &inputs[..count],
258 block_tables,
259 &mut items[..count],
260 &mut backend_outputs[..count],
261 &mut session_outputs[..count],
262 )?;
263
264 for (idx, &session_idx) in active_indices.iter().enumerate() {
265 let output = session_outputs[idx];
266 let session = &mut self.sessions[session_idx];
267 session.generated_tokens.push(output.next_token_id);
268 session.current_token_id = output.next_token_id;
269 session.position = session.position.saturating_add(1);
270 if output.next_token_id == eos_token_id {
271 session.is_finished = true;
272 }
273 }
274
275 Ok(receipt)
276 }
277
278 pub fn is_session_finished(&self, request_id: u64) -> bool {
280 self.sessions
281 .iter()
282 .find(|s| s.request_id == request_id)
283 .map(|s| s.is_finished)
284 .unwrap_or(true)
285 }
286
287 pub fn get_session_tokens(&self, request_id: u64) -> Option<&[u32]> {
289 self.sessions
290 .iter()
291 .find(|s| s.request_id == request_id)
292 .map(|s| s.generated_tokens.as_slice())
293 }
294
295 pub fn cancel_session(&mut self, request_id: u64) {
297 if let Some(session) = self
298 .sessions
299 .iter_mut()
300 .find(|s| s.request_id == request_id)
301 {
302 session.is_active = false;
303 }
304 }
305}
306
307#[cfg(test)]
308mod tests {
309 use super::*;
310 use qualia_core_db::inference::runtime::scheduler::MultiSequenceRaggedBackend;
311
312 #[test]
313 fn routes_multiple_client_sessions_through_one_backend_round() {
314 let mut backend =
315 MultiSequenceRaggedBackend::new(|request, slot, token, position, pages| {
316 request as u32 + slot + token + position + pages.len() as u32
317 });
318 let sessions = [
319 SessionDecodeInput {
320 request_id: 7,
321 slot: 0,
322 token_id: 3,
323 position: 4,
324 block_table_offset: 0,
325 logical_pages: 1,
326 },
327 SessionDecodeInput {
328 request_id: 9,
329 slot: 1,
330 token_id: 5,
331 position: 6,
332 block_table_offset: 1,
333 logical_pages: 2,
334 },
335 ];
336 let block_tables = [4, 8, 12];
337 let mut items = [RaggedBatchItem::default(); 2];
338 let mut backend_out = [RaggedBatchOutput::default(); 2];
339 let mut out = [SessionDecodeOutput::default(); 2];
340
341 let receipt = execute_continuous_batch_round(
342 &mut backend,
343 &sessions,
344 &block_tables,
345 &mut items,
346 &mut backend_out,
347 &mut out,
348 )
349 .unwrap();
350
351 assert_eq!(receipt.sessions_completed, 2);
352 assert_eq!(receipt.backend.backend_launches, 1);
353 assert_eq!(out[0].next_token_id, 15);
354 assert_eq!(out[1].next_token_id, 23);
355 }
356
357 #[test]
358 fn rejects_duplicate_session_request_ids_before_backend_execution() {
359 let mut backend = MultiSequenceRaggedBackend::new(|_, _, token, _, _| token);
360 let sessions = [
361 SessionDecodeInput {
362 request_id: 7,
363 ..SessionDecodeInput::default()
364 },
365 SessionDecodeInput {
366 request_id: 7,
367 ..SessionDecodeInput::default()
368 },
369 ];
370 let mut items = [RaggedBatchItem::default(); 2];
371 let mut backend_out = [RaggedBatchOutput::default(); 2];
372 let mut out = [SessionDecodeOutput::default(); 2];
373
374 let error = execute_continuous_batch_round(
375 &mut backend,
376 &sessions,
377 &[],
378 &mut items,
379 &mut backend_out,
380 &mut out,
381 )
382 .unwrap_err();
383
384 assert_eq!(error, ContinuousBatchError::DuplicateRequestId);
385 }
386
387 #[test]
388 fn coordinator_advances_multiple_active_chat_sessions_to_completion() {
389 let mut backend = MultiSequenceRaggedBackend::new(|_req, _slot, token, _pos, _pages| {
390 if token >= 10 {
391 99 } else {
393 token + 1
394 }
395 });
396
397 let mut coord = ChatContinuousBatchCoordinator::new();
398 coord
399 .admit_session("session-alpha", 101, 8, 0, 0, 1)
400 .unwrap();
401 coord
402 .admit_session("session-beta", 102, 9, 0, 1, 1)
403 .unwrap();
404
405 assert_eq!(coord.active_session_count(), 2);
406 let block_tables = [10, 20];
407
408 let r1 = coord.step_round(&mut backend, &block_tables, 99).unwrap();
410 assert_eq!(r1.sessions_completed, 2);
411 assert_eq!(coord.active_session_count(), 2);
412 assert!(!coord.is_session_finished(101));
413 assert!(!coord.is_session_finished(102));
414
415 let r2 = coord.step_round(&mut backend, &block_tables, 99).unwrap();
417 assert_eq!(r2.sessions_completed, 2);
418 assert_eq!(coord.active_session_count(), 1);
419 assert!(!coord.is_session_finished(101));
420 assert!(coord.is_session_finished(102));
421
422 let r3 = coord.step_round(&mut backend, &block_tables, 99).unwrap();
424 assert_eq!(r3.sessions_completed, 1);
425 assert_eq!(coord.active_session_count(), 0);
426 assert!(coord.is_session_finished(101));
427
428 assert_eq!(coord.get_session_tokens(101), Some(&[9, 10, 99][..]));
429 assert_eq!(coord.get_session_tokens(102), Some(&[10, 99][..]));
430 }
431}