qualia_core_db/inference/runtime/prepared/
capability.rs1use super::decode_plan::PreparedBackend;
8
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
11pub enum SupportedArchitecture {
12 Llama,
13 Mistral,
14 Qwen2,
15 Qwen3,
16 Qwen3_5MoE,
17 DeepSeek,
18 Gemma,
19 Phi,
20 DenseTransformer,
21}
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
25pub enum KvEncoding {
26 F32Dense,
27 F16Dense,
28 Q8_0Dense,
29 PagedF32,
30 PagedF16,
31 PagedQ8_0,
32}
33
34#[derive(Debug, Clone, PartialEq, Eq)]
36pub struct BackendCapabilities {
37 pub backend: PreparedBackend,
38 pub max_batch_size: u32,
39 pub supports_paged_attention: bool,
40 pub supports_cuda_graphs: bool,
41 pub supports_directml_graphs: bool,
42 pub supported_architectures: &'static [SupportedArchitecture],
43 pub supported_quants: &'static [&'static str],
44 pub supported_kv_encodings: &'static [KvEncoding],
45}
46
47impl BackendCapabilities {
48 pub const fn cpu_reference() -> Self {
50 Self {
51 backend: PreparedBackend::Cpu,
52 max_batch_size: 1,
53 supports_paged_attention: true,
54 supports_cuda_graphs: false,
55 supports_directml_graphs: false,
56 supported_architectures: &[
57 SupportedArchitecture::Llama,
58 SupportedArchitecture::Qwen2,
59 SupportedArchitecture::Qwen3,
60 SupportedArchitecture::DenseTransformer,
61 ],
62 supported_quants: &["F32", "F16", "Q8_0", "Q4_K_M", "Q4_0"],
63 supported_kv_encodings: &[
64 KvEncoding::F32Dense,
65 KvEncoding::PagedF32,
66 KvEncoding::PagedF16,
67 ],
68 }
69 }
70
71 pub const fn cuda_reference() -> Self {
73 Self {
74 backend: PreparedBackend::Cuda,
75 max_batch_size: 16,
76 supports_paged_attention: true,
77 supports_cuda_graphs: true,
78 supports_directml_graphs: false,
79 supported_architectures: &[
80 SupportedArchitecture::Llama,
81 SupportedArchitecture::Qwen2,
82 SupportedArchitecture::Qwen3,
83 SupportedArchitecture::DenseTransformer,
84 ],
85 supported_quants: &["F16", "Q8_0", "Q4_K_M"],
86 supported_kv_encodings: &[
87 KvEncoding::F16Dense,
88 KvEncoding::PagedF16,
89 KvEncoding::PagedQ8_0,
90 ],
91 }
92 }
93
94 pub fn is_architecture_supported(&self, arch: SupportedArchitecture) -> bool {
96 self.supported_architectures.contains(&arch)
97 }
98
99 pub fn is_quant_supported(&self, quant: &str) -> bool {
101 self.supported_quants
102 .iter()
103 .any(|&q| q.eq_ignore_ascii_case(quant))
104 }
105
106 pub fn is_kv_encoding_supported(&self, enc: KvEncoding) -> bool {
108 self.supported_kv_encodings.contains(&enc)
109 }
110}
111
112#[cfg(test)]
113mod tests {
114 use super::*;
115
116 #[test]
117 fn test_cpu_capabilities() {
118 let caps = BackendCapabilities::cpu_reference();
119 assert!(caps.is_architecture_supported(SupportedArchitecture::Llama));
120 assert!(caps.is_quant_supported("q4_k_m"));
121 assert!(caps.is_kv_encoding_supported(KvEncoding::PagedF32));
122 assert!(!caps.supports_cuda_graphs);
123 assert!(!caps.is_quant_supported("NVFP4"));
124 }
125
126 #[test]
127 fn test_cuda_capabilities() {
128 let caps = BackendCapabilities::cuda_reference();
129 assert!(caps.is_architecture_supported(SupportedArchitecture::Llama));
130 assert!(caps.is_quant_supported("q4_k_m"));
131 assert!(caps.is_kv_encoding_supported(KvEncoding::PagedQ8_0));
132 assert!(caps.supports_cuda_graphs);
133 assert_eq!(caps.max_batch_size, 16);
134 assert!(!caps.is_quant_supported("NVFP4"));
135 assert!(!caps.is_quant_supported("FP8"));
136 assert!(!caps.is_architecture_supported(SupportedArchitecture::Qwen3_5MoE));
137 }
138}