1#![cfg(all(not(target_arch = "wasm32"), feature = "gpu-runtime"))]
17
18use super::adapter_manager::{LoRAAdapter, LoRAError};
19
20#[repr(C)]
23#[derive(Clone, Copy, bytemuck::Pod, bytemuck::Zeroable)]
24pub struct LoraGpuParams {
25 pub n_in: u32,
26 pub n_out: u32,
27 pub rank: u32,
28 pub scaling: f32,
29}
30
31pub struct LoRAGpuApplicator {
37 device: etalon::Device,
38 queue: etalon::Queue,
39 pipeline: etalon::ComputePipeline,
40 bgl: etalon::BindGroupLayout,
41}
42
43impl LoRAGpuApplicator {
44 pub fn new() -> Result<Self, LoRAError> {
49 let rt = Box::leak(Box::new(
50 tokio::runtime::Runtime::new()
51 .map_err(|e| LoRAError::Io(format!("tokio runtime: {e}")))?,
52 ));
53 rt.block_on(Self::new_async())
54 }
55
56 async fn new_async() -> Result<Self, LoRAError> {
57 let instance = etalon::Instance::default();
58 let adapter = instance
59 .request_adapter(&etalon::RequestAdapterOptions {
60 power_preference: etalon::PowerPreference::HighPerformance,
61 ..Default::default()
62 })
63 .await
64 .map_err(|e| LoRAError::Io(format!("no wgpu adapter: {e}")))?;
65
66 let (device, queue) = adapter
69 .request_device(&etalon::DeviceDescriptor::default())
70 .await
71 .map_err(|e| LoRAError::Io(format!("wgpu device: {e}")))?;
72
73 let shader = device.create_shader_module(etalon::ShaderModuleDescriptor {
74 label: Some("lora_apply"),
75 source: etalon::ShaderSource::Wgsl(include_str!("../shaders/lora_apply.wgsl").into()),
76 });
77
78 let bgl = device.create_bind_group_layout(&etalon::BindGroupLayoutDescriptor {
79 label: Some("lora-bgl"),
80 entries: &[
81 storage_rw_entry(0), storage_r_entry(1), storage_r_entry(2), storage_r_entry(3), uniform_entry(4), ],
87 });
88
89 let pl = device.create_pipeline_layout(&etalon::PipelineLayoutDescriptor {
90 label: Some("lora-pl"),
91 bind_group_layouts: &[Some(&bgl)],
92 immediate_size: 0,
93 });
94
95 let pipeline = device.create_compute_pipeline(&etalon::ComputePipelineDescriptor {
96 label: Some("lora-pipeline"),
97 layout: Some(&pl),
98 module: &shader,
99 entry_point: Some("apply_lora"),
100 compilation_options: Default::default(),
101 cache: None,
102 });
103
104 Ok(Self {
105 device,
106 queue,
107 pipeline,
108 bgl,
109 })
110 }
111
112 pub fn apply(
119 &self,
120 adapter: &LoRAAdapter,
121 input: &[f32],
122 output: &mut [f32],
123 ) -> Result<(), LoRAError> {
124 let n_in = adapter.meta.n_in;
125 let n_out = adapter.meta.n_out;
126 let rank = adapter.meta.rank as usize;
127 let scaling = adapter.meta.scaling();
128
129 if input.len() != n_in || output.len() != n_out {
130 return Err(LoRAError::InferenceDimMismatch {
131 input_len: input.len(),
132 lora_n_in: n_in,
133 });
134 }
135
136 let params = LoraGpuParams {
137 n_in: n_in as u32,
138 n_out: n_out as u32,
139 rank: rank as u32,
140 scaling,
141 };
142
143 let dev = &self.device;
144
145 let out_buf = buf_rw(dev, bytemuck::cast_slice(output));
147 let in_buf = buf_r(dev, bytemuck::cast_slice(input));
148 let a_buf = buf_r(dev, bytemuck::cast_slice(&adapter.lora_a.data));
149 let b_buf = buf_r(dev, bytemuck::cast_slice(&adapter.lora_b.data));
150 let params_buf = buf_uniform(dev, bytemuck::bytes_of(¶ms));
151
152 let out_staging = dev.create_buffer(&etalon::BufferDescriptor {
153 label: Some("lora-out-staging"),
154 size: (n_out * 4) as u64,
155 usage: etalon::BufferUsages::MAP_READ | etalon::BufferUsages::COPY_DST,
156 mapped_at_creation: false,
157 });
158
159 let bg = dev.create_bind_group(&etalon::BindGroupDescriptor {
161 label: Some("lora-bg"),
162 layout: &self.bgl,
163 entries: &[
164 etalon::BindGroupEntry {
165 binding: 0,
166 resource: out_buf.as_entire_binding(),
167 },
168 etalon::BindGroupEntry {
169 binding: 1,
170 resource: in_buf.as_entire_binding(),
171 },
172 etalon::BindGroupEntry {
173 binding: 2,
174 resource: a_buf.as_entire_binding(),
175 },
176 etalon::BindGroupEntry {
177 binding: 3,
178 resource: b_buf.as_entire_binding(),
179 },
180 etalon::BindGroupEntry {
181 binding: 4,
182 resource: params_buf.as_entire_binding(),
183 },
184 ],
185 });
186
187 let mut enc = dev.create_command_encoder(&etalon::CommandEncoderDescriptor {
189 label: Some("lora-enc"),
190 });
191
192 {
193 let mut cpass = enc.begin_compute_pass(&etalon::ComputePassDescriptor {
194 label: Some("lora-pass"),
195 timestamp_writes: None,
196 });
197 cpass.set_pipeline(&self.pipeline);
198 cpass.set_bind_group(0, &bg, &[]);
199 cpass.dispatch_workgroups((n_out as u32 + 63) / 64, 1, 1);
200 }
201
202 enc.copy_buffer_to_buffer(&out_buf, 0, &out_staging, 0, (n_out * 4) as u64);
203 self.queue.submit(Some(enc.finish()));
204
205 let slice = out_staging.slice(..);
207 let (tx, rx) = std::sync::mpsc::channel();
208 slice.map_async(etalon::MapMode::Read, move |r| {
209 let _ = tx.send(r);
210 });
211 let _ = dev.poll(etalon::PollType::wait_indefinitely());
212 rx.recv()
213 .map_err(|_| LoRAError::Io("GPU readback channel closed".into()))?
214 .map_err(|e| LoRAError::Io(format!("map_async: {e:?}")))?;
215
216 let mapped = slice
217 .get_mapped_range()
218 .expect("wgpu buffer map_range failed");
219 let result: &[f32] = bytemuck::cast_slice(&mapped);
220 output.copy_from_slice(result);
221 Ok(())
222 }
223}
224
225fn buf_rw(dev: &etalon::Device, data: &[u8]) -> etalon::Buffer {
228 use etalon::util::DeviceExt;
229 dev.create_buffer_init(&etalon::util::BufferInitDescriptor {
230 label: Some("lora-rw"),
231 contents: data,
232 usage: etalon::BufferUsages::STORAGE
233 | etalon::BufferUsages::COPY_SRC
234 | etalon::BufferUsages::COPY_DST,
235 })
236}
237
238fn buf_r(dev: &etalon::Device, data: &[u8]) -> etalon::Buffer {
239 use etalon::util::DeviceExt;
240 dev.create_buffer_init(&etalon::util::BufferInitDescriptor {
241 label: Some("lora-r"),
242 contents: data,
243 usage: etalon::BufferUsages::STORAGE,
244 })
245}
246
247fn buf_uniform(dev: &etalon::Device, data: &[u8]) -> etalon::Buffer {
248 use etalon::util::DeviceExt;
249 dev.create_buffer_init(&etalon::util::BufferInitDescriptor {
250 label: Some("lora-uniform"),
251 contents: data,
252 usage: etalon::BufferUsages::UNIFORM,
253 })
254}
255
256fn storage_rw_entry(binding: u32) -> etalon::BindGroupLayoutEntry {
257 etalon::BindGroupLayoutEntry {
258 binding,
259 visibility: etalon::ShaderStages::COMPUTE,
260 ty: etalon::BindingType::Buffer {
261 ty: etalon::BufferBindingType::Storage { read_only: false },
262 has_dynamic_offset: false,
263 min_binding_size: None,
264 },
265 count: None,
266 }
267}
268
269fn storage_r_entry(binding: u32) -> etalon::BindGroupLayoutEntry {
270 etalon::BindGroupLayoutEntry {
271 binding,
272 visibility: etalon::ShaderStages::COMPUTE,
273 ty: etalon::BindingType::Buffer {
274 ty: etalon::BufferBindingType::Storage { read_only: true },
275 has_dynamic_offset: false,
276 min_binding_size: None,
277 },
278 count: None,
279 }
280}
281
282fn uniform_entry(binding: u32) -> etalon::BindGroupLayoutEntry {
283 etalon::BindGroupLayoutEntry {
284 binding,
285 visibility: etalon::ShaderStages::COMPUTE,
286 ty: etalon::BindingType::Buffer {
287 ty: etalon::BufferBindingType::Uniform,
288 has_dynamic_offset: false,
289 min_binding_size: None,
290 },
291 count: None,
292 }
293}