1use serde::{Deserialize, Serialize};
14
15use crate::inference_backend::InferenceBackendSettings;
16
17pub const DEFAULT_OLLAMA_BASE_URL: &str = "http://127.0.0.1:11434";
19
20#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
22pub struct OllamaStatus {
23 pub reachable: bool,
24 pub base_url: String,
25 pub version_hint: Option<String>,
26 pub models: Vec<OllamaModelInfo>,
27 pub error: Option<String>,
28 pub latency_ms: u64,
29}
30
31#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
32pub struct OllamaModelInfo {
33 pub name: String,
34 pub size_bytes: Option<u64>,
35 pub parameter_size: Option<String>,
36 pub quantization: Option<String>,
37 pub family: Option<String>,
38}
39
40#[derive(Debug, Clone, Serialize, Deserialize)]
42pub struct OllamaGenerateResult {
43 pub text: String,
44 pub model: String,
45 pub total_duration_ns: Option<u64>,
46 pub eval_count: Option<u32>,
47 pub prompt_eval_count: Option<u32>,
48}
49
50#[derive(Debug, Clone)]
51pub struct OllamaHarness {
52 pub base_url: String,
53 pub gen_model: String,
54 pub embed_model: String,
55 pub api_key: Option<String>,
56 pub num_ctx: u32,
57 pub timeout_secs: u64,
58 pub temperature: f32,
59}
60
61impl Default for OllamaHarness {
62 fn default() -> Self {
63 Self::from_settings(&InferenceBackendSettings::default())
64 }
65}
66
67impl OllamaHarness {
68 pub fn from_settings(settings: &InferenceBackendSettings) -> Self {
70 let base = settings
71 .ollama_base_url
72 .trim()
73 .trim_end_matches('/')
74 .to_string();
75 let base = if base.is_empty() {
76 std::env::var("OLLAMA_HOST").unwrap_or_else(|_| DEFAULT_OLLAMA_BASE_URL.to_string())
77 } else {
78 base
79 };
80 let gen = if settings.ollama_model.trim().is_empty() {
81 std::env::var("OLLAMA_MODEL").unwrap_or_else(|_| "llama3.2".to_string())
82 } else {
83 settings.ollama_model.trim().to_string()
84 };
85 let embed = if settings.ollama_embed_model.trim().is_empty() {
86 "nomic-embed-text".to_string()
87 } else {
88 settings.ollama_embed_model.trim().to_string()
89 };
90 Self {
91 base_url: base.trim_end_matches('/').to_string(),
92 gen_model: gen,
93 embed_model: embed,
94 api_key: settings
95 .ollama_api_key
96 .as_ref()
97 .map(|s| s.trim().to_string())
98 .filter(|s| !s.is_empty()),
99 num_ctx: settings.ollama_num_ctx.max(512),
100 timeout_secs: settings.ollama_timeout_secs.clamp(5, 3600),
101 temperature: settings.ollama_temperature.clamp(0.0, 2.0),
102 }
103 }
104
105 pub fn from_loaded_settings() -> Self {
107 Self::from_settings(&crate::inference_backend::load_inference_backend_settings())
108 }
109
110 fn url(&self, path: &str) -> String {
111 format!(
112 "{}{}",
113 self.base_url.trim_end_matches('/'),
114 if path.starts_with('/') {
115 path.to_string()
116 } else {
117 format!("/{path}")
118 }
119 )
120 }
121
122 fn apply_auth(
123 &self,
124 mut req: reqwest::blocking::RequestBuilder,
125 ) -> reqwest::blocking::RequestBuilder {
126 if let Some(key) = &self.api_key {
127 req = req.bearer_auth(key);
128 }
129 req
130 }
131
132 fn apply_auth_async(&self, mut req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
133 if let Some(key) = &self.api_key {
134 req = req.bearer_auth(key);
135 }
136 req
137 }
138
139 fn blocking_client(&self) -> Result<reqwest::blocking::Client, String> {
140 reqwest::blocking::Client::builder()
141 .timeout(std::time::Duration::from_secs(self.timeout_secs))
142 .build()
143 .map_err(|e| format!("ollama client: {e}"))
144 }
145
146 async fn async_client(&self) -> Result<reqwest::Client, String> {
147 reqwest::Client::builder()
148 .timeout(std::time::Duration::from_secs(self.timeout_secs))
149 .build()
150 .map_err(|e| format!("ollama client: {e}"))
151 }
152
153 pub fn probe_status(&self) -> OllamaStatus {
155 let t0 = std::time::Instant::now();
156 let client = match self.blocking_client() {
157 Ok(c) => c,
158 Err(e) => {
159 return OllamaStatus {
160 reachable: false,
161 base_url: self.base_url.clone(),
162 version_hint: None,
163 models: vec![],
164 error: Some(e),
165 latency_ms: 0,
166 };
167 }
168 };
169 let req = self.apply_auth(client.get(self.url("/api/tags")));
170 match req.send() {
171 Ok(resp) if resp.status().is_success() => {
172 let latency_ms = t0.elapsed().as_millis() as u64;
173 match resp.json::<TagsResponse>() {
174 Ok(tags) => OllamaStatus {
175 reachable: true,
176 base_url: self.base_url.clone(),
177 version_hint: Some("ollama".into()),
178 models: tags
179 .models
180 .into_iter()
181 .map(|m| OllamaModelInfo {
182 name: m.name,
183 size_bytes: m.size,
184 parameter_size: m
185 .details
186 .as_ref()
187 .and_then(|d| d.parameter_size.clone()),
188 quantization: m
189 .details
190 .as_ref()
191 .and_then(|d| d.quantization_level.clone()),
192 family: m.details.as_ref().and_then(|d| d.family.clone()),
193 })
194 .collect(),
195 error: None,
196 latency_ms,
197 },
198 Err(e) => OllamaStatus {
199 reachable: true,
200 base_url: self.base_url.clone(),
201 version_hint: None,
202 models: vec![],
203 error: Some(format!("parse /api/tags: {e}")),
204 latency_ms,
205 },
206 }
207 }
208 Ok(resp) => OllamaStatus {
209 reachable: false,
210 base_url: self.base_url.clone(),
211 version_hint: None,
212 models: vec![],
213 error: Some(format!("HTTP {}", resp.status())),
214 latency_ms: t0.elapsed().as_millis() as u64,
215 },
216 Err(e) => OllamaStatus {
217 reachable: false,
218 base_url: self.base_url.clone(),
219 version_hint: None,
220 models: vec![],
221 error: Some(e.to_string()),
222 latency_ms: t0.elapsed().as_millis() as u64,
223 },
224 }
225 }
226
227 pub async fn probe_status_async(&self) -> OllamaStatus {
229 let t0 = std::time::Instant::now();
230 let client = match self.async_client().await {
231 Ok(c) => c,
232 Err(e) => {
233 return OllamaStatus {
234 reachable: false,
235 base_url: self.base_url.clone(),
236 version_hint: None,
237 models: vec![],
238 error: Some(e),
239 latency_ms: 0,
240 };
241 }
242 };
243 let req = self.apply_auth_async(client.get(self.url("/api/tags")));
244 match req.send().await {
245 Ok(resp) if resp.status().is_success() => {
246 let latency_ms = t0.elapsed().as_millis() as u64;
247 match resp.json::<TagsResponse>().await {
248 Ok(tags) => OllamaStatus {
249 reachable: true,
250 base_url: self.base_url.clone(),
251 version_hint: Some("ollama".into()),
252 models: tags
253 .models
254 .into_iter()
255 .map(|m| OllamaModelInfo {
256 name: m.name,
257 size_bytes: m.size,
258 parameter_size: m
259 .details
260 .as_ref()
261 .and_then(|d| d.parameter_size.clone()),
262 quantization: m
263 .details
264 .as_ref()
265 .and_then(|d| d.quantization_level.clone()),
266 family: m.details.as_ref().and_then(|d| d.family.clone()),
267 })
268 .collect(),
269 error: None,
270 latency_ms,
271 },
272 Err(e) => OllamaStatus {
273 reachable: true,
274 base_url: self.base_url.clone(),
275 version_hint: None,
276 models: vec![],
277 error: Some(format!("parse /api/tags: {e}")),
278 latency_ms,
279 },
280 }
281 }
282 Ok(resp) => OllamaStatus {
283 reachable: false,
284 base_url: self.base_url.clone(),
285 version_hint: None,
286 models: vec![],
287 error: Some(format!("HTTP {}", resp.status())),
288 latency_ms: t0.elapsed().as_millis() as u64,
289 },
290 Err(e) => OllamaStatus {
291 reachable: false,
292 base_url: self.base_url.clone(),
293 version_hint: None,
294 models: vec![],
295 error: Some(e.to_string()),
296 latency_ms: t0.elapsed().as_millis() as u64,
297 },
298 }
299 }
300
301 pub fn generate(&self, system: &str, prompt: &str) -> Result<OllamaGenerateResult, String> {
303 self.generate_with_options(system, prompt, None)
304 }
305
306 pub fn generate_with_options(
309 &self,
310 system: &str,
311 prompt: &str,
312 max_output_tokens: Option<u32>,
313 ) -> Result<OllamaGenerateResult, String> {
314 let client = self.blocking_client()?;
315 let mut options = serde_json::json!({
316 "temperature": self.temperature,
317 "num_ctx": self.num_ctx,
318 });
319 if let Some(tokens) = max_output_tokens {
320 options["num_predict"] = serde_json::json!(tokens.max(1));
321 }
322 let body = serde_json::json!({
323 "model": self.gen_model,
324 "system": system,
325 "prompt": prompt,
326 "stream": false,
327 "options": options,
328 });
329 let req = self.apply_auth(client.post(self.url("/api/generate")).json(&body));
330 let resp = req.send().map_err(|e| format!("ollama generate: {e}"))?;
331 if !resp.status().is_success() {
332 let status = resp.status();
333 let body = resp.text().unwrap_or_default();
334 return Err(format!("ollama generate HTTP {status}: {body}"));
335 }
336 let parsed: GenerateResponse = resp
337 .json()
338 .map_err(|e| format!("ollama generate parse: {e}"))?;
339 Ok(OllamaGenerateResult {
340 text: parsed.response,
341 model: parsed.model.unwrap_or_else(|| self.gen_model.clone()),
342 total_duration_ns: parsed.total_duration,
343 eval_count: parsed.eval_count,
344 prompt_eval_count: parsed.prompt_eval_count,
345 })
346 }
347
348 pub async fn generate_async(
350 &self,
351 system: &str,
352 prompt: &str,
353 ) -> Result<OllamaGenerateResult, String> {
354 let client = self.async_client().await?;
355 let body = serde_json::json!({
356 "model": self.gen_model,
357 "system": system,
358 "prompt": prompt,
359 "stream": false,
360 "options": {
361 "temperature": self.temperature,
362 "num_ctx": self.num_ctx,
363 }
364 });
365 let req = self.apply_auth_async(client.post(self.url("/api/generate")).json(&body));
366 let resp = req
367 .send()
368 .await
369 .map_err(|e| format!("ollama generate: {e}"))?;
370 if !resp.status().is_success() {
371 let status = resp.status();
372 let body = resp.text().await.unwrap_or_default();
373 return Err(format!("ollama generate HTTP {status}: {body}"));
374 }
375 let parsed: GenerateResponse = resp
376 .json()
377 .await
378 .map_err(|e| format!("ollama generate parse: {e}"))?;
379 Ok(OllamaGenerateResult {
380 text: parsed.response,
381 model: parsed.model.unwrap_or_else(|| self.gen_model.clone()),
382 total_duration_ns: parsed.total_duration,
383 eval_count: parsed.eval_count,
384 prompt_eval_count: parsed.prompt_eval_count,
385 })
386 }
387
388 pub fn chat(&self, system: &str, user: &str) -> Result<OllamaGenerateResult, String> {
390 let client = self.blocking_client()?;
391 let body = serde_json::json!({
392 "model": self.gen_model,
393 "stream": false,
394 "messages": [
395 { "role": "system", "content": system },
396 { "role": "user", "content": user },
397 ],
398 "options": {
399 "temperature": self.temperature,
400 "num_ctx": self.num_ctx,
401 }
402 });
403 let req = self.apply_auth(client.post(self.url("/api/chat")).json(&body));
404 let resp = req.send().map_err(|e| format!("ollama chat: {e}"))?;
405 if !resp.status().is_success() {
406 let status = resp.status();
407 let body = resp.text().unwrap_or_default();
408 return Err(format!("ollama chat HTTP {status}: {body}"));
409 }
410 let parsed: ChatResponse = resp.json().map_err(|e| format!("ollama chat parse: {e}"))?;
411 Ok(OllamaGenerateResult {
412 text: parsed.message.map(|m| m.content).unwrap_or_default(),
413 model: parsed.model.unwrap_or_else(|| self.gen_model.clone()),
414 total_duration_ns: parsed.total_duration,
415 eval_count: parsed.eval_count,
416 prompt_eval_count: parsed.prompt_eval_count,
417 })
418 }
419
420 pub fn embed(&self, text: &str) -> Result<Vec<f32>, String> {
422 let client = self.blocking_client()?;
423 let body = serde_json::json!({
424 "model": self.embed_model,
425 "prompt": text,
426 });
427 let req = self.apply_auth(client.post(self.url("/api/embeddings")).json(&body));
428 let resp = req.send().map_err(|e| format!("ollama embed: {e}"))?;
429 if !resp.status().is_success() {
430 let status = resp.status();
431 let body = resp.text().unwrap_or_default();
432 return Err(format!("ollama embed HTTP {status}: {body}"));
433 }
434 let parsed: EmbeddingResponse = resp
435 .json()
436 .map_err(|e| format!("ollama embed parse: {e}"))?;
437 Ok(parsed.embedding)
438 }
439}
440
441pub fn probe_configured_ollama() -> OllamaStatus {
443 OllamaHarness::from_loaded_settings().probe_status()
444}
445
446pub async fn probe_configured_ollama_async() -> OllamaStatus {
448 OllamaHarness::from_loaded_settings()
449 .probe_status_async()
450 .await
451}
452
453#[derive(Debug, Deserialize)]
456struct TagsResponse {
457 #[serde(default)]
458 models: Vec<TagModel>,
459}
460
461#[derive(Debug, Deserialize)]
462struct TagModel {
463 name: String,
464 #[serde(default)]
465 size: Option<u64>,
466 #[serde(default)]
467 details: Option<TagDetails>,
468}
469
470#[derive(Debug, Deserialize)]
471struct TagDetails {
472 #[serde(default)]
473 parameter_size: Option<String>,
474 #[serde(default)]
475 quantization_level: Option<String>,
476 #[serde(default)]
477 family: Option<String>,
478}
479
480#[derive(Debug, Deserialize)]
481struct GenerateResponse {
482 response: String,
483 #[serde(default)]
484 model: Option<String>,
485 #[serde(default)]
486 total_duration: Option<u64>,
487 #[serde(default)]
488 eval_count: Option<u32>,
489 #[serde(default)]
490 prompt_eval_count: Option<u32>,
491}
492
493#[derive(Debug, Deserialize)]
494struct ChatResponse {
495 #[serde(default)]
496 message: Option<ChatMessage>,
497 #[serde(default)]
498 model: Option<String>,
499 #[serde(default)]
500 total_duration: Option<u64>,
501 #[serde(default)]
502 eval_count: Option<u32>,
503 #[serde(default)]
504 prompt_eval_count: Option<u32>,
505}
506
507#[derive(Debug, Deserialize)]
508struct ChatMessage {
509 #[serde(default)]
510 content: String,
511}
512
513#[derive(Debug, Deserialize)]
514struct EmbeddingResponse {
515 embedding: Vec<f32>,
516}
517
518#[cfg(test)]
519mod tests {
520 use super::*;
521 use crate::chat_agents::AgentBackendKind;
522 use crate::inference_backend::InferenceBackendSettings;
523
524 #[test]
525 fn harness_from_settings_defaults() {
526 let s = InferenceBackendSettings {
527 backend: AgentBackendKind::Ollama,
528 ollama_base_url: "http://127.0.0.1:11434/".into(),
529 ollama_model: "qwen2.5:7b".into(),
530 ollama_embed_model: "nomic-embed-text".into(),
531 ollama_timeout_secs: 120,
532 ollama_num_ctx: 8192,
533 ollama_temperature: 0.2,
534 ..Default::default()
535 };
536 let h = OllamaHarness::from_settings(&s);
537 assert_eq!(h.base_url, "http://127.0.0.1:11434");
538 assert_eq!(h.gen_model, "qwen2.5:7b");
539 assert_eq!(h.num_ctx, 8192);
540 }
541
542 #[test]
543 fn unreachable_probe_is_fail_closed() {
544 let h = OllamaHarness {
545 base_url: "http://127.0.0.1:1".into(),
546 gen_model: "x".into(),
547 embed_model: "y".into(),
548 api_key: None,
549 num_ctx: 2048,
550 timeout_secs: 1,
551 temperature: 0.1,
552 };
553 let st = h.probe_status();
554 assert!(!st.reachable);
555 assert!(st.error.is_some());
556 }
557}