Skip to main content

qualia_semantic_library/llm/
ollama.rs

1//! Ollama HTTP backend — an external LLM reached over its REST API.
2//!
3//! Uses blocking `reqwest` (this is a batch CLI tool, not an async service).
4//! `base_url`, model names and an optional bearer token are all configurable so
5//! the same backend also targets a hosted Ollama or a compatible proxy.
6
7use serde::Deserialize;
8
9use super::LlmBackend;
10
11/// Configuration for the Ollama backend.
12#[derive(Debug, Clone)]
13pub struct OllamaConfig {
14    /// e.g. `http://127.0.0.1:11434`.
15    pub base_url: String,
16    /// Embedding model (e.g. `nomic-embed-text`).
17    pub embed_model: String,
18    /// Generation model (e.g. `gemma4:e4b`).
19    pub gen_model: String,
20    /// Optional bearer token for hosted/proxied endpoints.
21    pub api_key: Option<String>,
22    /// Context window for generation.
23    pub num_ctx: u32,
24    pub timeout_secs: u64,
25}
26
27impl Default for OllamaConfig {
28    fn default() -> Self {
29        OllamaConfig {
30            base_url: std::env::var("OLLAMA_HOST")
31                .unwrap_or_else(|_| "http://127.0.0.1:11434".into()),
32            // Qwen3-Embedding (June 2025) tops the MTEB retrieval leaderboards;
33            // the 0.6B variant has 32k context and 1024-dim output — the best
34            // quality-per-size for a large technical corpus. Override via
35            // OllamaConfig / --embed-model.
36            embed_model: "qwen3-embedding:0.6b".into(),
37            gen_model: "gemma4:e4b".into(),
38            api_key: std::env::var("QSL_LLM_API_KEY").ok(),
39            num_ctx: 16384,
40            timeout_secs: 600,
41        }
42    }
43}
44
45pub struct OllamaBackend {
46    cfg: OllamaConfig,
47    client: reqwest::blocking::Client,
48    embed_dim: std::cell::Cell<u32>,
49}
50
51impl OllamaBackend {
52    /// Creates a new instance.
53    pub fn new(cfg: OllamaConfig) -> anyhow::Result<Self> {
54        let client = reqwest::blocking::Client::builder()
55            .timeout(std::time::Duration::from_secs(cfg.timeout_secs))
56            .build()?;
57        Ok(OllamaBackend {
58            cfg,
59            client,
60            embed_dim: std::cell::Cell::new(0),
61        })
62    }
63
64    fn post_json(
65        &self,
66        path: &str,
67        body: serde_json::Value,
68    ) -> anyhow::Result<reqwest::blocking::Response> {
69        let mut req = self
70            .client
71            .post(format!(
72                "{}{}",
73                self.cfg.base_url.trim_end_matches('/'),
74                path
75            ))
76            .json(&body);
77        if let Some(key) = &self.cfg.api_key {
78            req = req.bearer_auth(key);
79        }
80        let resp = req.send()?;
81        if !resp.status().is_success() {
82            anyhow::bail!("ollama {} -> HTTP {}", path, resp.status());
83        }
84        Ok(resp)
85    }
86}
87
88#[derive(Deserialize)]
89struct EmbeddingResp {
90    embedding: Vec<f32>,
91}
92
93#[derive(Deserialize)]
94struct GenerateResp {
95    response: String,
96}
97
98impl LlmBackend for OllamaBackend {
99    fn embed(&self, texts: &[String]) -> anyhow::Result<Vec<Vec<f32>>> {
100        let mut out = Vec::with_capacity(texts.len());
101        // /api/embeddings is one prompt per call — reliable across versions.
102        for t in texts {
103            let body = serde_json::json!({ "model": self.cfg.embed_model, "prompt": t });
104            let resp = self.post_json("/api/embeddings", body)?;
105            let parsed: EmbeddingResp = resp.json()?;
106            if self.embed_dim.get() == 0 {
107                self.embed_dim.set(parsed.embedding.len() as u32);
108            }
109            out.push(parsed.embedding);
110        }
111        Ok(out)
112    }
113
114    fn generate(&self, system: &str, prompt: &str) -> anyhow::Result<String> {
115        let body = serde_json::json!({
116            "model": self.cfg.gen_model,
117            "system": system,
118            "prompt": prompt,
119            "stream": false,
120            "options": { "temperature": 0.1, "num_ctx": self.cfg.num_ctx }
121        });
122        let resp = self.post_json("/api/generate", body)?;
123        let parsed: GenerateResp = resp.json()?;
124        Ok(parsed.response)
125    }
126
127    fn embed_dim(&self) -> u32 {
128        self.embed_dim.get()
129    }
130
131    fn model_id(&self) -> String {
132        format!("ollama/{}", self.cfg.embed_model)
133    }
134}