qualia_semantic_library/llm/
ollama.rs1use serde::Deserialize;
8
9use super::LlmBackend;
10
11#[derive(Debug, Clone)]
13pub struct OllamaConfig {
14 pub base_url: String,
16 pub embed_model: String,
18 pub gen_model: String,
20 pub api_key: Option<String>,
22 pub num_ctx: u32,
24 pub timeout_secs: u64,
25}
26
27impl Default for OllamaConfig {
28 fn default() -> Self {
29 OllamaConfig {
30 base_url: std::env::var("OLLAMA_HOST")
31 .unwrap_or_else(|_| "http://127.0.0.1:11434".into()),
32 embed_model: "qwen3-embedding:0.6b".into(),
37 gen_model: "gemma4:e4b".into(),
38 api_key: std::env::var("QSL_LLM_API_KEY").ok(),
39 num_ctx: 16384,
40 timeout_secs: 600,
41 }
42 }
43}
44
45pub struct OllamaBackend {
46 cfg: OllamaConfig,
47 client: reqwest::blocking::Client,
48 embed_dim: std::cell::Cell<u32>,
49}
50
51impl OllamaBackend {
52 pub fn new(cfg: OllamaConfig) -> anyhow::Result<Self> {
54 let client = reqwest::blocking::Client::builder()
55 .timeout(std::time::Duration::from_secs(cfg.timeout_secs))
56 .build()?;
57 Ok(OllamaBackend {
58 cfg,
59 client,
60 embed_dim: std::cell::Cell::new(0),
61 })
62 }
63
64 fn post_json(
65 &self,
66 path: &str,
67 body: serde_json::Value,
68 ) -> anyhow::Result<reqwest::blocking::Response> {
69 let mut req = self
70 .client
71 .post(format!(
72 "{}{}",
73 self.cfg.base_url.trim_end_matches('/'),
74 path
75 ))
76 .json(&body);
77 if let Some(key) = &self.cfg.api_key {
78 req = req.bearer_auth(key);
79 }
80 let resp = req.send()?;
81 if !resp.status().is_success() {
82 anyhow::bail!("ollama {} -> HTTP {}", path, resp.status());
83 }
84 Ok(resp)
85 }
86}
87
88#[derive(Deserialize)]
89struct EmbeddingResp {
90 embedding: Vec<f32>,
91}
92
93#[derive(Deserialize)]
94struct GenerateResp {
95 response: String,
96}
97
98impl LlmBackend for OllamaBackend {
99 fn embed(&self, texts: &[String]) -> anyhow::Result<Vec<Vec<f32>>> {
100 let mut out = Vec::with_capacity(texts.len());
101 for t in texts {
103 let body = serde_json::json!({ "model": self.cfg.embed_model, "prompt": t });
104 let resp = self.post_json("/api/embeddings", body)?;
105 let parsed: EmbeddingResp = resp.json()?;
106 if self.embed_dim.get() == 0 {
107 self.embed_dim.set(parsed.embedding.len() as u32);
108 }
109 out.push(parsed.embedding);
110 }
111 Ok(out)
112 }
113
114 fn generate(&self, system: &str, prompt: &str) -> anyhow::Result<String> {
115 let body = serde_json::json!({
116 "model": self.cfg.gen_model,
117 "system": system,
118 "prompt": prompt,
119 "stream": false,
120 "options": { "temperature": 0.1, "num_ctx": self.cfg.num_ctx }
121 });
122 let resp = self.post_json("/api/generate", body)?;
123 let parsed: GenerateResp = resp.json()?;
124 Ok(parsed.response)
125 }
126
127 fn embed_dim(&self) -> u32 {
128 self.embed_dim.get()
129 }
130
131 fn model_id(&self) -> String {
132 format!("ollama/{}", self.cfg.embed_model)
133 }
134}