Skip to main content

qualia_core_db/services/
poet_llm_api.rs

1//! Local-model chat boundary for the standalone POET browser.
2
3use std::path::Path;
4
5use axum::{
6    body::Bytes,
7    extract::State,
8    http::{HeaderMap, StatusCode},
9    response::{IntoResponse, Response},
10    Json,
11};
12use serde::Deserialize;
13
14use crate::llm_agent::{AgentIntent, AgentRuntime, LocalLlmAgent, WebizenVerdict};
15use crate::modalities::logic::n3_compiler::N3OutputMode;
16
17use super::poet_llm_conditioning::prepare_prompt;
18
19pub const LLM_REQUEST_LIMIT_BYTES: usize = 128 * 1024;
20const MAX_PROMPT_BYTES: usize = 32 * 1024;
21const MAX_CONTEXT_BYTES: usize = 64 * 1024;
22
23#[derive(Debug, Deserialize)]
24pub struct PoetLlmRequest {
25    pub model_path: String,
26    pub prompt: String,
27    #[serde(default)]
28    pub conditioning: Option<serde_json::Value>,
29    #[serde(default)]
30    pub graph_context: String,
31    #[serde(default = "default_agent_did")]
32    pub agent_did: String,
33    #[serde(default)]
34    pub principal_did: String,
35    #[serde(default = "default_max_tokens")]
36    pub max_tokens: u32,
37    #[serde(default)]
38    pub library_projects: Vec<String>,
39    #[serde(default)]
40    pub library_context_supplied: bool,
41}
42
43fn default_agent_did() -> String {
44    "did:qualia:poet-local-agent".into()
45}
46
47fn default_max_tokens() -> u32 {
48    256
49}
50
51/// Return only model files explicitly mounted, configured through
52/// `QUALIA_MODEL_PATHS`, or present in QualiaDB's documented model locations.
53/// Directories are scanned one level deep and the result is bounded.
54pub async fn models_handler() -> Response {
55    Json(serde_json::json!({
56        "ok": true,
57        "honesty": "live-local-catalogue",
58        "data": { "models": discover_local_models() }
59    }))
60    .into_response()
61}
62
63#[derive(Deserialize)]
64struct ModelLifecycleRequest {
65    model_path: String,
66}
67
68/// HTTP `activate-model` handler.
69pub async fn activate_model_handler(body: Bytes) -> Response {
70    if body.len() > 4096 {
71        return failure(
72            StatusCode::PAYLOAD_TOO_LARGE,
73            "model_lifecycle_payload",
74            "Model activation requests are limited to 4 KiB",
75        );
76    }
77    if super::poet_llm_jobs::has_active_jobs() {
78        return failure(
79            StatusCode::CONFLICT,
80            "model_in_use",
81            "Cancel or finish active local-model jobs before changing the resident model",
82        );
83    }
84    let request: ModelLifecycleRequest = match serde_json::from_slice(&body) {
85        Ok(request) => request,
86        Err(error) => return failure(StatusCode::BAD_REQUEST, "invalid_json", error.to_string()),
87    };
88    let path = request.model_path.trim().to_string();
89    if let Err(message) = validate_model_file(&path) {
90        return failure(StatusCode::BAD_REQUEST, "invalid_model", message);
91    }
92    let model_id = crate::q_hash(&path);
93    let mounted_path = path.clone();
94    match tokio::task::spawn_blocking(move || {
95        crate::resident_model::mount_resident_model(model_id, &mounted_path, false)
96    })
97    .await
98    {
99        Ok(Ok(report)) => Json(serde_json::json!({
100            "ok": true,
101            "honesty": "resident-model-active",
102            "data": {
103                "model_id": format!("model:{model_id:016x}"),
104                "model_path": crate::resident_model::resident_gguf_path().unwrap_or(path),
105                "mapped_bytes": report.mapped_bytes,
106                "layers": report.n_layer,
107                "heads": report.n_head,
108                "kv_heads": report.n_kv_head,
109                "directml": report.directml_enabled
110            }
111        }))
112        .into_response(),
113        Ok(Err(message)) => failure(
114            StatusCode::UNPROCESSABLE_ENTITY,
115            "model_activation_failed",
116            message,
117        ),
118        Err(error) => failure(
119            StatusCode::INTERNAL_SERVER_ERROR,
120            "model_activation_worker",
121            error.to_string(),
122        ),
123    }
124}
125
126/// HTTP `evict-model` handler.
127pub async fn evict_model_handler() -> Response {
128    if super::poet_llm_jobs::has_active_jobs() {
129        return failure(
130            StatusCode::CONFLICT,
131            "model_in_use",
132            "Cancel or finish active local-model jobs before evicting the resident model",
133        );
134    }
135    let previous = crate::resident_model::resident_gguf_path();
136    crate::resident_model::clear_resident_model();
137    Json(serde_json::json!({
138        "ok": true,
139        "honesty": "resident-model-evicted",
140        "data": { "previous_model_path": previous }
141    }))
142    .into_response()
143}
144
145fn validate_model_file(path: &str) -> Result<(), String> {
146    let path = Path::new(path);
147    let extension = path
148        .extension()
149        .and_then(|value| value.to_str())
150        .unwrap_or_default()
151        .to_ascii_lowercase();
152    if !matches!(extension.as_str(), "gguf" | "p64" | "safetensors") {
153        return Err("Select a local .gguf, .p64, or .safetensors model".into());
154    }
155    if !path.is_file() {
156        return Err("The selected local model file does not exist".into());
157    }
158    Ok(())
159}
160
161fn discover_local_models() -> Vec<serde_json::Value> {
162    const MAX_MODELS: usize = 64;
163    const KNOWN_MODELS: &[&str] = &[
164        r"C:\LLM_Models\P64\smollm2-360m-instruct-q8_0.f16.p64",
165        r"C:\LLM_Models\P64\smollm2-360m-instruct-q8_0.p64",
166        r"C:\LLM_Models\GGUF\smollm2-360m-instruct-q8_0.gguf",
167        r"C:\LLM_Models\GGUF\lmstudio-community\smollm2-360m-instruct-q8_0.gguf",
168    ];
169
170    let resident = crate::resident_model::resident_gguf_path();
171    let mut candidates = Vec::new();
172    if let Some(path) = resident.as_deref() {
173        push_model_path(&mut candidates, Path::new(path), MAX_MODELS);
174    }
175    if let Some(configured) = std::env::var_os("QUALIA_MODEL_PATHS").or_else(|| std::env::var_os("QUALIA_LLM_VAULT")) {
176        for path in std::env::split_paths(&configured).take(MAX_MODELS) {
177            if path.is_dir() {
178                scan_dir_for_models(&mut candidates, &path, 4, MAX_MODELS);
179            } else {
180                push_model_path(&mut candidates, &path, MAX_MODELS);
181            }
182            if candidates.len() >= MAX_MODELS {
183                break;
184            }
185        }
186    }
187    for path in KNOWN_MODELS {
188        push_model_path(&mut candidates, Path::new(path), MAX_MODELS);
189    }
190
191    // Common system model directories
192    let common_dirs = [
193        std::path::PathBuf::from(r"C:\LLM_files"),
194        std::path::PathBuf::from(r"C:\LLM_Models"),
195        std::path::PathBuf::from(r"C:\models"),
196        std::path::PathBuf::from("storage/Models"),
197        std::path::PathBuf::from("storage/models"),
198    ];
199    for dir in &common_dirs {
200        scan_dir_for_models(&mut candidates, dir, 4, MAX_MODELS);
201    }
202
203    // User home directory model locations
204    if let Some(home) = std::env::var_os("USERPROFILE").or_else(|| std::env::var_os("HOME")) {
205        let home_path = std::path::PathBuf::from(home);
206        let user_dirs = [
207            home_path.join("models"),
208            home_path.join(".cache").join("lm-studio").join("models"),
209            home_path.join(".qualia").join("models"),
210            home_path
211                .join("AppData")
212                .join("Roaming")
213                .join("qualia")
214                .join("Models"),
215            home_path
216                .join("AppData")
217                .join("Local")
218                .join("qualia")
219                .join("Models"),
220            home_path.join("Downloads"),
221        ];
222        for dir in &user_dirs {
223            // Shallow scan for Downloads (depth 1) to avoid heavy traversal
224            let depth = if dir.ends_with("Downloads") { 1 } else { 2 };
225            scan_dir_for_models(&mut candidates, dir, depth, MAX_MODELS);
226        }
227    }
228
229    candidates.sort();
230    candidates.dedup();
231    candidates
232        .into_iter()
233        .map(|path| {
234            let metadata = std::fs::metadata(&path).ok();
235            let path_text = path.to_string_lossy().into_owned();
236            let name = path
237                .file_name()
238                .and_then(|value| value.to_str())
239                .unwrap_or("local model");
240            let format = path
241                .extension()
242                .and_then(|value| value.to_str())
243                .unwrap_or_default()
244                .to_ascii_lowercase();
245            serde_json::json!({
246                "id": format!("model:{:016x}", crate::q_hash(&path_text)),
247                "name": name,
248                "path": path_text,
249                "format": format,
250                "bytes": metadata.map(|value| value.len()).unwrap_or(0),
251                "resident": resident.as_deref() == Some(path.to_string_lossy().as_ref())
252            })
253        })
254        .collect()
255}
256
257fn scan_dir_for_models(out: &mut Vec<std::path::PathBuf>, dir: &Path, depth: usize, limit: usize) {
258    if depth == 0 || out.len() >= limit || !dir.is_dir() {
259        return;
260    }
261    if let Ok(entries) = std::fs::read_dir(dir) {
262        for entry in entries.flatten() {
263            if out.len() >= limit {
264                break;
265            }
266            let p = entry.path();
267            if p.is_file() {
268                push_model_path(out, &p, limit);
269            } else if p.is_dir() && depth > 1 {
270                scan_dir_for_models(out, &p, depth - 1, limit);
271            }
272        }
273    }
274}
275
276fn push_model_path(out: &mut Vec<std::path::PathBuf>, path: &Path, limit: usize) {
277    if out.len() >= limit || !path.is_file() {
278        return;
279    }
280    let extension = path
281        .extension()
282        .and_then(|value| value.to_str())
283        .unwrap_or_default();
284    if !matches!(extension.to_ascii_lowercase().as_str(), "gguf" | "p64") {
285        return;
286    }
287    out.push(path.canonicalize().unwrap_or_else(|_| path.to_path_buf()));
288}
289
290fn failure(status: StatusCode, code: &'static str, message: impl Into<String>) -> Response {
291    (
292        status,
293        Json(serde_json::json!({
294            "ok": false,
295            "honesty": "unavailable",
296            "code": code,
297            "diagnostic": message.into()
298        })),
299    )
300        .into_response()
301}
302
303/// Decodes `request`; fails closed on malformed input.
304pub(crate) fn decode_request(body: &Bytes) -> Result<PoetLlmRequest, Response> {
305    if body.len() > LLM_REQUEST_LIMIT_BYTES {
306        return Err(failure(
307            StatusCode::PAYLOAD_TOO_LARGE,
308            "payload_too_large",
309            "Local LLM requests are limited to 128 KiB",
310        ));
311    }
312    let mut request: PoetLlmRequest = serde_json::from_slice(body)
313        .map_err(|error| failure(StatusCode::BAD_REQUEST, "invalid_json", error.to_string()))?;
314    request.model_path = request.model_path.trim().to_string();
315    request.prompt = request.prompt.trim().to_string();
316    request.agent_did = request.agent_did.trim().to_string();
317    request.principal_did = request.principal_did.trim().to_string();
318    if request.prompt.is_empty() || request.prompt.len() > MAX_PROMPT_BYTES {
319        return Err(failure(
320            StatusCode::BAD_REQUEST,
321            "invalid_prompt",
322            "Prompt must contain 1..=32768 bytes",
323        ));
324    }
325    if request.graph_context.len() > MAX_CONTEXT_BYTES {
326        return Err(failure(
327            StatusCode::BAD_REQUEST,
328            "context_too_large",
329            "Grounding context is limited to 64 KiB",
330        ));
331    }
332    if request.agent_did.is_empty() || !request.agent_did.starts_with("did:") {
333        return Err(failure(
334            StatusCode::BAD_REQUEST,
335            "invalid_agent",
336            "Agent identity must be a DID",
337        ));
338    }
339    if !(1..=256).contains(&request.max_tokens) {
340        return Err(failure(
341            StatusCode::BAD_REQUEST,
342            "invalid_token_budget",
343            "Local model token budget must be between 1 and 256",
344        ));
345    }
346    if request.library_projects.len() > 32
347        || request
348            .library_projects
349            .iter()
350            .any(|project| project.is_empty() || project.len() > 128)
351    {
352        return Err(failure(
353            StatusCode::BAD_REQUEST,
354            "invalid_library_scope",
355            "Library grounding supports at most 32 non-empty project tags of 128 bytes",
356        ));
357    }
358    if let Err(message) = validate_model_file(&request.model_path) {
359        return Err(failure(StatusCode::BAD_REQUEST, "model_not_found", message));
360    }
361    if let Err(message) = super::poet_record_api::authorize_agent_run(
362        &request.agent_did,
363        &request.principal_did,
364        &request.model_path,
365        &request.library_projects,
366        request.library_context_supplied,
367        request.max_tokens,
368    ) {
369        return Err(failure(
370            StatusCode::FORBIDDEN,
371            "agent_authority_denied",
372            message,
373        ));
374    }
375    Ok(request)
376}
377
378/// Generates `handler`.
379pub async fn generate_handler(
380    State(state): State<std::sync::Arc<super::webizen_server::WebizenState>>,
381    headers: HeaderMap,
382    body: Bytes,
383) -> Response {
384    let session = match super::poet_llm_authority::authenticate(&state, &headers) {
385        Ok(session) => session,
386        Err(response) => return response,
387    };
388    let request = match decode_request(&body) {
389        Ok(request) => request,
390        Err(response) => return response,
391    };
392    match tokio::task::spawn_blocking(move || run_local_turn(request, session)).await {
393        Ok(Ok(value)) => Json(value).into_response(),
394        Ok(Err(message)) => failure(
395            StatusCode::UNPROCESSABLE_ENTITY,
396            "inference_failed",
397            message,
398        ),
399        Err(error) => failure(
400            StatusCode::INTERNAL_SERVER_ERROR,
401            "inference_worker_failed",
402            error.to_string(),
403        ),
404    }
405}
406
407fn run_local_turn(
408    request: PoetLlmRequest,
409    session: std::sync::Arc<crate::llm_agent::native_request::NativeProductSession>,
410) -> Result<serde_json::Value, String> {
411    let prepared = prepare_prompt(&request)?;
412    let agent = LocalLlmAgent::new(&request.agent_did, &request.model_path);
413    super::poet_llm_authority::bind(&agent, &session)?;
414    let context_hash = crate::q_hash(&request.graph_context);
415    let intent = AgentIntent {
416        intent_predicate: crate::q_hash("llm:ReadGraph"),
417        requested_graph_scope: if request.graph_context.is_empty() {
418            Vec::new()
419        } else {
420            vec![context_hash]
421        },
422        context_namespaces: Vec::new(),
423        requires_network: false,
424        ilp_offer_micro_cents: 0,
425        principal_did_hash: crate::q_hash(&request.principal_did),
426        mcp_intent_frame_hash: crate::q_hash("poet:local-chat-turn"),
427        output_mode: N3OutputMode::FreeText,
428        clearance_ceiling: 0,
429        max_sentinel_depth: 32,
430        active_profile: None,
431    };
432    match agent.validate_intent(&intent) {
433        WebizenVerdict::Permit => {}
434        verdict => return Err(format!("Webizen rejected the model intent: {verdict:?}")),
435    }
436
437    let output = agent
438        .infer(&prepared.text, &request.graph_context)
439        .map_err(|error| format!("{error:?}"))?;
440    match agent.validate_output(&output) {
441        WebizenVerdict::Permit | WebizenVerdict::Sanitised { .. } => {}
442        verdict => return Err(format!("Webizen rejected the model output: {verdict:?}")),
443    }
444    let verified =
445        crate::inference::post_turn_verify::maybe_verify_turn(&prepared.text, &output.text);
446    let checks = verified
447        .checks
448        .iter()
449        .map(|check| {
450            serde_json::json!({
451                "id": check.id,
452                "ok": check.ok,
453                "detail": check.detail
454            })
455        })
456        .collect::<Vec<_>>();
457
458    Ok(serde_json::json!({
459        "ok": true,
460        "honesty": "live-local-model",
461        "assertion_status": "model_assertion_requires_verification",
462        "agent_did": request.agent_did,
463        "model_path": request.model_path,
464        "text": verified.final_text,
465        "draft": output.text,
466        "tokens_generated": output.tokens_generated,
467        "inference_duration_ms": output.inference_duration_ms,
468        "provenance_hashes": output.provenance_quins,
469        "context_hash": context_hash,
470        "context_supplied": !request.graph_context.is_empty(),
471        "conditioning": {
472            "applied": prepared.plan_id.is_some(),
473            "plan_id": prepared.plan_id,
474            "profile_id": prepared.profile_id,
475            "selected_evidence": prepared.selected_evidence,
476            "token_budget": prepared.token_budget,
477        },
478        "repaired": verified.repaired,
479        "checks": checks,
480        "semantic_quin": output.semantic_quin
481    }))
482}
483
484#[cfg(test)]
485mod tests {
486    use super::*;
487
488    fn request_with_conditioning(conditioning: serde_json::Value) -> PoetLlmRequest {
489        PoetLlmRequest {
490            model_path: "test.gguf".into(),
491            prompt: "Summarise the agreement.".into(),
492            conditioning: Some(conditioning),
493            graph_context: String::new(),
494            agent_did: "did:qualia:test-agent".into(),
495            principal_did: "did:qualia:test-principal".into(),
496            max_tokens: 128,
497            library_projects: Vec::new(),
498            library_context_supplied: false,
499        }
500    }
501
502    #[test]
503    fn conditioning_is_compiled_into_the_prompt_used_for_inference() {
504        let request = request_with_conditioning(serde_json::json!({
505            "profile_id": "urn:qualia:profile:agreement-summary",
506            "objective": "Summarise the agreement for its signatories.",
507            "requirements": [{
508                "id": "plain-language",
509                "class": "guidance",
510                "rule": "Use clear language and name any unresolved terms.",
511                "priority": 20
512            }],
513            "evidence": [{
514                "source_id": "agreement-v3",
515                "content": "The parties agreed to publish a quarterly report.",
516                "sensitivity": 0
517            }],
518            "budget": {"input_tokens": 512, "output_tokens": 48, "tool_rounds": 0}
519        }));
520
521        let prepared = prepare_prompt(&request).expect("profile compiles");
522        assert!(prepared.plan_id.is_some());
523        assert_eq!(
524            prepared.profile_id.as_deref(),
525            Some("urn:qualia:profile:agreement-summary")
526        );
527        assert_eq!(prepared.token_budget, 48);
528        assert_eq!(prepared.selected_evidence, 1);
529        assert!(prepared.text.contains("Objective: Summarise the agreement"));
530        assert!(prepared.text.contains("Use clear language"));
531        assert!(prepared.text.contains("publish a quarterly report"));
532    }
533
534    #[test]
535    fn conditioning_rejects_evidence_above_the_disclosure_ceiling() {
536        let request = request_with_conditioning(serde_json::json!({
537            "profile_id": "urn:qualia:profile:restricted",
538            "requirements": [{
539                "id": "source-needed",
540                "class": "evidence_obligation",
541                "rule": "Ground the answer in the authorised record.",
542                "required": true
543            }],
544            "evidence": [{
545                "source_id": "restricted-record",
546                "content": "Not disclosed.",
547                "sensitivity": 2
548            }],
549            "disclosure_ceiling": 0,
550            "budget": {"input_tokens": 512, "output_tokens": 48, "tool_rounds": 0}
551        }));
552
553        let prepared = prepare_prompt(&request).expect("plan records rejected evidence");
554        assert_eq!(prepared.selected_evidence, 0);
555        assert!(!prepared.text.contains("Not disclosed."));
556    }
557
558    #[test]
559    fn test_scan_dir_for_models_finds_gguf_and_p64() {
560        let temp_dir = tempfile::tempdir().expect("tempdir created");
561        let root = temp_dir.path();
562        let sub = root.join("nested");
563        std::fs::create_dir_all(&sub).expect("create nested");
564
565        let m1 = root.join("model_a.gguf");
566        let m2 = sub.join("model_b.p64");
567        let ignored = root.join("notes.txt");
568
569        std::fs::write(&m1, b"dummy gguf").expect("write m1");
570        std::fs::write(&m2, b"dummy p64").expect("write m2");
571        std::fs::write(&ignored, b"text").expect("write txt");
572
573        let mut results = Vec::new();
574        scan_dir_for_models(&mut results, root, 2, 10);
575
576        assert_eq!(results.len(), 2);
577        assert!(results.iter().any(|p| p.ends_with("model_a.gguf")));
578        assert!(results.iter().any(|p| p.ends_with("model_b.p64")));
579    }
580}