Skip to main content

qualia_core_db/services/
poet_llm_jobs.rs

1//! Streaming, cooperatively cancellable local-model jobs for POET.
2
3use std::collections::BTreeMap;
4use std::convert::Infallible;
5use std::sync::{
6    atomic::{AtomicBool, AtomicU64, Ordering},
7    Arc, Mutex, OnceLock,
8};
9use std::time::{Instant, SystemTime, UNIX_EPOCH};
10
11use axum::{
12    body::{Body, Bytes},
13    extract::{Query, State},
14    http::{header, HeaderMap, StatusCode},
15    response::{IntoResponse, Response},
16    Json,
17};
18use futures_util::StreamExt;
19use serde::Deserialize;
20use tokio::sync::{broadcast, mpsc};
21use tokio_stream::wrappers::UnboundedReceiverStream;
22
23use crate::inference::inference_agent::{AgentOutput, DecodeControl};
24use crate::llm_agent::{AgentIntent, AgentRuntime, LocalLlmAgent, WebizenVerdict};
25use crate::modalities::logic::n3_compiler::N3OutputMode;
26
27use super::poet_llm_api::{decode_request, PoetLlmRequest};
28use super::poet_llm_conditioning::prepare_prompt;
29
30const MAX_JOBS: usize = 32;
31const MAX_EVENTS_PER_JOB: usize = 512;
32const JOB_EVENT_CAPACITY: usize = 256;
33static JOB_SEQUENCE: AtomicU64 = AtomicU64::new(1);
34
35#[derive(Clone)]
36struct JobEvent {
37    seq: u64,
38    frame: String,
39}
40
41struct LlmJob {
42    authority: Arc<crate::llm_agent::native_request::NativeProductSession>,
43    control: DecodeControl,
44    events: Mutex<Vec<JobEvent>>,
45    tx: broadcast::Sender<JobEvent>,
46    next_event: AtomicU64,
47    terminal: AtomicBool,
48    created_at: u64,
49}
50
51impl LlmJob {
52    fn new(authority: Arc<crate::llm_agent::native_request::NativeProductSession>) -> Self {
53        let (tx, _) = broadcast::channel(JOB_EVENT_CAPACITY);
54        Self {
55            authority,
56            control: DecodeControl::default(),
57            events: Mutex::new(Vec::new()),
58            tx,
59            next_event: AtomicU64::new(1),
60            terminal: AtomicBool::new(false),
61            created_at: unix_now(),
62        }
63    }
64
65    fn emit(&self, kind: &str, data: serde_json::Value) {
66        let seq = self.next_event.fetch_add(1, Ordering::AcqRel);
67        let frame = format!(
68            "data: {}\n\n",
69            serde_json::json!({"seq": seq, "kind": kind, "data": data})
70        );
71        let event = JobEvent { seq, frame };
72        if let Ok(mut events) = self.events.lock() {
73            if events.len() >= MAX_EVENTS_PER_JOB {
74                events.remove(0);
75            }
76            events.push(event.clone());
77        }
78        let _ = self.tx.send(event);
79    }
80}
81
82fn jobs() -> &'static Mutex<BTreeMap<String, Arc<LlmJob>>> {
83    static JOBS: OnceLock<Mutex<BTreeMap<String, Arc<LlmJob>>>> = OnceLock::new();
84    JOBS.get_or_init(|| Mutex::new(BTreeMap::new()))
85}
86
87/// `true` when `active_jobs` present.
88pub fn has_active_jobs() -> bool {
89    jobs()
90        .lock()
91        .map(|guard| {
92            guard
93                .values()
94                .any(|job| !job.terminal.load(Ordering::Acquire))
95        })
96        .unwrap_or(true)
97}
98
99/// Host session replacement/logout cooperatively stops every old worker.
100pub(super) fn cancel_all() {
101    if let Ok(guard) = jobs().lock() {
102        for job in guard.values() {
103            if !job.terminal.load(Ordering::Acquire) {
104                job.control.cancel();
105            }
106        }
107    }
108}
109
110fn diagnostic(status: StatusCode, code: &'static str, message: impl Into<String>) -> Response {
111    (
112        status,
113        Json(serde_json::json!({
114            "ok": false,
115            "honesty": "unavailable",
116            "code": code,
117            "diagnostic": message.into()
118        })),
119    )
120        .into_response()
121}
122
123/// HTTP `start` handler.
124pub async fn start_handler(
125    State(state): State<Arc<super::webizen_server::WebizenState>>,
126    headers: HeaderMap,
127    body: Bytes,
128) -> Response {
129    let authority = match super::poet_llm_authority::authenticate(&state, &headers) {
130        Ok(authority) => authority,
131        Err(response) => return response,
132    };
133    let request = match decode_request(&body) {
134        Ok(request) => request,
135        Err(response) => return response,
136    };
137    let sequence = JOB_SEQUENCE.fetch_add(1, Ordering::AcqRel);
138    let job_id = format!(
139        "llm-{:016x}",
140        crate::q_hash(&format!(
141            "{}|{}|{}|{sequence}",
142            request.agent_did,
143            request.model_path,
144            unix_now()
145        ))
146    );
147    let job = Arc::new(LlmJob::new(authority));
148    job.emit("started", serde_json::json!({"job_id": job_id}));
149    {
150        let mut guard = match jobs().lock() {
151            Ok(guard) => guard,
152            Err(_) => {
153                return diagnostic(
154                    StatusCode::INTERNAL_SERVER_ERROR,
155                    "job_registry",
156                    "Local model job registry is unavailable",
157                );
158            }
159        };
160        let now = unix_now();
161        guard.retain(|_, job| !job.terminal.load(Ordering::Acquire) || now - job.created_at < 3600);
162        if guard.len() >= MAX_JOBS {
163            return diagnostic(
164                StatusCode::TOO_MANY_REQUESTS,
165                "job_capacity",
166                "At most 32 recent or active local model jobs are retained",
167            );
168        }
169        guard.insert(job_id.clone(), Arc::clone(&job));
170    }
171    let worker_job = Arc::clone(&job);
172    let worker_job_id = job_id.clone();
173    tokio::task::spawn_blocking(move || run_job(request, worker_job, worker_job_id));
174    Json(serde_json::json!({
175        "ok": true,
176        "honesty": "live-local-stream",
177        "job_id": job_id,
178        "events_path": format!("/llm/jobs/events?job_id={job_id}")
179    }))
180    .into_response()
181}
182
183#[derive(Deserialize)]
184pub struct JobQuery {
185    job_id: String,
186}
187
188/// HTTP `events` handler.
189pub async fn events_handler(
190    State(state): State<Arc<super::webizen_server::WebizenState>>,
191    headers: HeaderMap,
192    Query(query): Query<JobQuery>,
193) -> Response {
194    let authority = match super::poet_llm_authority::authenticate(&state, &headers) {
195        Ok(authority) => authority,
196        Err(response) => return response,
197    };
198    let job = match jobs()
199        .lock()
200        .ok()
201        .and_then(|guard| guard.get(&query.job_id).cloned())
202    {
203        Some(job) => job,
204        None => {
205            return diagnostic(
206                StatusCode::NOT_FOUND,
207                "job_not_found",
208                "The local model job was not found",
209            );
210        }
211    };
212    if !Arc::ptr_eq(&job.authority, &authority) {
213        return diagnostic(
214            StatusCode::FORBIDDEN,
215            "job_authority",
216            "Job belongs to a different native session",
217        );
218    }
219    let mut subscriber = job.tx.subscribe();
220    let backlog = job
221        .events
222        .lock()
223        .map(|events| events.clone())
224        .unwrap_or_default();
225    let terminal = job.terminal.load(Ordering::Acquire);
226    let (tx, rx) = mpsc::unbounded_channel::<String>();
227    let mut last_seq = 0u64;
228    for event in backlog {
229        last_seq = last_seq.max(event.seq);
230        let _ = tx.send(event.frame);
231    }
232    if !terminal {
233        tokio::spawn(async move {
234            while let Ok(event) = subscriber.recv().await {
235                if event.seq <= last_seq {
236                    continue;
237                }
238                last_seq = event.seq;
239                let terminal = event.frame.contains("\"kind\":\"done\"")
240                    || event.frame.contains("\"kind\":\"cancelled\"")
241                    || event.frame.contains("\"kind\":\"error\"");
242                if tx.send(event.frame).is_err() || terminal {
243                    break;
244                }
245            }
246        });
247    }
248    let stream =
249        UnboundedReceiverStream::new(rx).map(|chunk| Ok::<Bytes, Infallible>(Bytes::from(chunk)));
250    Response::builder()
251        .status(StatusCode::OK)
252        .header(header::CONTENT_TYPE, "text/event-stream")
253        .header(header::CACHE_CONTROL, "no-cache")
254        .body(Body::from_stream(stream))
255        .unwrap()
256}
257
258#[derive(Deserialize)]
259pub struct CancelRequest {
260    job_id: String,
261}
262
263/// HTTP `cancel` handler.
264pub async fn cancel_handler(
265    State(state): State<Arc<super::webizen_server::WebizenState>>,
266    headers: HeaderMap,
267    body: Bytes,
268) -> Response {
269    let authority = match super::poet_llm_authority::authenticate(&state, &headers) {
270        Ok(authority) => authority,
271        Err(response) => return response,
272    };
273    let request: CancelRequest = match serde_json::from_slice(&body) {
274        Ok(request) if body.len() <= 4096 => request,
275        _ => {
276            return diagnostic(
277                StatusCode::BAD_REQUEST,
278                "invalid_cancel",
279                "Cancellation requires a bounded JSON job_id",
280            );
281        }
282    };
283    let job = match jobs()
284        .lock()
285        .ok()
286        .and_then(|guard| guard.get(&request.job_id).cloned())
287    {
288        Some(job) => job,
289        None => {
290            return diagnostic(StatusCode::NOT_FOUND, "job_not_found", "Job was not found");
291        }
292    };
293    if !Arc::ptr_eq(&job.authority, &authority) {
294        return diagnostic(
295            StatusCode::FORBIDDEN,
296            "job_authority",
297            "Job belongs to a different native session",
298        );
299    }
300    if job.terminal.load(Ordering::Acquire) {
301        return diagnostic(
302            StatusCode::CONFLICT,
303            "job_finished",
304            "Job is already finished",
305        );
306    }
307    job.control.cancel();
308    job.emit("cancelling", serde_json::json!({"job_id": request.job_id}));
309    Json(serde_json::json!({
310        "ok": true,
311        "honesty": "cancellation-requested",
312        "job_id": request.job_id
313    }))
314    .into_response()
315}
316
317fn run_job(request: PoetLlmRequest, job: Arc<LlmJob>, job_id: String) {
318    let prepared = match prepare_prompt(&request) {
319        Ok(prepared) => prepared,
320        Err(message) => {
321            persist_run_receipt(&job_id, &request, "failed", 0, 0, &message);
322            finish_error(&job, &message);
323            return;
324        }
325    };
326    job.control.set_token_budget(prepared.token_budget);
327    let agent = LocalLlmAgent::new(&request.agent_did, &request.model_path);
328    if let Err(message) = super::poet_llm_authority::bind(&agent, &job.authority) {
329        finish_error(&job, &message);
330        return;
331    }
332    let context_hash = crate::q_hash(&request.graph_context);
333    let intent = AgentIntent {
334        intent_predicate: crate::q_hash("llm:ReadGraph"),
335        requested_graph_scope: if request.graph_context.is_empty() {
336            Vec::new()
337        } else {
338            vec![context_hash]
339        },
340        context_namespaces: Vec::new(),
341        requires_network: false,
342        ilp_offer_micro_cents: 0,
343        principal_did_hash: crate::q_hash(&request.principal_did),
344        mcp_intent_frame_hash: crate::q_hash("poet:local-chat-stream"),
345        output_mode: N3OutputMode::FreeText,
346        clearance_ceiling: 0,
347        max_sentinel_depth: 32,
348        active_profile: None,
349    };
350    if !matches!(agent.validate_intent(&intent), WebizenVerdict::Permit) {
351        persist_run_receipt(
352            &job_id,
353            &request,
354            "failed",
355            0,
356            0,
357            "Webizen rejected the streaming model intent",
358        );
359        finish_error(&job, "Webizen rejected the streaming model intent");
360        return;
361    }
362    let started = Instant::now();
363    let event_job = Arc::clone(&job);
364    let (text, provenance, tokens, semantic_quin) = agent.infer_local_model_controlled(
365        &prepared.text,
366        &request.graph_context,
367        job.control.clone(),
368        Some(move |delta| event_job.emit("token", serde_json::json!({"delta": delta}))),
369    );
370    if job.control.is_cancelled() {
371        persist_run_receipt(
372            &job_id,
373            &request,
374            "cancelled",
375            tokens,
376            started.elapsed().as_millis() as u64,
377            "Cancelled by the requesting user",
378        );
379        job.emit(
380            "cancelled",
381            serde_json::json!({"tokens_generated": tokens, "partial_text": text}),
382        );
383        job.terminal.store(true, Ordering::Release);
384        return;
385    }
386    let output = AgentOutput {
387        text,
388        semantic_quin,
389        provenance_quins: provenance,
390        tokens_generated: tokens,
391        inference_duration_ms: started.elapsed().as_millis() as u64,
392        peak_memory_bytes: 0,
393    };
394    if !matches!(
395        agent.validate_output(&output),
396        WebizenVerdict::Permit | WebizenVerdict::Sanitised { .. }
397    ) {
398        persist_run_receipt(
399            &job_id,
400            &request,
401            "failed",
402            output.tokens_generated,
403            output.inference_duration_ms,
404            "Webizen rejected the streaming model output",
405        );
406        finish_error(&job, "Webizen rejected the streaming model output");
407        return;
408    }
409    let verified =
410        crate::inference::post_turn_verify::maybe_verify_turn(&prepared.text, &output.text);
411    let checks = verified
412        .checks
413        .iter()
414        .map(|check| serde_json::json!({"id": check.id, "ok": check.ok, "detail": check.detail}))
415        .collect::<Vec<_>>();
416    let receipt_persisted = persist_run_receipt(
417        &job_id,
418        &request,
419        "completed",
420        output.tokens_generated,
421        output.inference_duration_ms,
422        "",
423    );
424    job.emit(
425        "done",
426        serde_json::json!({
427            "assertion_status": "model_assertion_requires_verification",
428            "agent_did": request.agent_did,
429            "model_path": request.model_path,
430            "text": verified.final_text,
431            "draft": output.text,
432            "tokens_generated": output.tokens_generated,
433            "inference_duration_ms": output.inference_duration_ms,
434            "provenance_hashes": output.provenance_quins,
435            "context_hash": context_hash,
436            "context_supplied": !request.graph_context.is_empty(),
437            "token_budget": prepared.token_budget,
438            "conditioning": {
439                "applied": prepared.plan_id.is_some(),
440                "plan_id": prepared.plan_id,
441                "profile_id": prepared.profile_id,
442                "selected_evidence": prepared.selected_evidence,
443            },
444            "repaired": verified.repaired,
445            "checks": checks,
446            "semantic_quin": output.semantic_quin
447            ,"run_receipt_persisted": receipt_persisted
448        }),
449    );
450    job.terminal.store(true, Ordering::Release);
451}
452
453fn persist_run_receipt(
454    job_id: &str,
455    request: &PoetLlmRequest,
456    status: &str,
457    tokens: u32,
458    duration_ms: u64,
459    diagnostic: &str,
460) -> bool {
461    let fields = BTreeMap::from([
462        ("job_id".into(), serde_json::json!(job_id)),
463        ("agent_did".into(), serde_json::json!(request.agent_did)),
464        ("model_path".into(), serde_json::json!(request.model_path)),
465        ("status".into(), serde_json::json!(status)),
466        (
467            "token_budget".into(),
468            serde_json::json!(request.max_tokens.to_string()),
469        ),
470        (
471            "tokens_generated".into(),
472            serde_json::json!(tokens.to_string()),
473        ),
474        (
475            "duration_ms".into(),
476            serde_json::json!(duration_ms.to_string()),
477        ),
478        (
479            "finished_at".into(),
480            serde_json::json!(unix_now().to_string()),
481        ),
482        (
483            "diagnostic".into(),
484            serde_json::json!(diagnostic.chars().take(900).collect::<String>()),
485        ),
486    ]);
487    crate::services::poet_record_api::try_upsert(
488        "project_agent_run",
489        &format!("{status} ยท {job_id}"),
490        fields,
491    )
492    .is_ok()
493}
494
495fn finish_error(job: &LlmJob, message: &str) {
496    job.emit("error", serde_json::json!({"diagnostic": message}));
497    job.terminal.store(true, Ordering::Release);
498}
499
500fn unix_now() -> u64 {
501    SystemTime::now()
502        .duration_since(UNIX_EPOCH)
503        .map(|duration| duration.as_secs())
504        .unwrap_or(0)
505}