1use std::collections::BTreeMap;
4use std::convert::Infallible;
5use std::sync::{
6 atomic::{AtomicBool, AtomicU64, Ordering},
7 Arc, Mutex, OnceLock,
8};
9use std::time::{Instant, SystemTime, UNIX_EPOCH};
10
11use axum::{
12 body::{Body, Bytes},
13 extract::{Query, State},
14 http::{header, HeaderMap, StatusCode},
15 response::{IntoResponse, Response},
16 Json,
17};
18use futures_util::StreamExt;
19use serde::Deserialize;
20use tokio::sync::{broadcast, mpsc};
21use tokio_stream::wrappers::UnboundedReceiverStream;
22
23use crate::inference::inference_agent::{AgentOutput, DecodeControl};
24use crate::llm_agent::{AgentIntent, AgentRuntime, LocalLlmAgent, WebizenVerdict};
25use crate::modalities::logic::n3_compiler::N3OutputMode;
26
27use super::poet_llm_api::{decode_request, PoetLlmRequest};
28use super::poet_llm_conditioning::prepare_prompt;
29
30const MAX_JOBS: usize = 32;
31const MAX_EVENTS_PER_JOB: usize = 512;
32const JOB_EVENT_CAPACITY: usize = 256;
33static JOB_SEQUENCE: AtomicU64 = AtomicU64::new(1);
34
35#[derive(Clone)]
36struct JobEvent {
37 seq: u64,
38 frame: String,
39}
40
41struct LlmJob {
42 authority: Arc<crate::llm_agent::native_request::NativeProductSession>,
43 control: DecodeControl,
44 events: Mutex<Vec<JobEvent>>,
45 tx: broadcast::Sender<JobEvent>,
46 next_event: AtomicU64,
47 terminal: AtomicBool,
48 created_at: u64,
49}
50
51impl LlmJob {
52 fn new(authority: Arc<crate::llm_agent::native_request::NativeProductSession>) -> Self {
53 let (tx, _) = broadcast::channel(JOB_EVENT_CAPACITY);
54 Self {
55 authority,
56 control: DecodeControl::default(),
57 events: Mutex::new(Vec::new()),
58 tx,
59 next_event: AtomicU64::new(1),
60 terminal: AtomicBool::new(false),
61 created_at: unix_now(),
62 }
63 }
64
65 fn emit(&self, kind: &str, data: serde_json::Value) {
66 let seq = self.next_event.fetch_add(1, Ordering::AcqRel);
67 let frame = format!(
68 "data: {}\n\n",
69 serde_json::json!({"seq": seq, "kind": kind, "data": data})
70 );
71 let event = JobEvent { seq, frame };
72 if let Ok(mut events) = self.events.lock() {
73 if events.len() >= MAX_EVENTS_PER_JOB {
74 events.remove(0);
75 }
76 events.push(event.clone());
77 }
78 let _ = self.tx.send(event);
79 }
80}
81
82fn jobs() -> &'static Mutex<BTreeMap<String, Arc<LlmJob>>> {
83 static JOBS: OnceLock<Mutex<BTreeMap<String, Arc<LlmJob>>>> = OnceLock::new();
84 JOBS.get_or_init(|| Mutex::new(BTreeMap::new()))
85}
86
87pub fn has_active_jobs() -> bool {
89 jobs()
90 .lock()
91 .map(|guard| {
92 guard
93 .values()
94 .any(|job| !job.terminal.load(Ordering::Acquire))
95 })
96 .unwrap_or(true)
97}
98
99pub(super) fn cancel_all() {
101 if let Ok(guard) = jobs().lock() {
102 for job in guard.values() {
103 if !job.terminal.load(Ordering::Acquire) {
104 job.control.cancel();
105 }
106 }
107 }
108}
109
110fn diagnostic(status: StatusCode, code: &'static str, message: impl Into<String>) -> Response {
111 (
112 status,
113 Json(serde_json::json!({
114 "ok": false,
115 "honesty": "unavailable",
116 "code": code,
117 "diagnostic": message.into()
118 })),
119 )
120 .into_response()
121}
122
123pub async fn start_handler(
125 State(state): State<Arc<super::webizen_server::WebizenState>>,
126 headers: HeaderMap,
127 body: Bytes,
128) -> Response {
129 let authority = match super::poet_llm_authority::authenticate(&state, &headers) {
130 Ok(authority) => authority,
131 Err(response) => return response,
132 };
133 let request = match decode_request(&body) {
134 Ok(request) => request,
135 Err(response) => return response,
136 };
137 let sequence = JOB_SEQUENCE.fetch_add(1, Ordering::AcqRel);
138 let job_id = format!(
139 "llm-{:016x}",
140 crate::q_hash(&format!(
141 "{}|{}|{}|{sequence}",
142 request.agent_did,
143 request.model_path,
144 unix_now()
145 ))
146 );
147 let job = Arc::new(LlmJob::new(authority));
148 job.emit("started", serde_json::json!({"job_id": job_id}));
149 {
150 let mut guard = match jobs().lock() {
151 Ok(guard) => guard,
152 Err(_) => {
153 return diagnostic(
154 StatusCode::INTERNAL_SERVER_ERROR,
155 "job_registry",
156 "Local model job registry is unavailable",
157 );
158 }
159 };
160 let now = unix_now();
161 guard.retain(|_, job| !job.terminal.load(Ordering::Acquire) || now - job.created_at < 3600);
162 if guard.len() >= MAX_JOBS {
163 return diagnostic(
164 StatusCode::TOO_MANY_REQUESTS,
165 "job_capacity",
166 "At most 32 recent or active local model jobs are retained",
167 );
168 }
169 guard.insert(job_id.clone(), Arc::clone(&job));
170 }
171 let worker_job = Arc::clone(&job);
172 let worker_job_id = job_id.clone();
173 tokio::task::spawn_blocking(move || run_job(request, worker_job, worker_job_id));
174 Json(serde_json::json!({
175 "ok": true,
176 "honesty": "live-local-stream",
177 "job_id": job_id,
178 "events_path": format!("/llm/jobs/events?job_id={job_id}")
179 }))
180 .into_response()
181}
182
183#[derive(Deserialize)]
184pub struct JobQuery {
185 job_id: String,
186}
187
188pub async fn events_handler(
190 State(state): State<Arc<super::webizen_server::WebizenState>>,
191 headers: HeaderMap,
192 Query(query): Query<JobQuery>,
193) -> Response {
194 let authority = match super::poet_llm_authority::authenticate(&state, &headers) {
195 Ok(authority) => authority,
196 Err(response) => return response,
197 };
198 let job = match jobs()
199 .lock()
200 .ok()
201 .and_then(|guard| guard.get(&query.job_id).cloned())
202 {
203 Some(job) => job,
204 None => {
205 return diagnostic(
206 StatusCode::NOT_FOUND,
207 "job_not_found",
208 "The local model job was not found",
209 );
210 }
211 };
212 if !Arc::ptr_eq(&job.authority, &authority) {
213 return diagnostic(
214 StatusCode::FORBIDDEN,
215 "job_authority",
216 "Job belongs to a different native session",
217 );
218 }
219 let mut subscriber = job.tx.subscribe();
220 let backlog = job
221 .events
222 .lock()
223 .map(|events| events.clone())
224 .unwrap_or_default();
225 let terminal = job.terminal.load(Ordering::Acquire);
226 let (tx, rx) = mpsc::unbounded_channel::<String>();
227 let mut last_seq = 0u64;
228 for event in backlog {
229 last_seq = last_seq.max(event.seq);
230 let _ = tx.send(event.frame);
231 }
232 if !terminal {
233 tokio::spawn(async move {
234 while let Ok(event) = subscriber.recv().await {
235 if event.seq <= last_seq {
236 continue;
237 }
238 last_seq = event.seq;
239 let terminal = event.frame.contains("\"kind\":\"done\"")
240 || event.frame.contains("\"kind\":\"cancelled\"")
241 || event.frame.contains("\"kind\":\"error\"");
242 if tx.send(event.frame).is_err() || terminal {
243 break;
244 }
245 }
246 });
247 }
248 let stream =
249 UnboundedReceiverStream::new(rx).map(|chunk| Ok::<Bytes, Infallible>(Bytes::from(chunk)));
250 Response::builder()
251 .status(StatusCode::OK)
252 .header(header::CONTENT_TYPE, "text/event-stream")
253 .header(header::CACHE_CONTROL, "no-cache")
254 .body(Body::from_stream(stream))
255 .unwrap()
256}
257
258#[derive(Deserialize)]
259pub struct CancelRequest {
260 job_id: String,
261}
262
263pub async fn cancel_handler(
265 State(state): State<Arc<super::webizen_server::WebizenState>>,
266 headers: HeaderMap,
267 body: Bytes,
268) -> Response {
269 let authority = match super::poet_llm_authority::authenticate(&state, &headers) {
270 Ok(authority) => authority,
271 Err(response) => return response,
272 };
273 let request: CancelRequest = match serde_json::from_slice(&body) {
274 Ok(request) if body.len() <= 4096 => request,
275 _ => {
276 return diagnostic(
277 StatusCode::BAD_REQUEST,
278 "invalid_cancel",
279 "Cancellation requires a bounded JSON job_id",
280 );
281 }
282 };
283 let job = match jobs()
284 .lock()
285 .ok()
286 .and_then(|guard| guard.get(&request.job_id).cloned())
287 {
288 Some(job) => job,
289 None => {
290 return diagnostic(StatusCode::NOT_FOUND, "job_not_found", "Job was not found");
291 }
292 };
293 if !Arc::ptr_eq(&job.authority, &authority) {
294 return diagnostic(
295 StatusCode::FORBIDDEN,
296 "job_authority",
297 "Job belongs to a different native session",
298 );
299 }
300 if job.terminal.load(Ordering::Acquire) {
301 return diagnostic(
302 StatusCode::CONFLICT,
303 "job_finished",
304 "Job is already finished",
305 );
306 }
307 job.control.cancel();
308 job.emit("cancelling", serde_json::json!({"job_id": request.job_id}));
309 Json(serde_json::json!({
310 "ok": true,
311 "honesty": "cancellation-requested",
312 "job_id": request.job_id
313 }))
314 .into_response()
315}
316
317fn run_job(request: PoetLlmRequest, job: Arc<LlmJob>, job_id: String) {
318 let prepared = match prepare_prompt(&request) {
319 Ok(prepared) => prepared,
320 Err(message) => {
321 persist_run_receipt(&job_id, &request, "failed", 0, 0, &message);
322 finish_error(&job, &message);
323 return;
324 }
325 };
326 job.control.set_token_budget(prepared.token_budget);
327 let agent = LocalLlmAgent::new(&request.agent_did, &request.model_path);
328 if let Err(message) = super::poet_llm_authority::bind(&agent, &job.authority) {
329 finish_error(&job, &message);
330 return;
331 }
332 let context_hash = crate::q_hash(&request.graph_context);
333 let intent = AgentIntent {
334 intent_predicate: crate::q_hash("llm:ReadGraph"),
335 requested_graph_scope: if request.graph_context.is_empty() {
336 Vec::new()
337 } else {
338 vec![context_hash]
339 },
340 context_namespaces: Vec::new(),
341 requires_network: false,
342 ilp_offer_micro_cents: 0,
343 principal_did_hash: crate::q_hash(&request.principal_did),
344 mcp_intent_frame_hash: crate::q_hash("poet:local-chat-stream"),
345 output_mode: N3OutputMode::FreeText,
346 clearance_ceiling: 0,
347 max_sentinel_depth: 32,
348 active_profile: None,
349 };
350 if !matches!(agent.validate_intent(&intent), WebizenVerdict::Permit) {
351 persist_run_receipt(
352 &job_id,
353 &request,
354 "failed",
355 0,
356 0,
357 "Webizen rejected the streaming model intent",
358 );
359 finish_error(&job, "Webizen rejected the streaming model intent");
360 return;
361 }
362 let started = Instant::now();
363 let event_job = Arc::clone(&job);
364 let (text, provenance, tokens, semantic_quin) = agent.infer_local_model_controlled(
365 &prepared.text,
366 &request.graph_context,
367 job.control.clone(),
368 Some(move |delta| event_job.emit("token", serde_json::json!({"delta": delta}))),
369 );
370 if job.control.is_cancelled() {
371 persist_run_receipt(
372 &job_id,
373 &request,
374 "cancelled",
375 tokens,
376 started.elapsed().as_millis() as u64,
377 "Cancelled by the requesting user",
378 );
379 job.emit(
380 "cancelled",
381 serde_json::json!({"tokens_generated": tokens, "partial_text": text}),
382 );
383 job.terminal.store(true, Ordering::Release);
384 return;
385 }
386 let output = AgentOutput {
387 text,
388 semantic_quin,
389 provenance_quins: provenance,
390 tokens_generated: tokens,
391 inference_duration_ms: started.elapsed().as_millis() as u64,
392 peak_memory_bytes: 0,
393 };
394 if !matches!(
395 agent.validate_output(&output),
396 WebizenVerdict::Permit | WebizenVerdict::Sanitised { .. }
397 ) {
398 persist_run_receipt(
399 &job_id,
400 &request,
401 "failed",
402 output.tokens_generated,
403 output.inference_duration_ms,
404 "Webizen rejected the streaming model output",
405 );
406 finish_error(&job, "Webizen rejected the streaming model output");
407 return;
408 }
409 let verified =
410 crate::inference::post_turn_verify::maybe_verify_turn(&prepared.text, &output.text);
411 let checks = verified
412 .checks
413 .iter()
414 .map(|check| serde_json::json!({"id": check.id, "ok": check.ok, "detail": check.detail}))
415 .collect::<Vec<_>>();
416 let receipt_persisted = persist_run_receipt(
417 &job_id,
418 &request,
419 "completed",
420 output.tokens_generated,
421 output.inference_duration_ms,
422 "",
423 );
424 job.emit(
425 "done",
426 serde_json::json!({
427 "assertion_status": "model_assertion_requires_verification",
428 "agent_did": request.agent_did,
429 "model_path": request.model_path,
430 "text": verified.final_text,
431 "draft": output.text,
432 "tokens_generated": output.tokens_generated,
433 "inference_duration_ms": output.inference_duration_ms,
434 "provenance_hashes": output.provenance_quins,
435 "context_hash": context_hash,
436 "context_supplied": !request.graph_context.is_empty(),
437 "token_budget": prepared.token_budget,
438 "conditioning": {
439 "applied": prepared.plan_id.is_some(),
440 "plan_id": prepared.plan_id,
441 "profile_id": prepared.profile_id,
442 "selected_evidence": prepared.selected_evidence,
443 },
444 "repaired": verified.repaired,
445 "checks": checks,
446 "semantic_quin": output.semantic_quin
447 ,"run_receipt_persisted": receipt_persisted
448 }),
449 );
450 job.terminal.store(true, Ordering::Release);
451}
452
453fn persist_run_receipt(
454 job_id: &str,
455 request: &PoetLlmRequest,
456 status: &str,
457 tokens: u32,
458 duration_ms: u64,
459 diagnostic: &str,
460) -> bool {
461 let fields = BTreeMap::from([
462 ("job_id".into(), serde_json::json!(job_id)),
463 ("agent_did".into(), serde_json::json!(request.agent_did)),
464 ("model_path".into(), serde_json::json!(request.model_path)),
465 ("status".into(), serde_json::json!(status)),
466 (
467 "token_budget".into(),
468 serde_json::json!(request.max_tokens.to_string()),
469 ),
470 (
471 "tokens_generated".into(),
472 serde_json::json!(tokens.to_string()),
473 ),
474 (
475 "duration_ms".into(),
476 serde_json::json!(duration_ms.to_string()),
477 ),
478 (
479 "finished_at".into(),
480 serde_json::json!(unix_now().to_string()),
481 ),
482 (
483 "diagnostic".into(),
484 serde_json::json!(diagnostic.chars().take(900).collect::<String>()),
485 ),
486 ]);
487 crate::services::poet_record_api::try_upsert(
488 "project_agent_run",
489 &format!("{status} ยท {job_id}"),
490 fields,
491 )
492 .is_ok()
493}
494
495fn finish_error(job: &LlmJob, message: &str) {
496 job.emit("error", serde_json::json!({"diagnostic": message}));
497 job.terminal.store(true, Ordering::Release);
498}
499
500fn unix_now() -> u64 {
501 SystemTime::now()
502 .duration_since(UNIX_EPOCH)
503 .map(|duration| duration.as_secs())
504 .unwrap_or(0)
505}