qualia_core_db/inference/inference_agent/
config.rs

1// ─── Constants ──────────────────────────────────────────────────────────────
2/// Hard memory ceiling for the LLM runtime within the 512MB system floor.
3/// Leaves the remaining 384MB for the Webizen VM, SLG Arena, and WASM stack.
4pub const LLM_MEMORY_BUDGET_BYTES: u64 = 128 * 1024 * 1024; // 128 MB
5
6/// Maximum tokens the agent may generate in a single turn. Enforces deterministic
7/// compute cost — no runaway generation that blocks the edge device.
8pub const MAX_OUTPUT_TOKENS: u32 = 2048;
9
10/// Token budget for the autoregressive loop (`MAX_OUTPUT_TOKENS` in release).
11// NOTE: `pub(super)` widening (was a private module-level const) so the decode
12// path in `decode.rs` can read it across the new submodule boundary.
13#[cfg(test)]
14pub(super) const DECODE_TOKEN_BUDGET: u32 = 16;
15/// MC2b harness iteration: CPU SDPA decode is very slow in wasm; trim budget until Option B.
16#[cfg(all(not(test), target_arch = "wasm32"))]
17pub(super) const DECODE_TOKEN_BUDGET: u32 = 32;
18// Codex P0: default per-turn decode cap. Was MAX_OUTPUT_TOKENS (2048) → at ~3 tok/s a no-EOS reply
19// ran ~11 min and the app looked frozen. 256 keeps a turn bounded; MAX_OUTPUT_TOKENS stays the
20// absolute ceiling and the cooperative deadline (INFERENCE_TIMEOUT_MS, checked INSIDE the decode
21// loop) bounds wall-clock time independently.
22#[cfg(all(not(test), not(target_arch = "wasm32")))]
23pub(super) const DECODE_TOKEN_BUDGET: u32 = 256;
24
25/// Layer cap for transformer forward during unit tests (full depth in release).
26// NOTE: `pub(super)` widening — read by `decode.rs` and `decode_helpers.rs`.
27#[cfg(test)]
28pub(super) const TEST_TRANSFORMER_LAYER_CAP: u32 = 2;
29#[cfg(not(test))]
30pub(super) const TEST_TRANSFORMER_LAYER_CAP: u32 = 0;
31
32/// Vocab chunk cap during unit tests (full sweep in release).
33// NOTE: `pub(super)` widening — read by `decode.rs` and `decode_helpers.rs`.
34#[cfg(test)]
35pub(super) const TEST_VOCAB_CHUNK_CAP: u32 = 4;
36#[cfg(not(test))]
37pub(super) const TEST_VOCAB_CHUNK_CAP: u32 = 0;
38
39/// Default maximum milliseconds for a local inference call (interactive).
40/// Batch/overnight profile raises this via `llm_bench::inference_timeout_ms()`.
41pub const INFERENCE_TIMEOUT_MS: u64 = 30_000;
42
43/// Effective timeout: batch profile / env may extend (e.g. 8h overnight jobs).
44// NOTE: `pub(super)` widening — read by `decode.rs` and `runtime.rs`.
45#[inline]
46pub(super) fn effective_inference_timeout_ms() -> u64 {
47    #[cfg(not(target_arch = "wasm32"))]
48    {
49        return crate::llm_bench::inference_timeout_ms();
50    }
51    #[cfg(target_arch = "wasm32")]
52    {
53        INFERENCE_TIMEOUT_MS
54    }
55}
qualia_core_db/inference/inference_agent/config.rs

qualia_core_db/inference/inference_agent/
config.rs
