qualia_cli/handlers/inference_policy/
eval.rs1use super::{io, profile};
3use qualia_client_core::{api::inference_policy as api, inference_policy::InferenceActorSession};
4use qualia_core_db::{
5 key_vault::KeyVault,
6 llm_agent::native_request::{NativeInputBudget, NativePreparationSpec},
7 inference::speculative_decode::DominoPreparationBudget,
8};
9use serde::Deserialize;
10use std::path::{Path, PathBuf};
11
12#[derive(Deserialize)]
13#[serde(deny_unknown_fields)]
14struct EvalAuthority {
15 profile: PathBuf,
16 subject: String,
17 identifier: qualia_client_core::inference_policy::SelectedInferenceIdentifier,
18 signer_label: String,
19 lifetime_seconds: u64,
20 model: String,
21 model_vault: PathBuf,
22 preparation: Preparation,
23 retained_input_bytes: usize,
24 tokenizer_scratch_bytes: usize,
25 domino_owned_bytes: usize,
26 domino_text_bytes: usize,
27 domino_stack_depth: usize,
28}
29
30#[derive(Deserialize)]
31#[serde(tag = "kind", deny_unknown_fields)]
32enum Preparation {
33 Leased {
34 primary: LeaseFile,
35 shards: Vec<LeaseFile>,
36 source_byte_budget: u64,
37 mapped_byte_budget: u64,
38 total_byte_budget: u64,
39 metadata_budget: usize,
40 },
41 Snapshot { byte_budget: usize },
42}
43#[derive(Deserialize)]
44#[serde(deny_unknown_fields)]
45struct LeaseFile { path: PathBuf, expected_sha256: [u8; 32] }
46
47impl Preparation {
48 fn open(self, inputs: NativeInputBudget) -> Result<NativePreparationSpec, String> {
49 match self {
50 Self::Snapshot { byte_budget } => Ok(NativePreparationSpec::Snapshot { byte_budget, inputs }),
51 Self::Leased { primary, shards, source_byte_budget, mapped_byte_budget,
52 total_byte_budget, metadata_budget } => {
53 if shards.len() > 64 { return Err("At most 64 ordered model shards are admitted".into()); }
54 let cancelled = std::sync::atomic::AtomicBool::new(false);
55 let mut remaining = [source_byte_budget, mapped_byte_budget, total_byte_budget];
56 let mut open = |file: LeaseFile| -> Result<_, String> {
57 let lease = NativePreparationSpec::open_model_lease(
58 &file.path, file.expected_sha256, remaining, &cancelled)?;
59 let length = lease.identity().byte_len;
60 let total = length.checked_mul(2).ok_or("Model lease accounting overflow")?;
61 for (budget, charge) in remaining.iter_mut().zip([length, length, total]) {
62 *budget = budget.checked_sub(charge).ok_or("Aggregate model lease budget exhausted")?;
63 }
64 Ok(lease)
65 };
66 let primary_path = std::fs::canonicalize(&primary.path)
67 .map_err(|error| format!("Could not resolve primary model: {error}"))?;
68 let primary_digest = primary.expected_sha256;
69 let primary = open(primary)?;
70 let mut retained = Vec::with_capacity(shards.len());
73 for (index, file) in shards.into_iter().enumerate() {
74 if index == 0 {
75 let shard_path = std::fs::canonicalize(&file.path)
76 .map_err(|error| format!("Could not resolve first model shard: {error}"))?;
77 if shard_path != primary_path || file.expected_sha256 != primary_digest {
78 return Err("The first ordered shard must be the primary model file with the same expected digest".into());
79 }
80 retained.push(std::sync::Arc::clone(&primary));
83 } else {
84 retained.push(open(file)?);
85 }
86 }
87 Ok(NativePreparationSpec::Leased { primary, shards: retained, metadata_budget, inputs })
88 }
89 }
90 }
91}
92
93pub(crate) struct EvalLogin(pub(super) InferenceActorSession);
95impl Drop for EvalLogin {
96 fn drop(&mut self) {
97 let _ = self.0.logout();
98 qualia_client_core::inference_policy::disconnect_inference_controller();
99 }
100}
101
102pub(crate) fn prepare_eval(path: &Path) -> Result<EvalLogin, String> {
103 let input: EvalAuthority = io::json(path, 64 * 1024)?;
104 let root = profile::bind_child(&input.profile, false)?;
105 let vault = KeyVault::open_native_secure(root.to_str().ok_or("Profile is not UTF-8")?, false)?;
106 qualia_client_core::state::init_inference_host_state(vault)?;
107 let actor = EvalLogin(api::login_inference_actor(
110 input.subject, input.identifier, input.signer_label, input.lifetime_seconds,
111 )?);
112 let inputs = NativeInputBudget {
113 retained_bytes: input.retained_input_bytes,
114 tokenizer_scratch_bytes: input.tokenizer_scratch_bytes,
115 domino: DominoPreparationBudget {
116 owned_bytes: input.domino_owned_bytes,
117 text_bytes: input.domino_text_bytes,
118 stack_depth: input.domino_stack_depth,
119 },
120 };
121 let session = api::get_native_inference_session(&actor.0, input.preparation.open(inputs)?)?;
122 crate::llm_lifecycle::run_load(&input.model_vault, &input.model)?;
123 crate::llm_lifecycle::bind_session_authority(session)?;
124 Ok(actor)
125}