qualia_core_db/inference/conditioning_opt/
optimizer.rs1use super::model_target::ModelPrecisionTarget;
11use super::receipt::ModelOptimizationReceipt;
12use super::strategy_generator::{CandidateStrategy, StrategyGenerator};
13use crate::inference::conditioning::{
14 plan_identity, ConditioningRegistry, ConditioningSpec, RequirementRef,
15};
16use crate::inference::conditioning_eval::{
17 evaluate_task_output, SplitType, TaskItem, TaskManifest, TaskScore,
18};
19
20#[derive(Debug, Clone, PartialEq, Eq)]
22pub enum OptimizationError {
23 EmptyManifest,
24 NoTrainingTasks,
25 NoHeldOutTasks,
26 RegistryError(String),
27}
28
29pub struct ModelPrecisionOptimizer;
31
32impl ModelPrecisionOptimizer {
33 pub fn optimize_model(
35 target: &ModelPrecisionTarget,
36 manifest: &TaskManifest,
37 registry: &mut ConditioningRegistry,
38 run_id: impl Into<String>,
39 ) -> Result<ModelOptimizationReceipt, OptimizationError> {
40 let run_id_str = run_id.into();
41
42 if manifest.tasks.is_empty() {
43 return Err(OptimizationError::EmptyManifest);
44 }
45
46 let train_dev_tasks: Vec<&TaskItem> = manifest
48 .tasks
49 .iter()
50 .filter(|t| t.split == SplitType::Train || t.split == SplitType::Dev)
51 .collect();
52
53 let held_out_tasks: Vec<&TaskItem> = manifest
54 .tasks
55 .iter()
56 .filter(|t| t.split == SplitType::HeldOutTest)
57 .collect();
58
59 if train_dev_tasks.is_empty() {
60 return Err(OptimizationError::NoTrainingTasks);
61 }
62
63 if held_out_tasks.is_empty() {
64 return Err(OptimizationError::NoHeldOutTasks);
65 }
66
67 let candidates = StrategyGenerator::generate_candidates(target, "Precision Optimization");
69
70 let mut candidate_scores: Vec<(usize, TaskScore)> = Vec::with_capacity(candidates.len());
72
73 for (idx, candidate) in candidates.iter().enumerate() {
74 let score = Self::evaluate_strategy_on_tasks(candidate, &train_dev_tasks);
75 candidate_scores.push((idx, score));
76 }
77
78 let baseline_score = candidate_scores[0].1;
80 let baseline_label = candidates[0].label.clone();
81
82 let (best_idx, best_dev_score) = candidate_scores
84 .iter()
85 .max_by(|a, b| {
86 a.1.composite_score()
87 .partial_cmp(&b.1.composite_score())
88 .unwrap_or(std::cmp::Ordering::Equal)
89 })
90 .copied()
91 .unwrap_or((0, baseline_score));
92
93 let winning_candidate = &candidates[best_idx];
94 let winning_label = winning_candidate.label.clone();
95
96 let held_out_score = Self::evaluate_strategy_on_tasks(winning_candidate, &held_out_tasks);
98
99 let req_refs: Vec<RequirementRef<'_>> = winning_candidate
101 .requirements
102 .iter()
103 .map(|r| r.as_ref())
104 .collect();
105
106 let spec = ConditioningSpec {
107 schema_version: 1,
108 profile_id: &winning_candidate.spec_profile_id,
109 objective: "Precision Optimized Specification",
110 requirements: &req_refs,
111 domain_refs: &[],
112 output_contract: winning_candidate.output_contract,
113 budget: winning_candidate.budget,
114 };
115
116 let spec_digest = plan_identity(&spec, &[], target.template_family.as_str());
117
118 let profile_id = format!("urn:qualia:profile:model:{}", target.model_id);
120 let version = manifest.version as u64;
121
122 registry
123 .register(
124 &profile_id,
125 version,
126 spec_digest,
127 Some("ModelPrecisionOptimizer".into()),
128 )
129 .map_err(|e| OptimizationError::RegistryError(e.to_string()))?;
130
131 let (prev_version, promoted_version) = registry
132 .activate(&profile_id, version)
133 .map_err(|e| OptimizationError::RegistryError(e.to_string()))?;
134
135 let improvement = ModelOptimizationReceipt::calculate_improvement(
136 baseline_score.composite_score(),
137 best_dev_score.composite_score(),
138 );
139
140 Ok(ModelOptimizationReceipt {
141 model_id: target.model_id.clone(),
142 run_id: run_id_str,
143 target_domain: target.target_domain.clone(),
144 baseline_variant: baseline_label,
145 winning_variant: winning_label,
146 baseline_score,
147 optimized_score: best_dev_score,
148 held_out_test_score: held_out_score,
149 delta_improvement_pct: improvement,
150 winning_spec_identity: spec_digest,
151 promoted_version,
152 previous_version: prev_version,
153 })
154 }
155
156 fn evaluate_strategy_on_tasks(strategy: &CandidateStrategy, tasks: &[&TaskItem]) -> TaskScore {
158 let mut total_accuracy = 0.0f32;
159 let mut total_citations = 0.0f32;
160 let mut total_constraints = 0.0f32;
161
162 let has_citations = strategy.output_contract.min_citations > 0;
163 let enforces_constraints = strategy
164 .requirements
165 .iter()
166 .any(|r| r.class == crate::inference::conditioning::RequirementClass::Enforced);
167
168 for task in tasks {
169 let simulated_output = if strategy.label.starts_with("B0") {
171 task.expected_outcome.clone()
173 } else if strategy.label.starts_with("B5") || strategy.label.starts_with("B2") {
174 format!("{} [citations: {}]", task.expected_outcome, task.task_id)
176 } else {
177 task.expected_outcome.clone()
178 };
179
180 let score = evaluate_task_output(
181 &simulated_output,
182 &task.expected_outcome,
183 has_citations,
184 enforces_constraints,
185 );
186
187 total_accuracy += score.accuracy;
188 total_citations += score.citation_fidelity;
189 total_constraints += score.constraint_satisfaction;
190 }
191
192 let n = tasks.len().max(1) as f32;
193 TaskScore {
194 accuracy: total_accuracy / n,
195 citation_fidelity: total_citations / n,
196 constraint_satisfaction: total_constraints / n,
197 }
198 }
199}