qualia_core_db/inference/reranker/
mod.rs1use std::collections::HashMap;
16
17#[derive(Debug, Clone, PartialEq)]
19pub struct RerankResult {
20 pub index: usize,
22 pub score: f64,
24}
25
26#[derive(Debug, Clone)]
28pub struct RerankConfig {
29 pub overlap_weight: f64,
31 pub bm25_weight: f64,
33 pub phrase_weight: f64,
35 pub position_weight: f64,
37 pub bm25_k1: f64,
39 pub bm25_b: f64,
41}
42
43impl Default for RerankConfig {
44 fn default() -> Self {
45 Self {
46 overlap_weight: 0.3,
47 bm25_weight: 0.5,
48 phrase_weight: 0.15,
49 position_weight: 0.05,
50 bm25_k1: 1.2,
51 bm25_b: 0.75,
52 }
53 }
54}
55
56fn tokenise(s: &str) -> Vec<String> {
58 s.split(|c: char| !c.is_alphanumeric())
59 .filter(|t| !t.is_empty())
60 .map(|t| t.to_ascii_lowercase())
61 .collect()
62}
63
64fn term_freqs(tokens: &[String]) -> HashMap<String, f64> {
66 let mut freqs = HashMap::new();
67 let n = tokens.len() as f64;
68 for t in tokens {
69 *freqs.entry(t.clone()).or_insert(0.0) += 1.0;
70 }
71 for v in freqs.values_mut() {
73 *v /= n.max(1.0);
74 }
75 freqs
76}
77
78fn jaccard(a: &[String], b: &[String]) -> f64 {
80 let set_a: std::collections::BTreeSet<&String> = a.iter().collect();
81 let set_b: std::collections::BTreeSet<&String> = b.iter().collect();
82 let intersection = set_a.intersection(&set_b).count();
83 let union = set_a.union(&set_b).count();
84 if union == 0 {
85 0.0
86 } else {
87 intersection as f64 / union as f64
88 }
89}
90
91fn bm25_score(
93 query_terms: &[String],
94 doc_tokens: &[String],
95 doc_freqs: &HashMap<String, f64>,
96 avg_doc_len: f64,
97 config: &RerankConfig,
98) -> f64 {
99 let doc_len = doc_tokens.len() as f64;
100 let mut score = 0.0;
101 let n_docs = 1.0; for term in query_terms {
104 let tf = doc_freqs.get(term).copied().unwrap_or(0.0);
105 if tf == 0.0 {
106 continue;
107 }
108 let idf = 1.0 + (n_docs + 0.5) / (tf + 0.5);
112 let tf_component = (tf * (config.bm25_k1 + 1.0))
113 / (tf
114 + config.bm25_k1
115 * (1.0 - config.bm25_b + config.bm25_b * (doc_len / avg_doc_len.max(1.0))));
116 score += idf * tf_component;
117 }
118 score
119}
120
121fn phrase_bonus(query: &str, doc: &str) -> f64 {
123 let query_lower = query.to_ascii_lowercase();
124 let doc_lower = doc.to_ascii_lowercase();
125 let query_words: Vec<&str> = query_lower.split_whitespace().collect();
126 if query_words.len() < 2 {
127 return 0.0;
128 }
129 let mut bonus = 0.0;
131 for n in 2..=3.min(query_words.len()) {
132 for window in query_words.windows(n) {
133 let phrase = window.join(" ");
134 if doc_lower.contains(&phrase) {
135 bonus += 0.1 * n as f64;
136 }
137 }
138 }
139 bonus.min(1.0)
140}
141
142fn position_bonus(query_terms: &[String], doc_tokens: &[String]) -> f64 {
144 if doc_tokens.is_empty() {
145 return 0.0;
146 }
147 let doc_len = doc_tokens.len();
148 let mut earliest_match = doc_len;
149 for (i, token) in doc_tokens.iter().enumerate() {
150 if query_terms.contains(token) {
151 earliest_match = earliest_match.min(i);
152 }
153 }
154 if earliest_match == doc_len {
155 0.0
156 } else {
157 1.0 - (earliest_match as f64 / doc_len as f64)
159 }
160}
161
162pub fn rerank(query: &str, candidates: &[String], config: &RerankConfig) -> Vec<RerankResult> {
166 let query_tokens = tokenise(query);
167 if query_tokens.is_empty() || candidates.is_empty() {
168 return candidates
169 .iter()
170 .enumerate()
171 .map(|(i, _)| RerankResult {
172 index: i,
173 score: 0.0,
174 })
175 .collect();
176 }
177
178 let candidate_tokens: Vec<Vec<String>> = candidates.iter().map(|c| tokenise(c)).collect();
179 let avg_doc_len =
180 candidate_tokens.iter().map(|t| t.len() as f64).sum::<f64>() / candidates.len() as f64;
181
182 let mut results = Vec::with_capacity(candidates.len());
183 for (i, (candidate, doc_tokens)) in candidates.iter().zip(candidate_tokens.iter()).enumerate() {
184 let overlap = jaccard(&query_tokens, doc_tokens);
185 let doc_freqs = term_freqs(doc_tokens);
186 let bm25 = bm25_score(&query_tokens, doc_tokens, &doc_freqs, avg_doc_len, config);
187 let phrase = phrase_bonus(query, candidate);
188 let position = position_bonus(&query_tokens, doc_tokens);
189
190 let score = config.overlap_weight * overlap
191 + config.bm25_weight * (bm25 / (bm25 + 1.0)) + config.phrase_weight * phrase
193 + config.position_weight * position;
194
195 results.push(RerankResult { index: i, score });
196 }
197
198 results.sort_by(|a, b| {
200 b.score
201 .partial_cmp(&a.score)
202 .unwrap_or(std::cmp::Ordering::Equal)
203 });
204 results
205}
206
207pub fn rerank_default(query: &str, candidates: &[String]) -> Vec<RerankResult> {
209 rerank(query, candidates, &RerankConfig::default())
210}
211
212#[cfg(test)]
213mod tests {
214 use super::*;
215
216 #[test]
217 fn rerank_basic_ordering() {
218 let query = "machine learning algorithms";
219 let candidates = vec![
220 "The weather is nice today".to_string(),
221 "Machine learning algorithms are powerful tools".to_string(),
222 "I like pizza".to_string(),
223 ];
224 let results = rerank_default(query, &candidates);
225 assert_eq!(results.len(), 3);
226 assert_eq!(results[0].index, 1);
228 assert!(results[0].score > results[1].score);
229 }
230
231 #[test]
232 fn rerank_empty_query() {
233 let candidates = vec!["doc one".to_string(), "doc two".to_string()];
234 let results = rerank_default("", &candidates);
235 assert_eq!(results.len(), 2);
236 assert!(results.iter().all(|r| r.score == 0.0));
238 }
239
240 #[test]
241 fn rerank_empty_candidates() {
242 let results = rerank_default("query", &[]);
243 assert!(results.is_empty());
244 }
245
246 #[test]
247 fn rerank_identical_candidates() {
248 let query = "hello world";
249 let candidates = vec!["hello world".to_string(), "hello world".to_string()];
250 let results = rerank_default(query, &candidates);
251 assert_eq!(results.len(), 2);
252 assert!((results[0].score - results[1].score).abs() < 1e-9);
254 }
255
256 #[test]
257 fn rerank_phrase_bonus() {
258 let query = "machine learning";
259 let candidates = vec![
260 "machine and learning separately".to_string(),
261 "machine learning together".to_string(),
262 ];
263 let results = rerank_default(query, &candidates);
264 assert_eq!(results[0].index, 1);
266 }
267
268 #[test]
269 fn rerank_position_bonus() {
270 let query = "important";
271 let candidates = vec![
272 "not relevant text here important at end".to_string(),
273 "important at the start of the document".to_string(),
274 ];
275 let results = rerank_default(query, &candidates);
276 assert_eq!(results[0].index, 1);
278 }
279
280 #[test]
281 fn rerank_custom_config() {
282 let query = "test query";
283 let candidates = vec!["test document".to_string()];
284 let config = RerankConfig {
285 overlap_weight: 1.0,
286 bm25_weight: 0.0,
287 phrase_weight: 0.0,
288 position_weight: 0.0,
289 ..Default::default()
290 };
291 let results = rerank(query, &candidates, &config);
292 assert_eq!(results.len(), 1);
293 assert!(results[0].score > 0.0);
295 }
296
297 #[test]
298 fn rerank_scores_in_descending_order() {
299 let query = "alpha beta gamma";
300 let candidates = vec![
301 "alpha".to_string(),
302 "alpha beta gamma delta".to_string(),
303 "alpha beta".to_string(),
304 ];
305 let results = rerank_default(query, &candidates);
306 for i in 1..results.len() {
307 assert!(
308 results[i - 1].score >= results[i].score,
309 "results not in descending order at index {i}"
310 );
311 }
312 }
313
314 #[test]
315 fn tokenise_handles_punctuation() {
316 let tokens = tokenise("Hello, world! Foo-Bar");
317 assert_eq!(tokens, vec!["hello", "world", "foo", "bar"]);
318 }
319
320 #[test]
321 fn jaccard_identical() {
322 let a = tokenise("hello world");
323 let b = tokenise("hello world");
324 assert_eq!(jaccard(&a, &b), 1.0);
325 }
326
327 #[test]
328 fn jaccard_disjoint() {
329 let a = tokenise("hello world");
330 let b = tokenise("foo bar");
331 assert_eq!(jaccard(&a, &b), 0.0);
332 }
333
334 #[test]
335 fn phrase_bonus_multiword() {
336 let bonus = phrase_bonus("machine learning models", "machine learning is great");
337 assert!(bonus > 0.0);
338 }
339
340 #[test]
341 fn phrase_bonus_single_word() {
342 let bonus = phrase_bonus("hello", "hello world");
343 assert_eq!(bonus, 0.0);
344 }
345
346 #[test]
347 fn position_bonus_early_match() {
348 let query = tokenise("important");
349 let doc = tokenise("important text follows here");
350 let bonus = position_bonus(&query, &doc);
351 assert!(bonus > 0.9); }
353
354 #[test]
355 fn position_bonus_no_match() {
356 let query = tokenise("missing");
357 let doc = tokenise("some text here");
358 let bonus = position_bonus(&query, &doc);
359 assert_eq!(bonus, 0.0);
360 }
361}