Skip to main content

qualia_core_db/inference/reranker/
mod.rs

1//! Cross-encoder reranker for candidate re-scoring.
2//!
3//! A reranker takes a query and a list of candidate documents and re-scores
4//! them based on query-document relevance. This implementation uses a
5//! deterministic symbolic scoring approach (no neural network required):
6//!
7//! 1. **Token overlap**: Jaccard similarity between query and document tokens.
8//! 2. **BM25-style scoring**: Term frequency / inverse document frequency.
9//! 3. **Phrase bonus**: Bonus for multi-word query phrases appearing in the document.
10//! 4. **Position bonus**: Bonus for matches appearing early in the document.
11//!
12//! The final score is a weighted combination of these signals. Candidates
13//! are returned sorted by descending score.
14
15use std::collections::HashMap;
16
17/// A single reranked result.
18#[derive(Debug, Clone, PartialEq)]
19pub struct RerankResult {
20    /// Original index of the candidate in the input list.
21    pub index: usize,
22    /// Relevance score (higher = more relevant).
23    pub score: f64,
24}
25
26/// Configuration for the reranker.
27#[derive(Debug, Clone)]
28pub struct RerankConfig {
29    /// Weight for token overlap (Jaccard) signal. Default: 0.3.
30    pub overlap_weight: f64,
31    /// Weight for BM25 signal. Default: 0.5.
32    pub bm25_weight: f64,
33    /// Weight for phrase bonus. Default: 0.15.
34    pub phrase_weight: f64,
35    /// Weight for position bonus. Default: 0.05.
36    pub position_weight: f64,
37    /// BM25 k1 parameter. Default: 1.2.
38    pub bm25_k1: f64,
39    /// BM25 b parameter. Default: 0.75.
40    pub bm25_b: f64,
41}
42
43impl Default for RerankConfig {
44    fn default() -> Self {
45        Self {
46            overlap_weight: 0.3,
47            bm25_weight: 0.5,
48            phrase_weight: 0.15,
49            position_weight: 0.05,
50            bm25_k1: 1.2,
51            bm25_b: 0.75,
52        }
53    }
54}
55
56/// Tokenise a string into lowercase alphanumeric tokens.
57fn tokenise(s: &str) -> Vec<String> {
58    s.split(|c: char| !c.is_alphanumeric())
59        .filter(|t| !t.is_empty())
60        .map(|t| t.to_ascii_lowercase())
61        .collect()
62}
63
64/// Compute term frequencies for a token list.
65fn term_freqs(tokens: &[String]) -> HashMap<String, f64> {
66    let mut freqs = HashMap::new();
67    let n = tokens.len() as f64;
68    for t in tokens {
69        *freqs.entry(t.clone()).or_insert(0.0) += 1.0;
70    }
71    // Normalise by document length.
72    for v in freqs.values_mut() {
73        *v /= n.max(1.0);
74    }
75    freqs
76}
77
78/// Compute Jaccard similarity between two token sets.
79fn jaccard(a: &[String], b: &[String]) -> f64 {
80    let set_a: std::collections::BTreeSet<&String> = a.iter().collect();
81    let set_b: std::collections::BTreeSet<&String> = b.iter().collect();
82    let intersection = set_a.intersection(&set_b).count();
83    let union = set_a.union(&set_b).count();
84    if union == 0 {
85        0.0
86    } else {
87        intersection as f64 / union as f64
88    }
89}
90
91/// Compute BM25 score for a single document given query terms.
92fn bm25_score(
93    query_terms: &[String],
94    doc_tokens: &[String],
95    doc_freqs: &HashMap<String, f64>,
96    avg_doc_len: f64,
97    config: &RerankConfig,
98) -> f64 {
99    let doc_len = doc_tokens.len() as f64;
100    let mut score = 0.0;
101    let n_docs = 1.0; // Single-doc scoring; IDF computed externally if needed.
102
103    for term in query_terms {
104        let tf = doc_freqs.get(term).copied().unwrap_or(0.0);
105        if tf == 0.0 {
106            continue;
107        }
108        // Simplified IDF: since we're scoring one doc at a time, use a
109        // fixed IDF of 1.0 (all terms equally rare). A full implementation
110        // would compute IDF across the corpus.
111        let idf = 1.0 + (n_docs + 0.5) / (tf + 0.5);
112        let tf_component = (tf * (config.bm25_k1 + 1.0))
113            / (tf
114                + config.bm25_k1
115                    * (1.0 - config.bm25_b + config.bm25_b * (doc_len / avg_doc_len.max(1.0))));
116        score += idf * tf_component;
117    }
118    score
119}
120
121/// Check if a multi-word phrase from the query appears in the document.
122fn phrase_bonus(query: &str, doc: &str) -> f64 {
123    let query_lower = query.to_ascii_lowercase();
124    let doc_lower = doc.to_ascii_lowercase();
125    let query_words: Vec<&str> = query_lower.split_whitespace().collect();
126    if query_words.len() < 2 {
127        return 0.0;
128    }
129    // Check for 2-grams and 3-grams from the query in the document.
130    let mut bonus = 0.0;
131    for n in 2..=3.min(query_words.len()) {
132        for window in query_words.windows(n) {
133            let phrase = window.join(" ");
134            if doc_lower.contains(&phrase) {
135                bonus += 0.1 * n as f64;
136            }
137        }
138    }
139    bonus.min(1.0)
140}
141
142/// Position bonus: matches early in the document get higher scores.
143fn position_bonus(query_terms: &[String], doc_tokens: &[String]) -> f64 {
144    if doc_tokens.is_empty() {
145        return 0.0;
146    }
147    let doc_len = doc_tokens.len();
148    let mut earliest_match = doc_len;
149    for (i, token) in doc_tokens.iter().enumerate() {
150        if query_terms.contains(token) {
151            earliest_match = earliest_match.min(i);
152        }
153    }
154    if earliest_match == doc_len {
155        0.0
156    } else {
157        // Score decreases linearly with position.
158        1.0 - (earliest_match as f64 / doc_len as f64)
159    }
160}
161
162/// Rerank candidates given a query.
163///
164/// Returns results sorted by descending score, each with the original index.
165pub fn rerank(query: &str, candidates: &[String], config: &RerankConfig) -> Vec<RerankResult> {
166    let query_tokens = tokenise(query);
167    if query_tokens.is_empty() || candidates.is_empty() {
168        return candidates
169            .iter()
170            .enumerate()
171            .map(|(i, _)| RerankResult {
172                index: i,
173                score: 0.0,
174            })
175            .collect();
176    }
177
178    let candidate_tokens: Vec<Vec<String>> = candidates.iter().map(|c| tokenise(c)).collect();
179    let avg_doc_len =
180        candidate_tokens.iter().map(|t| t.len() as f64).sum::<f64>() / candidates.len() as f64;
181
182    let mut results = Vec::with_capacity(candidates.len());
183    for (i, (candidate, doc_tokens)) in candidates.iter().zip(candidate_tokens.iter()).enumerate() {
184        let overlap = jaccard(&query_tokens, doc_tokens);
185        let doc_freqs = term_freqs(doc_tokens);
186        let bm25 = bm25_score(&query_tokens, doc_tokens, &doc_freqs, avg_doc_len, config);
187        let phrase = phrase_bonus(query, candidate);
188        let position = position_bonus(&query_tokens, doc_tokens);
189
190        let score = config.overlap_weight * overlap
191            + config.bm25_weight * (bm25 / (bm25 + 1.0)) // Normalise BM25 to [0,1]
192            + config.phrase_weight * phrase
193            + config.position_weight * position;
194
195        results.push(RerankResult { index: i, score });
196    }
197
198    // Sort by descending score.
199    results.sort_by(|a, b| {
200        b.score
201            .partial_cmp(&a.score)
202            .unwrap_or(std::cmp::Ordering::Equal)
203    });
204    results
205}
206
207/// Rerank candidates with default configuration.
208pub fn rerank_default(query: &str, candidates: &[String]) -> Vec<RerankResult> {
209    rerank(query, candidates, &RerankConfig::default())
210}
211
212#[cfg(test)]
213mod tests {
214    use super::*;
215
216    #[test]
217    fn rerank_basic_ordering() {
218        let query = "machine learning algorithms";
219        let candidates = vec![
220            "The weather is nice today".to_string(),
221            "Machine learning algorithms are powerful tools".to_string(),
222            "I like pizza".to_string(),
223        ];
224        let results = rerank_default(query, &candidates);
225        assert_eq!(results.len(), 3);
226        // The ML document should be ranked first.
227        assert_eq!(results[0].index, 1);
228        assert!(results[0].score > results[1].score);
229    }
230
231    #[test]
232    fn rerank_empty_query() {
233        let candidates = vec!["doc one".to_string(), "doc two".to_string()];
234        let results = rerank_default("", &candidates);
235        assert_eq!(results.len(), 2);
236        // All scores should be 0.
237        assert!(results.iter().all(|r| r.score == 0.0));
238    }
239
240    #[test]
241    fn rerank_empty_candidates() {
242        let results = rerank_default("query", &[]);
243        assert!(results.is_empty());
244    }
245
246    #[test]
247    fn rerank_identical_candidates() {
248        let query = "hello world";
249        let candidates = vec!["hello world".to_string(), "hello world".to_string()];
250        let results = rerank_default(query, &candidates);
251        assert_eq!(results.len(), 2);
252        // Both should have the same score.
253        assert!((results[0].score - results[1].score).abs() < 1e-9);
254    }
255
256    #[test]
257    fn rerank_phrase_bonus() {
258        let query = "machine learning";
259        let candidates = vec![
260            "machine and learning separately".to_string(),
261            "machine learning together".to_string(),
262        ];
263        let results = rerank_default(query, &candidates);
264        // The phrase match should rank higher.
265        assert_eq!(results[0].index, 1);
266    }
267
268    #[test]
269    fn rerank_position_bonus() {
270        let query = "important";
271        let candidates = vec![
272            "not relevant text here important at end".to_string(),
273            "important at the start of the document".to_string(),
274        ];
275        let results = rerank_default(query, &candidates);
276        // Earlier match should rank higher.
277        assert_eq!(results[0].index, 1);
278    }
279
280    #[test]
281    fn rerank_custom_config() {
282        let query = "test query";
283        let candidates = vec!["test document".to_string()];
284        let config = RerankConfig {
285            overlap_weight: 1.0,
286            bm25_weight: 0.0,
287            phrase_weight: 0.0,
288            position_weight: 0.0,
289            ..Default::default()
290        };
291        let results = rerank(query, &candidates, &config);
292        assert_eq!(results.len(), 1);
293        // With only overlap weight, score should be the Jaccard similarity.
294        assert!(results[0].score > 0.0);
295    }
296
297    #[test]
298    fn rerank_scores_in_descending_order() {
299        let query = "alpha beta gamma";
300        let candidates = vec![
301            "alpha".to_string(),
302            "alpha beta gamma delta".to_string(),
303            "alpha beta".to_string(),
304        ];
305        let results = rerank_default(query, &candidates);
306        for i in 1..results.len() {
307            assert!(
308                results[i - 1].score >= results[i].score,
309                "results not in descending order at index {i}"
310            );
311        }
312    }
313
314    #[test]
315    fn tokenise_handles_punctuation() {
316        let tokens = tokenise("Hello, world! Foo-Bar");
317        assert_eq!(tokens, vec!["hello", "world", "foo", "bar"]);
318    }
319
320    #[test]
321    fn jaccard_identical() {
322        let a = tokenise("hello world");
323        let b = tokenise("hello world");
324        assert_eq!(jaccard(&a, &b), 1.0);
325    }
326
327    #[test]
328    fn jaccard_disjoint() {
329        let a = tokenise("hello world");
330        let b = tokenise("foo bar");
331        assert_eq!(jaccard(&a, &b), 0.0);
332    }
333
334    #[test]
335    fn phrase_bonus_multiword() {
336        let bonus = phrase_bonus("machine learning models", "machine learning is great");
337        assert!(bonus > 0.0);
338    }
339
340    #[test]
341    fn phrase_bonus_single_word() {
342        let bonus = phrase_bonus("hello", "hello world");
343        assert_eq!(bonus, 0.0);
344    }
345
346    #[test]
347    fn position_bonus_early_match() {
348        let query = tokenise("important");
349        let doc = tokenise("important text follows here");
350        let bonus = position_bonus(&query, &doc);
351        assert!(bonus > 0.9); // Very early match
352    }
353
354    #[test]
355    fn position_bonus_no_match() {
356        let query = tokenise("missing");
357        let doc = tokenise("some text here");
358        let bonus = position_bonus(&query, &doc);
359        assert_eq!(bonus, 0.0);
360    }
361}