qualia_semantic_library/
library.rs1use std::path::{Path, PathBuf};
13
14use walkdir::WalkDir;
15
16use crate::container::{AssetKind, HmcContainer, HmcManifest};
17use crate::embedding::{centroid, cosine, decode_f32_matrix};
18
19pub struct Entry {
21 pub path: PathBuf,
22 pub manifest: HmcManifest,
23}
24
25impl Entry {
26 pub fn title(&self) -> &str {
28 if self.manifest.source.title.is_empty() {
29 &self.manifest.source.filename
30 } else {
31 &self.manifest.source.title
32 }
33 }
34}
35
36pub struct Library {
38 pub root: PathBuf,
39 pub entries: Vec<Entry>,
40}
41
42pub struct Hit {
44 pub doc_id: String,
45 pub title: String,
46 pub chunk_idx: usize,
47 pub score: f32,
48 pub snippet: String,
49}
50
51impl Library {
52 pub fn scan(root: &Path) -> anyhow::Result<Self> {
54 let mut entries = Vec::new();
55 for e in WalkDir::new(root).into_iter().filter_map(|e| e.ok()) {
56 if !e.file_type().is_file() {
57 continue;
58 }
59 let p = e.path();
60 if p.extension().and_then(|x| x.to_str()) != Some(crate::container::HMC_EXTENSION) {
61 continue;
62 }
63 match HmcContainer::open(p) {
64 Ok(c) => entries.push(Entry {
65 path: p.to_path_buf(),
66 manifest: c.manifest().clone(),
67 }),
68 Err(err) => eprintln!("[library] skip {}: {err}", p.display()),
69 }
70 }
71 Ok(Library {
72 root: root.to_path_buf(),
73 entries,
74 })
75 }
76
77 pub fn len(&self) -> usize {
79 self.entries.len()
80 }
81
82 pub fn is_empty(&self) -> bool {
84 self.entries.is_empty()
85 }
86
87 pub fn exact_duplicates(&self) -> Vec<(String, Vec<PathBuf>)> {
90 use std::collections::BTreeMap;
91 let mut by_id: BTreeMap<String, Vec<PathBuf>> = BTreeMap::new();
92 for e in &self.entries {
93 by_id
94 .entry(e.manifest.doc_id.clone())
95 .or_default()
96 .push(e.path.clone());
97 }
98 by_id.into_iter().filter(|(_, v)| v.len() > 1).collect()
99 }
100
101 fn load_vectors(&self, entry: &Entry) -> Option<(Vec<String>, Vec<Vec<f32>>)> {
103 let dim = entry.manifest.pipeline.embed_dim as usize;
104 if dim == 0 {
105 return None;
106 }
107 let mut c = HmcContainer::open(&entry.path).ok()?;
108 let emb = c.read_kind(AssetKind::Embeddings).ok()?;
109 let vectors = decode_f32_matrix(&emb, dim);
110 let chunks_raw = c.read_kind(AssetKind::Chunks).ok()?;
111 let texts: Vec<String> = String::from_utf8_lossy(&chunks_raw)
112 .lines()
113 .filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
114 .map(|v| {
115 v.get("text")
116 .and_then(|t| t.as_str())
117 .unwrap_or("")
118 .to_string()
119 })
120 .collect();
121 Some((texts, vectors))
122 }
123
124 pub fn document_centroid(&self, entry: &Entry) -> Option<Vec<f32>> {
127 let (_, vectors) = self.load_vectors(entry)?;
128 let c = centroid(&vectors);
129 if c.is_empty() {
130 None
131 } else {
132 Some(c)
133 }
134 }
135
136 pub fn near_duplicates(&self, threshold: f32) -> Vec<(String, String, f32)> {
139 let cents: Vec<(String, Vec<f32>)> = self
140 .entries
141 .iter()
142 .filter_map(|e| {
143 self.document_centroid(e)
144 .map(|c| (e.manifest.doc_id.clone(), c))
145 })
146 .collect();
147 let mut out = Vec::new();
148 for i in 0..cents.len() {
149 for j in (i + 1)..cents.len() {
150 let s = cosine(¢s[i].1, ¢s[j].1);
151 if s >= threshold {
152 out.push((cents[i].0.clone(), cents[j].0.clone(), s));
153 }
154 }
155 }
156 out.sort_by(|a, b| b.2.total_cmp(&a.2));
157 out
158 }
159
160 pub fn search(&self, query: &[f32], k: usize) -> Vec<Hit> {
163 let mut hits: Vec<Hit> = Vec::new();
164 for e in &self.entries {
165 let Some((texts, vectors)) = self.load_vectors(e) else {
166 continue;
167 };
168 for (i, v) in vectors.iter().enumerate() {
169 let score = cosine(query, v);
170 let snippet = texts.get(i).map(|t| snippet(t)).unwrap_or_default();
171 hits.push(Hit {
172 doc_id: e.manifest.doc_id.clone(),
173 title: e.title().to_string(),
174 chunk_idx: i,
175 score,
176 snippet,
177 });
178 }
179 }
180 hits.sort_by(|a, b| b.score.total_cmp(&a.score));
181 hits.truncate(k);
182 hits
183 }
184
185 pub fn novelty_ranking(&self, known: &[Vec<f32>]) -> Vec<(String, String, f32)> {
189 let mut out = Vec::new();
190 for e in &self.entries {
191 let Some(c) = self.document_centroid(e) else {
192 continue;
193 };
194 let max_sim = known.iter().map(|k| cosine(&c, k)).fold(0.0f32, f32::max);
195 out.push((
196 e.manifest.doc_id.clone(),
197 e.title().to_string(),
198 1.0 - max_sim,
199 ));
200 }
201 out.sort_by(|a, b| b.2.total_cmp(&a.2));
202 out
203 }
204}
205
206fn snippet(text: &str) -> String {
207 let s: String = text.chars().take(160).collect();
208 s.replace('\n', " ")
209}