Skip to main content

qualia_semantic_library/
library.rs

1//! The semantic library: an index over a directory of `.hmc` containers.
2//!
3//! Reading just the manifests is cheap (they are stored uncompressed at the top
4//! of each zip), so a catalog of thousands of documents builds fast without
5//! unpacking heavy assets. Embeddings are loaded on demand for dedup and search.
6//!
7//! This is where the "optimisation manifold" lives: documents become points in
8//! embedding space, so the library can collapse near-duplicates, route a query
9//! to the relevant region, and rank chunks by relevance/novelty — instead of
10//! scanning every byte of a 60 GB corpus.
11
12use std::path::{Path, PathBuf};
13
14use walkdir::WalkDir;
15
16use crate::container::{AssetKind, HmcContainer, HmcManifest};
17use crate::embedding::{centroid, cosine, decode_f32_matrix};
18
19/// One catalogued container (manifest + path; assets read on demand).
20pub struct Entry {
21    pub path: PathBuf,
22    pub manifest: HmcManifest,
23}
24
25impl Entry {
26    /// Title.
27    pub fn title(&self) -> &str {
28        if self.manifest.source.title.is_empty() {
29            &self.manifest.source.filename
30        } else {
31            &self.manifest.source.title
32        }
33    }
34}
35
36/// In-memory catalog of a library directory.
37pub struct Library {
38    pub root: PathBuf,
39    pub entries: Vec<Entry>,
40}
41
42/// A chunk-level search hit.
43pub struct Hit {
44    pub doc_id: String,
45    pub title: String,
46    pub chunk_idx: usize,
47    pub score: f32,
48    pub snippet: String,
49}
50
51impl Library {
52    /// Scan a directory for `.hmc` containers and read their manifests.
53    pub fn scan(root: &Path) -> anyhow::Result<Self> {
54        let mut entries = Vec::new();
55        for e in WalkDir::new(root).into_iter().filter_map(|e| e.ok()) {
56            if !e.file_type().is_file() {
57                continue;
58            }
59            let p = e.path();
60            if p.extension().and_then(|x| x.to_str()) != Some(crate::container::HMC_EXTENSION) {
61                continue;
62            }
63            match HmcContainer::open(p) {
64                Ok(c) => entries.push(Entry {
65                    path: p.to_path_buf(),
66                    manifest: c.manifest().clone(),
67                }),
68                Err(err) => eprintln!("[library] skip {}: {err}", p.display()),
69            }
70        }
71        Ok(Library {
72            root: root.to_path_buf(),
73            entries,
74        })
75    }
76
77    /// Length.
78    pub fn len(&self) -> usize {
79        self.entries.len()
80    }
81
82    /// `true` when empty.
83    pub fn is_empty(&self) -> bool {
84        self.entries.is_empty()
85    }
86
87    /// Exact duplicates by document identity (BLAKE3). Returns groups with >1
88    /// member: (doc_id, paths).
89    pub fn exact_duplicates(&self) -> Vec<(String, Vec<PathBuf>)> {
90        use std::collections::BTreeMap;
91        let mut by_id: BTreeMap<String, Vec<PathBuf>> = BTreeMap::new();
92        for e in &self.entries {
93            by_id
94                .entry(e.manifest.doc_id.clone())
95                .or_default()
96                .push(e.path.clone());
97        }
98        by_id.into_iter().filter(|(_, v)| v.len() > 1).collect()
99    }
100
101    /// Load a container's chunk texts and embedding matrix, if present.
102    fn load_vectors(&self, entry: &Entry) -> Option<(Vec<String>, Vec<Vec<f32>>)> {
103        let dim = entry.manifest.pipeline.embed_dim as usize;
104        if dim == 0 {
105            return None;
106        }
107        let mut c = HmcContainer::open(&entry.path).ok()?;
108        let emb = c.read_kind(AssetKind::Embeddings).ok()?;
109        let vectors = decode_f32_matrix(&emb, dim);
110        let chunks_raw = c.read_kind(AssetKind::Chunks).ok()?;
111        let texts: Vec<String> = String::from_utf8_lossy(&chunks_raw)
112            .lines()
113            .filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
114            .map(|v| {
115                v.get("text")
116                    .and_then(|t| t.as_str())
117                    .unwrap_or("")
118                    .to_string()
119            })
120            .collect();
121        Some((texts, vectors))
122    }
123
124    /// Document-level centroid embedding (mean of chunk vectors), for near-dup
125    /// detection and novelty scoring.
126    pub fn document_centroid(&self, entry: &Entry) -> Option<Vec<f32>> {
127        let (_, vectors) = self.load_vectors(entry)?;
128        let c = centroid(&vectors);
129        if c.is_empty() {
130            None
131        } else {
132            Some(c)
133        }
134    }
135
136    /// Near-duplicate document pairs above `threshold` cosine on their
137    /// centroids (catches versions/reprints byte-hashing misses).
138    pub fn near_duplicates(&self, threshold: f32) -> Vec<(String, String, f32)> {
139        let cents: Vec<(String, Vec<f32>)> = self
140            .entries
141            .iter()
142            .filter_map(|e| {
143                self.document_centroid(e)
144                    .map(|c| (e.manifest.doc_id.clone(), c))
145            })
146            .collect();
147        let mut out = Vec::new();
148        for i in 0..cents.len() {
149            for j in (i + 1)..cents.len() {
150                let s = cosine(&cents[i].1, &cents[j].1);
151                if s >= threshold {
152                    out.push((cents[i].0.clone(), cents[j].0.clone(), s));
153                }
154            }
155        }
156        out.sort_by(|a, b| b.2.total_cmp(&a.2));
157        out
158    }
159
160    /// Semantic search: rank chunks across the whole library by cosine to a
161    /// query vector. Returns the top `k` hits.
162    pub fn search(&self, query: &[f32], k: usize) -> Vec<Hit> {
163        let mut hits: Vec<Hit> = Vec::new();
164        for e in &self.entries {
165            let Some((texts, vectors)) = self.load_vectors(e) else {
166                continue;
167            };
168            for (i, v) in vectors.iter().enumerate() {
169                let score = cosine(query, v);
170                let snippet = texts.get(i).map(|t| snippet(t)).unwrap_or_default();
171                hits.push(Hit {
172                    doc_id: e.manifest.doc_id.clone(),
173                    title: e.title().to_string(),
174                    chunk_idx: i,
175                    score,
176                    snippet,
177                });
178            }
179        }
180        hits.sort_by(|a, b| b.score.total_cmp(&a.score));
181        hits.truncate(k);
182        hits
183    }
184
185    /// Novelty of each document relative to a set of "known capability" vectors:
186    /// 1 − max cosine to any known vector. Higher = more novel = worth attention.
187    /// Returns (doc_id, title, novelty), most-novel first.
188    pub fn novelty_ranking(&self, known: &[Vec<f32>]) -> Vec<(String, String, f32)> {
189        let mut out = Vec::new();
190        for e in &self.entries {
191            let Some(c) = self.document_centroid(e) else {
192                continue;
193            };
194            let max_sim = known.iter().map(|k| cosine(&c, k)).fold(0.0f32, f32::max);
195            out.push((
196                e.manifest.doc_id.clone(),
197                e.title().to_string(),
198                1.0 - max_sim,
199            ));
200        }
201        out.sort_by(|a, b| b.2.total_cmp(&a.2));
202        out
203    }
204}
205
206fn snippet(text: &str) -> String {
207    let s: String = text.chars().take(160).collect();
208    s.replace('\n', " ")
209}