Skip to main content

qualia_client_core/
cml_context.rs

1//! **CML context loop** — chat logs carry Context Markup that binds topic-related semantics into the
2//! person's *inforg* (their private hypermedia library), which is then reused, permissively, to improve
3//! the context given to the local agent.
4//!
5//! CML (Timothy C. Holborn's Context Markup Language — a working draft) treats *a concept as a context
6//! hash*: `q_hash(concept)` addresses the concept's sub-graph. Here a person marks context inline in a
7//! message —
8//!   `#project:tax-2026  #task:file-return  #topic:deductions  [[capital gains]]`
9//! — and each tag becomes a `cml:Proposed` concept stored alongside the turn in the inforg. A later turn
10//! that shares those concepts pulls the earlier context back in. Multi-part by construction: one message
11//! may carry a general concept, a project, a topic, and a task at once.
12//!
13//! v1 substrate: the concept identity is the hypermedia store's `fnv60(label)` edge object (so a search
14//! for the label matches). The fuller `cml.n3` IRI-hash concept graph compiled into `.q42` for the graph
15//! retrieval path is the next increment; this loop already works end-to-end over the inforg. Reuse is
16//! **permission-gated**: guardian-flagged (sensitive) entries are never auto-injected.
17
18use crate::wellfair::hypermedia_store::{HypermediaStore, LibraryEntry};
19use qualia_core_db::hypermedia::fnv60;
20use qualia_core_db::hypermedia::{ingest_with, Descriptors, TextProcessor};
21use std::collections::HashSet;
22use std::path::Path;
23use std::time::{SystemTime, UNIX_EPOCH};
24
25/// The CML concept tiers a chat message may carry (multi-part).
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub enum ConceptTier {
28    General,
29    Project,
30    Topic,
31    Task,
32    Pursuit,
33}
34
35impl ConceptTier {
36    fn from_key(k: &str) -> Option<Self> {
37        match k.to_ascii_lowercase().as_str() {
38            "general" => Some(Self::General),
39            "project" | "proj" => Some(Self::Project),
40            "topic" => Some(Self::Topic),
41            "task" => Some(Self::Task),
42            "pursuit" | "goal" => Some(Self::Pursuit),
43            _ => None,
44        }
45    }
46    /// `str` form.
47    pub fn as_str(&self) -> &'static str {
48        match self {
49            Self::General => "general",
50            Self::Project => "project",
51            Self::Topic => "topic",
52            Self::Task => "task",
53            Self::Pursuit => "pursuit",
54        }
55    }
56}
57
58/// One context tag parsed from a message: a `cml:Proposed` concept at a tier.
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct ContextTag {
61    pub tier: ConceptTier,
62    pub label: String,
63}
64
65impl ContextTag {
66    /// The CML concept IRI for this tag (`https://ns.webcivics.net/cml/concept/<tier>/<slug>`).
67    pub fn iri(&self) -> String {
68        format!(
69            "https://ns.webcivics.net/cml/concept/{}/{}",
70            self.tier.as_str(),
71            slug(&self.label)
72        )
73    }
74    /// The concept's context hash — CML's "a concept is a context hash".
75    pub fn context_hash(&self) -> u64 {
76        fnv60(self.iri().as_bytes())
77    }
78}
79
80fn slug(s: &str) -> String {
81    let lowered: String = s
82        .trim()
83        .to_lowercase()
84        .chars()
85        .map(|c| if c.is_alphanumeric() { c } else { '-' })
86        .collect();
87    lowered.trim_matches('-').to_string()
88}
89
90/// Parse inline CML from a message. Recognises `#project:label`, `#task:label`, `#topic:label`,
91/// `#pursuit:label`, `#general:label`, bare `#hashtag` (→ topic), and `[[general concept]]` (may contain
92/// spaces). Underscores in a `#key:value` label become spaces. Duplicates (same tier + label) removed.
93pub fn parse_cml_tags(text: &str) -> Vec<ContextTag> {
94    let mut tags: Vec<ContextTag> = Vec::new();
95    let mut push = |tier: ConceptTier, label: String| {
96        let label = label.trim().to_string();
97        if !label.is_empty()
98            && !tags
99                .iter()
100                .any(|t| t.tier == tier && t.label.eq_ignore_ascii_case(&label))
101        {
102            tags.push(ContextTag { tier, label });
103        }
104    };
105
106    // [[general concepts]] (may contain spaces)
107    let mut rest = text;
108    while let Some(start) = rest.find("[[") {
109        let after = &rest[start + 2..];
110        if let Some(end) = after.find("]]") {
111            push(ConceptTier::General, after[..end].to_string());
112            rest = &after[end + 2..];
113        } else {
114            break;
115        }
116    }
117
118    // #key:value and bare #hashtag tokens
119    for raw in text.split_whitespace() {
120        let tok = raw.trim_matches(|c: char| matches!(c, '.' | ',' | '!' | '?' | ';' | ')' | '('));
121        if let Some(body) = tok.strip_prefix('#') {
122            if body.is_empty() {
123                continue;
124            }
125            if let Some((k, v)) = body.split_once(':') {
126                match ConceptTier::from_key(k) {
127                    Some(tier) => push(tier, v.replace('_', " ")),
128                    None => push(ConceptTier::Topic, body.replace('_', " ")),
129                }
130            } else {
131                push(ConceptTier::Topic, body.replace('_', " "));
132            }
133        }
134    }
135    tags
136}
137
138fn now_unix() -> u64 {
139    SystemTime::now()
140        .duration_since(UNIX_EPOCH)
141        .map(|d| d.as_secs())
142        .unwrap_or(0)
143}
144
145/// Store a chat turn's CML context into the inforg — **only when the message carries explicit tags**
146/// (deliberate, permissive). The turn's text + its concept facets become a searchable library entry.
147/// Returns the concepts stored (empty if the message had no tags).
148pub fn ingest_turn(
149    storage: &Path,
150    session_id: &str,
151    text: &str,
152) -> Result<Vec<ContextTag>, String> {
153    let tags = parse_cml_tags(text);
154    if tags.is_empty() {
155        return Ok(Vec::new());
156    }
157    let store = HypermediaStore::open(storage).map_err(|e| e.to_string())?;
158
159    let mut topics: Vec<String> = Vec::new();
160    let mut projects: Vec<String> = Vec::new();
161    let mut purposes: Vec<String> = Vec::new();
162    for t in &tags {
163        match t.tier {
164            ConceptTier::Project => projects.push(t.label.clone()),
165            ConceptTier::Task | ConceptTier::Pursuit => purposes.push(t.label.clone()),
166            ConceptTier::Topic | ConceptTier::General => topics.push(t.label.clone()),
167        }
168    }
169
170    let digest = fnv60(text.as_bytes());
171    let uri = format!("urn:qualia:chat:{session_id}:{digest:016x}");
172    let r = ingest_with(
173        &TextProcessor::default(),
174        &uri,
175        "text/plain",
176        digest,
177        text.as_bytes(),
178    );
179    let subject = r.container.primary.subject();
180    let mut quins = r.quins;
181    let desc = Descriptors {
182        topics: topics.clone(),
183        projects: projects.clone(),
184        purposes: purposes.clone(),
185        ..Default::default()
186    };
187    let (dq, _lex) = qualia_core_db::hypermedia::descriptors_to_nquins(subject, &desc);
188    quins.extend(dq);
189
190    // Enrich chat turn with the same Rust CML context graph (TEXT→CONCEPT→LOGIC).
191    let units = crate::wellfair::cml_context::units_from_headings(text);
192    let g = crate::wellfair::cml_context::build_document_context(&uri, "chat-turn", &units);
193    for t in &g.topics {
194        if !topics.iter().any(|x| x == t) {
195            topics.push(t.clone());
196        }
197    }
198    for p in &g.purposes {
199        if !purposes.iter().any(|x| x == p) {
200            purposes.push(p.clone());
201        }
202    }
203    quins.extend(g.quins);
204
205    let mut entry = LibraryEntry {
206        asset_uri: uri,
207        primary_subject: subject,
208        media_type: "text/plain".to_string(),
209        quins,
210        topics,
211        projects,
212        purposes: purposes.clone(),
213        place: None,
214        occurred_at: None,
215        lat: None,
216        lon: None,
217        flags: Vec::new(),
218        ingested_unix: now_unix(),
219        excerpt: text.chars().take(240).collect(),
220        sensitivity: "public".into(),
221        section: "personal".into(),
222        commons_visibility: Default::default(),
223        cml_signals: g.signal_tags,
224        cml_concept_count: g.concepts.len() as u32,
225        cml_n3: if g.n3.len() > 16_000 {
226            format!("{}…", &g.n3[..16_000])
227        } else {
228            g.n3
229        },
230        cof_html: String::new(),
231        cof_segment_count: 0,
232        cof_segment_index: 0,
233        cof_profile: String::new(),
234    };
235    entry.recompute_section();
236    store.add(entry).map_err(|e| e.to_string())?;
237    Ok(tags)
238}
239
240const STOPWORDS: &[&str] = &[
241    "the", "and", "for", "with", "that", "this", "what", "how", "why", "who", "you", "your", "are",
242    "was", "can", "will", "into", "from", "about", "have", "has", "not", "but", "get", "got",
243    "tell", "give", "please", "would", "could", "should", "them", "they", "our", "out",
244];
245
246fn salient_terms(prompt: &str) -> Vec<String> {
247    let mut seen = HashSet::new();
248    let mut out = Vec::new();
249    for w in prompt.split(|c: char| !c.is_alphanumeric()) {
250        let w = w.to_lowercase();
251        if w.len() > 3 && !STOPWORDS.contains(&w.as_str()) && seen.insert(w.clone()) {
252            out.push(w);
253        }
254    }
255    out
256}
257
258fn absorb(entries: Vec<LibraryEntry>, chosen: &mut Vec<LibraryEntry>, seen: &mut HashSet<String>) {
259    for e in entries {
260        // Permission gate: guardian-flagged (sensitive) entries are never auto-injected.
261        if e.flags.is_empty() && seen.insert(e.asset_uri.clone()) {
262            chosen.push(e);
263        }
264    }
265}
266
267/// Retrieve inforg context relevant to `prompt` and format it as a prompt block (empty if nothing).
268/// Matches the prompt's explicit tags first (reliable), then a few salient terms (best-effort), over
269/// the person's library. Permission-gated (flagged entries excluded).
270pub fn retrieve_context(storage: &Path, prompt: &str, max_snippets: usize) -> String {
271    let store = match HypermediaStore::open(storage) {
272        Ok(s) => s,
273        Err(_) => return String::new(),
274    };
275    let mut chosen: Vec<LibraryEntry> = Vec::new();
276    let mut seen: HashSet<String> = HashSet::new();
277
278    for tag in parse_cml_tags(prompt) {
279        let facet = match tag.tier {
280            ConceptTier::Project => "project",
281            ConceptTier::Task | ConceptTier::Pursuit => "purpose",
282            _ => "topic",
283        };
284        if let Ok(entries) = store.search(facet, &tag.label) {
285            absorb(entries, &mut chosen, &mut seen);
286        }
287    }
288    for term in salient_terms(prompt).into_iter().take(6) {
289        if chosen.len() >= max_snippets {
290            break;
291        }
292        if let Ok(entries) = store.search("topic", &term) {
293            absorb(entries, &mut chosen, &mut seen);
294        }
295    }
296
297    if chosen.is_empty() {
298        return String::new();
299    }
300    let mut lines = Vec::new();
301    for e in chosen.into_iter().take(max_snippets) {
302        let mut facets: Vec<String> = Vec::new();
303        facets.extend(e.projects.iter().cloned());
304        facets.extend(e.topics.iter().cloned());
305        let tag = if facets.is_empty() {
306            String::new()
307        } else {
308            format!(" [{}]", facets.join(", "))
309        };
310        lines.push(format!("- {}{}", e.excerpt.trim(), tag));
311    }
312    format!(
313        "Relevant context from your library (inforg) — use for grounding, cite if used:\n{}",
314        lines.join("\n")
315    )
316}
317
318#[cfg(test)]
319mod tests {
320    use super::*;
321
322    #[test]
323    fn parses_multi_part_context() {
324        let tags = parse_cml_tags(
325            "note #project:tax_2026 [[capital gains]] #deductions #task:file-return",
326        );
327        assert!(tags.contains(&ContextTag {
328            tier: ConceptTier::Project,
329            label: "tax 2026".into()
330        }));
331        assert!(tags.contains(&ContextTag {
332            tier: ConceptTier::General,
333            label: "capital gains".into()
334        }));
335        assert!(tags.contains(&ContextTag {
336            tier: ConceptTier::Topic,
337            label: "deductions".into()
338        }));
339        assert!(tags.contains(&ContextTag {
340            tier: ConceptTier::Task,
341            label: "file-return".into()
342        }));
343    }
344
345    #[test]
346    fn untagged_message_stores_nothing() {
347        let dir = tempfile::tempdir().unwrap();
348        let stored = ingest_turn(dir.path(), "s1", "just a plain message with no markup").unwrap();
349        assert!(stored.is_empty());
350        assert!(retrieve_context(dir.path(), "anything", 4).is_empty());
351    }
352
353    #[test]
354    fn tagged_turn_is_stored_and_reused() {
355        let dir = tempfile::tempdir().unwrap();
356        let stored = ingest_turn(
357            dir.path(),
358            "s1",
359            "The liver secretes bile. #project:hep-notes #topic:anatomy",
360        )
361        .unwrap();
362        assert_eq!(stored.len(), 2);
363        // A later turn sharing the project pulls the earlier context back in.
364        let ctx = retrieve_context(dir.path(), "remind me about #project:hep-notes", 4);
365        assert!(ctx.contains("liver"), "expected inforg recall, got: {ctx}");
366        // An unrelated project matches nothing.
367        assert!(retrieve_context(dir.path(), "unrelated #project:nothing-here", 4).is_empty());
368    }
369
370    #[test]
371    fn concept_is_a_context_hash() {
372        let t = ContextTag {
373            tier: ConceptTier::Topic,
374            label: "Capital Gains".into(),
375        };
376        assert_eq!(
377            t.iri(),
378            "https://ns.webcivics.net/cml/concept/topic/capital-gains"
379        );
380        assert_ne!(t.context_hash(), 0);
381    }
382}