qualia_client_core/wellfair/cml_context/
cof_html.rs

1//! **COF — Context Optimisation Format** (HTML+RDFa profile).
2//!
3//! Design (see `tools/legislation-etl/cof.n3` and plan `docs/plans/cof-html-rdfa-etl.md`):
4//!
5//! - **CML** = TEXT → CONCEPT → LOGIC graph (N3 / NQuin).
6//! - **COF** = how that graph is serialised for **agent context windows**: constrained
7//!   HTML5 + RDFa (`typeof` / `about` / `property` / `resource` / `rel`) with almost no
8//!   layout tokens. Domain meaning stays on `cml:` / `values:` / `skos:`.
9//!
10//! Profile IRI: `https://ns.webcivics.net/cof/profile/html-rdfa-1`
11//!
12//! Large instruments are **segmented** at section boundaries so a host can load only the
13//! token budget required for a turn (index segment + one body segment).
14
15use serde::{Deserialize, Serialize};
16
17use super::extract::{classify_deontic, extract_privacy_signals};
18use super::graph::ContextUnit;
19
20/// Official COF HTML+RDFa profile (must match `cof.n3` / legis2cml).
21pub const COF_PROFILE: &str = "https://ns.webcivics.net/cof/profile/html-rdfa-1";
22pub const COF_NS: &str = "https://ns.webcivics.net/cof/";
23pub const CML_NS: &str = "https://ns.webcivics.net/cml/";
24pub const VALUES_NS: &str = "https://ns.webcivics.net/values/";
25pub const MEDIA_TYPE_COF: &str =
26    "text/html;profile=\"https://ns.webcivics.net/cof/profile/html-rdfa-1\"";
27
28/// Default agent body-segment budget (~6–8k tokens at ~4 chars/token; leave headroom).
29pub const DEFAULT_SEGMENT_MAX_CHARS: usize = 24_000;
30/// Soft floor: never emit a body segment smaller than this unless it is the only content.
31pub const DEFAULT_SEGMENT_MIN_CHARS: usize = 2_000;
32
33/// One COF HTML segment (self-contained HTML document, RDFa-complete).
34#[derive(Debug, Clone, Serialize, Deserialize)]
35pub struct CofSegment {
36    /// 0 = index/TOC; 1..N = body segments.
37    pub index: u32,
38    pub total: u32,
39    pub id: String,
40    pub title: String,
41    /// Full HTML document for this segment.
42    pub html: String,
43    pub char_count: usize,
44    /// Approximate token estimate (chars / 4).
45    pub approx_tokens: usize,
46    /// Concept / unit frags included in this segment.
47    pub unit_frags: Vec<String>,
48    pub is_index: bool,
49}
50
51/// A complete COF package: index + ordered body segments.
52#[derive(Debug, Clone, Serialize, Deserialize)]
53pub struct CofPackage {
54    pub document_uri: String,
55    pub title: String,
56    pub profile: String,
57    pub segment_max_chars: usize,
58    pub segments: Vec<CofSegment>,
59    pub total_chars: usize,
60    pub total_approx_tokens: usize,
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq)]
64pub enum CofStyle {
65    /// Minimal markup for agent windows (no CSS, short header).
66    AgentLean,
67    /// Thin human CSS + proposed banner (still COF attributes for machines).
68    DualSurface,
69}
70
71impl Default for CofStyle {
72    fn default() -> Self {
73        Self::AgentLean
74    }
75}
76
77fn esc(s: &str) -> String {
78    let mut out = String::with_capacity(s.len());
79    for c in s.chars() {
80        match c {
81            '&' => out.push_str("&amp;"),
82            '<' => out.push_str("&lt;"),
83            '>' => out.push_str("&gt;"),
84            '"' => out.push_str("&quot;"),
85            '\'' => out.push_str("&#39;"),
86            _ => out.push(c),
87        }
88    }
89    out
90}
91
92fn approx_tokens(chars: usize) -> usize {
93    chars.div_ceil(4)
94}
95
96fn prefix_attr() -> String {
97    format!(
98        "rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# \
99         rdfs: http://www.w3.org/2000/01/rdf-schema# \
100         dc: http://purl.org/dc/terms/ \
101         skos: http://www.w3.org/2004/02/skos/core# \
102         prov: http://www.w3.org/ns/prov# \
103         cml: {CML_NS} \
104         values: {VALUES_NS} \
105         cof: {COF_NS}"
106    )
107}
108
109fn dual_css() -> &'static str {
110    /* Intentionally tiny — COF forbids presentation bloat; this is human affordance only. */
111    "body{font:15px/1.45 system-ui,sans-serif;max-width:48rem;margin:1rem auto;padding:0 1rem;color:#111}\
112     h1{font-size:1.25rem;margin:0 0 .5rem}h2,h3{font-size:1rem;margin:1rem 0 .35rem}\
113     .banner{border-left:3px solid #6a4c93;background:#f4f1f7;padding:.5rem .75rem;margin:.75rem 0;font-size:.85rem}\
114     section{margin:.9rem 0;padding-bottom:.5rem;border-bottom:1px solid #eee}\
115     .text{white-space:pre-wrap}.meta{color:#666;font-size:.8rem}\
116     .sig{display:inline-block;font-size:.7rem;background:#eee;border-radius:999px;padding:.05rem .4rem;margin:.1rem}\
117     aside.logic{font:12px ui-monospace,monospace;background:#f7f5fa;border-left:2px solid #6a4c93;padding:.4rem .6rem;margin:.4rem 0}"
118}
119
120/// Render one unit as a COF section fragment (no document chrome).
121pub fn render_unit_fragment(doc_uri: &str, unit: &ContextUnit, style: CofStyle) -> String {
122    let _ = style;
123    let frag = esc(&unit.frag);
124    let label = esc(&unit.label);
125    let kind = esc(&unit.kind);
126    let about = esc(&format!("{doc_uri}#{}", unit.frag));
127    let resource = about.clone();
128    let (deontic, dconf) = if unit.text.trim().is_empty() {
129        (super::extract::DeonticClass::Undertaking, 0u8)
130    } else {
131        classify_deontic(&unit.text)
132    };
133    let privacy = extract_privacy_signals(&unit.text);
134    let signals: Vec<String> = privacy
135        .iter()
136        .map(|s| format!("privacy:{}", s.signal))
137        .chain(std::iter::once(format!("deontic:{}", deontic.as_str())))
138        .collect();
139    let sig_attr = esc(&signals.join(" "));
140    let conf = format!("{:.2}", dconf as f32 / 100.0);
141    let page_attr = unit
142        .page
143        .map(|p| format!(" data-page=\"{p}\""))
144        .unwrap_or_default();
145    let part_of = unit
146        .parent
147        .as_ref()
148        .map(|p| {
149            format!(
150                " rel=\"values:partOf\" resource=\"{}\"",
151                esc(&format!("{doc_uri}#{p}"))
152            )
153        })
154        .unwrap_or_default();
155
156    let mut sig_chips = String::new();
157    for s in &signals {
158        sig_chips.push_str(&format!(
159            "<span class=\"sig\" property=\"cml:hasSignal\" content=\"{}\">{}</span> ",
160            esc(s),
161            esc(s)
162        ));
163    }
164
165    let text_block = if unit.text.trim().is_empty() {
166        String::new()
167    } else {
168        let claim_id = format!("{doc_uri}#{}-claim", unit.frag);
169        format!(
170            "<div class=\"text\" typeof=\"cof:Block\" property=\"cof:hasBlock values:originalText\" \
171             resource=\"{doc_uri}#{frag}-text\"{page_attr}>\
172             <span typeof=\"cof:Claim\" property=\"cof:hasClaim\" about=\"{claim}\" \
173             resource=\"{claim}\" data-confidence=\"{conf}\" data-deontic=\"{deontic}\" \
174             data-signals=\"{sig_attr}\">{text}</span></div>",
175            claim = esc(&claim_id),
176            deontic = deontic.as_str(),
177            text = esc(&unit.text),
178        )
179    };
180
181    let logic =
182        if matches!(deontic, super::extract::DeonticClass::Undertaking) && privacy.is_empty() {
183            String::new()
184        } else {
185            format!(
186                "<aside class=\"logic\" typeof=\"cml:LogicApplication\" property=\"cml:asserts\" \
187             about=\"{about}-norm\">\
188             <span property=\"cml:modality\" resource=\"cml:Deontic\">deontic</span> \
189             <span property=\"values:deonticClass\">{deontic}</span> · conf {conf} · \
190             <span property=\"cml:curationStatus\" resource=\"cml:Proposed\">cml:Proposed</span>\
191             </aside>",
192                deontic = deontic.as_str(),
193            )
194        };
195
196    format!(
197        "<section id=\"{frag}\" typeof=\"cml:Concept cof:Section\" about=\"{about}\" \
198         resource=\"{resource}\" property=\"cof:hasSection\" data-kind=\"{kind}\"{page_attr}{part_of} \
199         data-confidence=\"{conf}\" data-deontic=\"{deontic}\" data-signals=\"{sig_attr}\">\
200         <h3><span property=\"skos:prefLabel cof:title\">{label}</span></h3>\
201         <div class=\"meta\">{sig_chips}</div>\
202         {text_block}{logic}\
203         <link rel=\"cml:realizedBy\" href=\"{doc_uri}#{frag}\" />\
204         <link rel=\"cml:curationStatus\" href=\"{CML_NS}Proposed\" />\
205         </section>\n",
206        deontic = deontic.as_str(),
207    )
208}
209
210fn wrap_document(
211    doc_uri: &str,
212    title: &str,
213    segment: &CofSegmentMeta,
214    body_inner: &str,
215    style: CofStyle,
216) -> String {
217    let prefix = prefix_attr();
218    let css = match style {
219        CofStyle::AgentLean => String::new(),
220        CofStyle::DualSurface => format!("<style>{}</style>", dual_css()),
221    };
222    let banner = match style {
223        CofStyle::AgentLean => format!(
224            "<div property=\"cml:curationStatus\" resource=\"cml:Proposed\" \
225             content=\"cml:Proposed\">cml:Proposed · COF segment {}/{} · profile {COF_PROFILE}</div>",
226            segment.index + 1,
227            segment.total
228        ),
229        CofStyle::DualSurface => format!(
230            "<div class=\"banner\">⚑ <strong>cml:Proposed</strong> — machine layer only. \
231             COF segment <span property=\"cof:segmentIndex\" content=\"{}\">{}/{}</span> · \
232             profile <span property=\"cof:profile\" content=\"{COF_PROFILE}\">{COF_PROFILE}</span>. \
233             Load only the segments you need (token optimisation).</div>",
234            segment.index,
235            segment.index + 1,
236            segment.total
237        ),
238    };
239    let nav = {
240        let mut n = String::new();
241        if let Some(p) = &segment.prev_id {
242            n.push_str(&format!(
243                "<link rel=\"cof:prevSegment\" href=\"{}\" />\n",
244                esc(p)
245            ));
246        }
247        if let Some(nx) = &segment.next_id {
248            n.push_str(&format!(
249                "<link rel=\"cof:nextSegment\" href=\"{}\" />\n",
250                esc(nx)
251            ));
252        }
253        n.push_str(&format!(
254            "<meta name=\"cof-segment\" content=\"{}\">\n\
255             <meta name=\"cof-segment-total\" content=\"{}\">\n\
256             <meta name=\"cof-segment-id\" content=\"{}\">\n",
257            segment.index,
258            segment.total,
259            esc(&segment.id)
260        ));
261        n
262    };
263
264    format!(
265        r#"<!DOCTYPE html>
266<html lang="en" prefix="{prefix}">
267<head>
268<meta charset="utf-8">
269<meta name="viewport" content="width=device-width, initial-scale=1">
270<meta name="cof-profile" content="{COF_PROFILE}">
271<meta name="cml-schema" content="2">
272{nav}{css}
273<title>{title_esc} — COF {seg_label}</title>
274</head>
275<body typeof="cof:Document" about="{doc}" resource="{doc}" vocab="{COF_NS}"
276      data-cof-segment="{idx}" data-cof-segment-total="{total}">
277<header>
278  <h1 property="dc:title cof:title">{title_esc}</h1>
279  <div class="meta">HTML+RDFa COF · CML TEXT→CONCEPT→LOGIC · engine=qualia-rust</div>
280  <link rel="prov:wasDerivedFrom" href="{doc}" />
281  {banner}
282</header>
283<main property="cof:body">
284{body}
285</main>
286</body>
287</html>
288"#,
289        title_esc = esc(title),
290        seg_label = if segment.is_index {
291            "index".into()
292        } else {
293            format!("seg-{}", segment.index)
294        },
295        doc = esc(doc_uri),
296        idx = segment.index,
297        total = segment.total,
298        body = body_inner,
299    )
300}
301
302struct CofSegmentMeta {
303    index: u32,
304    total: u32,
305    id: String,
306    prev_id: Option<String>,
307    next_id: Option<String>,
308    is_index: bool,
309}
310
311/// Pack units into COF segments under a character budget (section-aligned).
312pub fn pack_units_into_segments(
313    units: &[ContextUnit],
314    max_chars: usize,
315) -> Vec<(Vec<usize>, usize)> {
316    // Returns list of (unit_indices, estimated_chars).
317    let mut packs: Vec<(Vec<usize>, usize)> = Vec::new();
318    let mut cur: Vec<usize> = Vec::new();
319    let mut cur_chars = 0usize;
320    let max_chars = max_chars.max(DEFAULT_SEGMENT_MIN_CHARS);
321
322    for (i, u) in units.iter().enumerate() {
323        // Rough size: label + text + markup overhead (~200).
324        let unit_chars = u.label.len() + u.text.len() + 200;
325        if unit_chars > max_chars {
326            // Oversized unit: flush current, then emit alone (caller may further split text).
327            if !cur.is_empty() {
328                packs.push((cur, cur_chars));
329                cur = Vec::new();
330                cur_chars = 0;
331            }
332            packs.push((vec![i], unit_chars));
333            continue;
334        }
335        if !cur.is_empty() && cur_chars + unit_chars > max_chars {
336            packs.push((cur, cur_chars));
337            cur = Vec::new();
338            cur_chars = 0;
339        }
340        cur.push(i);
341        cur_chars += unit_chars;
342    }
343    if !cur.is_empty() {
344        packs.push((cur, cur_chars));
345    }
346    if packs.is_empty() {
347        packs.push((Vec::new(), 0));
348    }
349    packs
350}
351
352/// Build a full COF package (index + body segments) for agent token optimisation.
353pub fn build_cof_package(
354    doc_uri: &str,
355    title: &str,
356    units: &[ContextUnit],
357    max_chars: usize,
358    style: CofStyle,
359) -> CofPackage {
360    let packs = pack_units_into_segments(units, max_chars);
361    let body_count = packs.len() as u32;
362    // total segments = 1 index + body_count (even if empty body → index only)
363    let total = body_count + 1;
364
365    // --- Index segment (TOC): titles + signals, no full body text ---
366    let mut index_body = String::from(
367        "<nav typeof=\"cof:Section\" property=\"cof:hasSection\" about=\"#index\" id=\"index\">\n\
368         <h2 property=\"cof:title\">Index (token-cheap map)</h2>\n\
369         <ol>\n",
370    );
371    for (seg_i, (idxs, _chars)) in packs.iter().enumerate() {
372        let seg_id = format!("{doc_uri}#cof-seg-{}", seg_i + 1);
373        index_body.push_str(&format!(
374            "<li property=\"cof:hasSegment\" resource=\"{}\">segment {} · {} unit(s)<ul>",
375            esc(&seg_id),
376            seg_i + 1,
377            idxs.len()
378        ));
379        for &ui in idxs {
380            let u = &units[ui];
381            let (deontic, _) = classify_deontic(&u.text);
382            let priv_n = extract_privacy_signals(&u.text).len();
383            index_body.push_str(&format!(
384                "<li><a property=\"cof:ref\" href=\"{doc_uri}#{frag}\" resource=\"{doc_uri}#{frag}\">{label}</a> \
385                 <span class=\"meta\">deontic:{deontic} · privacy:{priv_n}</span></li>",
386                frag = esc(&u.frag),
387                label = esc(&u.label),
388                deontic = deontic.as_str(),
389            ));
390        }
391        index_body.push_str("</ul></li>\n");
392    }
393    index_body.push_str("</ol>\n<p class=\"meta\">Load body segments by <code>cof:nextSegment</code> / segment id. Bodies hold <code>values:originalText</code> claims.</p>\n</nav>\n");
394
395    let mut segment_ids: Vec<String> = Vec::with_capacity(total as usize);
396    segment_ids.push(format!("{doc_uri}#cof-seg-0"));
397    for i in 0..body_count {
398        segment_ids.push(format!("{doc_uri}#cof-seg-{}", i + 1));
399    }
400
401    let mut segments = Vec::new();
402
403    // Index HTML
404    {
405        let meta = CofSegmentMeta {
406            index: 0,
407            total,
408            id: segment_ids[0].clone(),
409            prev_id: None,
410            next_id: segment_ids.get(1).cloned(),
411            is_index: true,
412        };
413        let html = wrap_document(doc_uri, title, &meta, &index_body, style);
414        let char_count = html.len();
415        segments.push(CofSegment {
416            index: 0,
417            total,
418            id: meta.id,
419            title: format!("{title} — index"),
420            html,
421            char_count,
422            approx_tokens: approx_tokens(char_count),
423            unit_frags: Vec::new(),
424            is_index: true,
425        });
426    }
427
428    // Body segments
429    for (seg_i, (idxs, _)) in packs.iter().enumerate() {
430        let mut body = String::new();
431        let mut frags = Vec::new();
432        // Page markers when present
433        let mut last_page: Option<u32> = None;
434        for &ui in idxs {
435            let u = &units[ui];
436            if let Some(p) = u.page {
437                if last_page != Some(p) {
438                    last_page = Some(p);
439                    body.push_str(&format!(
440                        "<div class=\"meta\" typeof=\"cof:Page\" property=\"cof:hasPage\" \
441                         resource=\"{doc_uri}#page-{p}\" content=\"{p}\">\
442                         page <span property=\"cof:pageNumber\">{p}</span></div>\n"
443                    ));
444                }
445            }
446            body.push_str(&render_unit_fragment(doc_uri, u, style));
447            frags.push(u.frag.clone());
448        }
449        let idx = (seg_i as u32) + 1;
450        let meta = CofSegmentMeta {
451            index: idx,
452            total,
453            id: segment_ids[seg_i + 1].clone(),
454            prev_id: Some(segment_ids[seg_i].clone()),
455            next_id: segment_ids.get(seg_i + 2).cloned(),
456            is_index: false,
457        };
458        let html = wrap_document(doc_uri, title, &meta, &body, style);
459        let char_count = html.len();
460        segments.push(CofSegment {
461            index: idx,
462            total,
463            id: meta.id,
464            title: format!("{title} — segment {idx}"),
465            html,
466            char_count,
467            approx_tokens: approx_tokens(char_count),
468            unit_frags: frags,
469            is_index: false,
470        });
471    }
472
473    let total_chars: usize = segments.iter().map(|s| s.char_count).sum();
474    CofPackage {
475        document_uri: doc_uri.into(),
476        title: title.into(),
477        profile: COF_PROFILE.into(),
478        segment_max_chars: max_chars,
479        total_chars,
480        total_approx_tokens: approx_tokens(total_chars),
481        segments,
482    }
483}
484
485/// Single-document COF (no multi-segment) — still valid html-rdfa-1.
486pub fn render_cof_document(
487    doc_uri: &str,
488    title: &str,
489    units: &[ContextUnit],
490    style: CofStyle,
491) -> String {
492    let pkg = build_cof_package(doc_uri, title, units, usize::MAX / 4, style);
493    // Prefer body-only when one pack; else concatenate is wrong — return first body or index+first.
494    if pkg.segments.len() >= 2 {
495        pkg.segments[1].html.clone()
496    } else {
497        pkg.segments[0].html.clone()
498    }
499}
500
501#[cfg(test)]
502mod tests {
503    use super::*;
504
505    fn sample_units(n: usize, body_len: usize) -> Vec<ContextUnit> {
506        (0..n)
507            .map(|i| ContextUnit {
508                frag: format!("sec-{i}"),
509                kind: "section".into(),
510                label: format!("{i} Sample section"),
511                text: format!(
512                    "Section {i}. The controller shall not process personal data without consent. {}",
513                    "word ".repeat(body_len / 5)
514                ),
515                page: Some((i as u32 / 3) + 1),
516                parent: None,
517            })
518            .collect()
519    }
520
521    #[test]
522    fn cof_profile_and_rdfa_attributes_present() {
523        let units = sample_units(2, 40);
524        let html = render_cof_document("urn:doc:t", "Test Act", &units, CofStyle::AgentLean);
525        assert!(html.contains(COF_PROFILE));
526        assert!(
527            html.contains("typeof=\"cof:Document\"")
528                || html.contains("typeof=\"cml:Concept cof:Section\"")
529        );
530        assert!(html.contains("cof:hasSection") || html.contains("property=\"cof:hasSection\""));
531        assert!(html.contains("values:originalText") || html.contains("cof:Claim"));
532        assert!(html.contains("cml:Proposed"));
533        assert!(!html.contains("<style>")); // agent-lean
534    }
535
536    #[test]
537    fn large_doc_segments_for_token_budget() {
538        let units = sample_units(20, 2000);
539        let pkg = build_cof_package("urn:doc:big", "Big Act", &units, 8_000, CofStyle::AgentLean);
540        assert!(
541            pkg.segments.len() >= 3,
542            "index + ≥2 body segs, got {}",
543            pkg.segments.len()
544        );
545        assert!(pkg.segments[0].is_index);
546        // Each body segment under soft ceiling (markup can exceed pack estimate slightly).
547        for s in pkg.segments.iter().filter(|s| !s.is_index) {
548            assert!(
549                s.char_count < 40_000,
550                "segment {} too large: {}",
551                s.index,
552                s.char_count
553            );
554            assert!(!s.unit_frags.is_empty());
555            assert!(s.html.contains("cof-segment"));
556            assert!(s.html.contains("cof:prevSegment") || s.index == 1);
557        }
558        // Index lists segments without dumping full bodies.
559        assert!(pkg.segments[0].html.contains("Index"));
560        assert!(pkg.segments[0].html.len() < pkg.total_chars);
561    }
562
563    #[test]
564    fn dual_surface_has_minimal_css() {
565        let units = sample_units(1, 20);
566        let html = render_cof_document("urn:doc:h", "Human", &units, CofStyle::DualSurface);
567        assert!(html.contains("<style>"));
568        assert!(html.contains("cml:Proposed"));
569    }
570}
qualia_client_core/wellfair/cml_context/cof_html.rs

qualia_client_core/wellfair/cml_context/
cof_html.rs
