Skip to main content

qualia_client_core/wellfair/
legislation_ingest.rs

1//! Native legislation structure parse → hypermedia Library ingest.
2//!
3//! Ports the structural (no-LLM) path of `tools/legislation-etl/legis2cml.py`:
4//! Part / Division / Schedule / Section / Subsection decomposition, then seeds
5//! each provision as a findable Library entry under **Work** with purpose
6//! `legislation`. Full section body text is always stored (the Python ETL used
7//! to drop bodies from N3/JSON-LD — this path does not).
8//!
9//! PDF → text uses `pdf_extract` (already a client-core dependency).
10
11use std::collections::BTreeMap;
12use std::path::Path;
13
14use regex::Regex;
15use serde::{Deserialize, Serialize};
16
17use super::cml_context::{
18    build_cof_package, build_document_context, CofStyle, ContextUnit, DEFAULT_SEGMENT_MAX_CHARS,
19    MEDIA_TYPE_COF,
20};
21use super::hypermedia_store::{CommonsVisibility, HypermediaStore, LibraryEntry, LibrarySection};
22
23/// Media type for native legislation provision entries.
24pub const LEGISLATION_MEDIA_TYPE: &str = "text/x-legislation-provision";
25pub const LEGISLATION_INSTRUMENT_MEDIA: &str = "text/x-legislation-instrument";
26
27#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
28pub struct Provision {
29    pub frag: String,
30    /// part | division | schedule | section | subsection
31    pub kind: String,
32    pub number: String,
33    pub heading: String,
34    pub text: String,
35    /// Pre-subsection-split body (containers keep full text).
36    pub full_text: String,
37    pub start_page: u32,
38    pub parent: Option<String>,
39}
40
41impl Provision {
42    /// Source text.
43    pub fn source_text(&self) -> &str {
44        if !self.full_text.trim().is_empty() {
45            &self.full_text
46        } else {
47            &self.text
48        }
49    }
50}
51
52#[derive(Debug, Clone, Serialize, Deserialize)]
53pub struct LegislationInstrument {
54    pub title: String,
55    pub slug: String,
56    pub jurisdiction: String,
57    pub register_id: Option<String>,
58    pub provisions: Vec<Provision>,
59    pub pages: usize,
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize)]
63pub struct LegislationIngestReport {
64    pub title: String,
65    pub slug: String,
66    pub sections: usize,
67    pub subsections: usize,
68    pub structural: usize,
69    pub concepts_with_text: usize,
70    pub empty_text: usize,
71    pub library_entries_written: usize,
72    pub coverage_ok: bool,
73    /// Proposed CML concepts across all provisions.
74    pub cml_concepts: usize,
75    pub cml_deontic_norms: usize,
76    pub cml_privacy_hits: usize,
77    pub cml_rights_hits: usize,
78    /// COF HTML+RDFa segments (index + body packs for token budgets).
79    pub cof_segments: usize,
80    pub cof_approx_tokens: usize,
81    pub cof_profile: String,
82}
83
84fn slugify(s: &str) -> String {
85    let lower = s.to_ascii_lowercase();
86    let mut out = String::with_capacity(lower.len());
87    let mut prev_dash = false;
88    for c in lower.chars() {
89        if c.is_ascii_alphanumeric() {
90            out.push(c);
91            prev_dash = false;
92        } else if !prev_dash {
93            out.push('-');
94            prev_dash = true;
95        }
96    }
97    out.trim_matches('-').to_string()
98}
99
100fn looks_like_section_heading(title: &str) -> bool {
101    let t = title.trim();
102    if t.len() < 2 {
103        return false;
104    }
105    let first = t.chars().next().unwrap_or(' ');
106    if first.is_lowercase() {
107        return false;
108    }
109    let lower = t.to_ascii_lowercase();
110    for bad in [
111        "of ",
112        "or ",
113        "and ",
114        "to ",
115        "in ",
116        "for ",
117        "as ",
118        "under ",
119        "made by",
120        "has no effect",
121    ] {
122        if lower.starts_with(bad) {
123            return false;
124        }
125    }
126    if Regex::new(r"^\d{4}$").unwrap().is_match(t) {
127        return false;
128    }
129    let words: Vec<_> = t.split_whitespace().collect();
130    if words.len() > 12 {
131        return false;
132    }
133    if t.ends_with('.') && words.len() > 4 {
134        return false;
135    }
136    if Regex::new(r"^(If|Subject|This|When|Where|Unless|Despite|For the purposes)\b")
137        .unwrap()
138        .is_match(t)
139    {
140        return false;
141    }
142    true
143}
144
145/// Clean PDF page text into lines (mirrors legis2cml.clean lightly).
146fn clean_page(raw: &str) -> String {
147    raw.replace('\r', "\n")
148        .replace('\u{00a0}', " ")
149        .lines()
150        .map(|l| l.trim_end())
151        .collect::<Vec<_>>()
152        .join("\n")
153}
154
155/// Parse legislation pages into structured provisions (AU-oriented; EU Article/Chapter supported).
156pub fn parse_pages(pages: &[(u32, String)], title_hint: Option<&str>) -> LegislationInstrument {
157    let re_part = Regex::new(r"^(Part|PART)\s+([0-9IVXLC]+[A-Za-z]?)\b[\s—\-:.]*(.*)$").unwrap();
158    let re_div = Regex::new(r"^(Division|DIVISION)\s+([0-9]+[A-Za-z]?)\b[\s—\-:.]*(.*)$").unwrap();
159    let re_schedule =
160        Regex::new(r"^(Schedule|SCHEDULE)\s+([0-9]+[A-Za-z]?)\s*[—–]\s*(.+)$").unwrap();
161    let re_section = Regex::new(r"^(\d+[A-Z]{0,2})\s+([A-Z0-9][^\n]{1,160})$").unwrap();
162    let re_hist = Regex::new(r"^(\d+[A-Z]{0,2})\.\s*[—–\-]?\s*([A-Z].{0,100})$").unwrap();
163    let re_eu_chapter = Regex::new(r"(?i)^CHAPTER\s+([IVXLC]+)$").unwrap();
164    let re_eu_article = Regex::new(r"(?i)^Article\s+(\d+[A-Z]?)$").unwrap();
165    let re_enacting = Regex::new(r"(?i)Parliament of Australia enacts|BE IT ENACTED").unwrap();
166
167    let mut frag_counts: BTreeMap<String, u32> = BTreeMap::new();
168    let mut unique_frag = |base: String| -> String {
169        let n = frag_counts.entry(base.clone()).or_insert(0);
170        *n += 1;
171        if *n == 1 {
172            base
173        } else {
174            format!("{base}-{n}")
175        }
176    };
177
178    let cleaned_all: Vec<String> = pages
179        .iter()
180        .flat_map(|(_, raw)| {
181            clean_page(raw)
182                .lines()
183                .map(|s| s.trim().to_string())
184                .collect::<Vec<_>>()
185        })
186        .collect();
187    let has_enacting = cleaned_all.iter().any(|ln| re_enacting.is_match(ln));
188    let has_eu_articles = cleaned_all.iter().any(|ln| re_eu_article.is_match(ln));
189    let mut in_body = !(has_enacting || has_eu_articles);
190
191    let mut title = title_hint.unwrap_or("").to_string();
192    if title.is_empty() {
193        title = infer_title(pages).unwrap_or_else(|| "Untitled Legislative Instrument".into());
194    }
195
196    let mut provisions: Vec<Provision> = Vec::new();
197    let mut cur: Option<Provision> = None;
198    let mut buf: Vec<String> = Vec::new();
199    let mut current_schedule: Option<String> = None;
200
201    let flush =
202        |cur: &mut Option<Provision>, buf: &mut Vec<String>, provisions: &mut Vec<Provision>| {
203            if let Some(mut p) = cur.take() {
204                p.text = buf.join("\n").trim().to_string();
205                p.full_text = p.text.clone();
206                provisions.push(p);
207            }
208            buf.clear();
209        };
210
211    let is_heading_title = |head: &str| {
212        let h = head.trim();
213        h.is_empty() || !h.chars().next().map(|c| c.is_lowercase()).unwrap_or(false)
214    };
215
216    for (page_no, raw) in pages {
217        for ln in clean_page(raw).lines() {
218            let s = ln.trim();
219            if title.is_empty()
220                && !Regex::new(r"^[\d(]").unwrap().is_match(s)
221                && Regex::new(r"(?i)\bAct\s+(No\.\s*\d+\s+of\s+)?\d{4}\b")
222                    .unwrap()
223                    .is_match(s)
224            {
225                title = s.to_string();
226            }
227            if !in_body {
228                if re_enacting.is_match(s) {
229                    in_body = true;
230                    continue;
231                }
232                if re_eu_chapter.is_match(s) || re_eu_article.is_match(s) {
233                    in_body = true;
234                } else {
235                    continue;
236                }
237            }
238            if s.is_empty() {
239                if cur.is_some() {
240                    buf.push(String::new());
241                }
242                continue;
243            }
244
245            if let Some(m) = re_eu_chapter.captures(s) {
246                flush(&mut cur, &mut buf, &mut provisions);
247                current_schedule = None;
248                let number = m.get(1).unwrap().as_str().to_ascii_uppercase();
249                provisions.push(Provision {
250                    frag: unique_frag(format!("chapter-{}", slugify(&number))),
251                    kind: "part".into(),
252                    number: number.clone(),
253                    heading: format!("Chapter {number}"),
254                    text: String::new(),
255                    full_text: String::new(),
256                    start_page: *page_no,
257                    parent: None,
258                });
259                continue;
260            }
261            if let Some(m) = re_eu_article.captures(s) {
262                flush(&mut cur, &mut buf, &mut provisions);
263                let number = m.get(1).unwrap().as_str().to_ascii_uppercase();
264                cur = Some(Provision {
265                    frag: unique_frag(format!("article-{}", slugify(&number))),
266                    kind: "section".into(),
267                    number,
268                    heading: String::new(),
269                    text: String::new(),
270                    full_text: String::new(),
271                    start_page: *page_no,
272                    parent: None,
273                });
274                // next non-empty line becomes heading if short — append as body for simplicity
275                continue;
276            }
277            if let Some(m) = re_schedule.captures(s) {
278                let sched = slugify(m.get(2).unwrap().as_str());
279                if current_schedule.as_deref() != Some(&sched) {
280                    flush(&mut cur, &mut buf, &mut provisions);
281                    current_schedule = Some(sched.clone());
282                    provisions.push(Provision {
283                        frag: unique_frag(format!("sch-{sched}")),
284                        kind: "schedule".into(),
285                        number: m.get(2).unwrap().as_str().into(),
286                        heading: m.get(3).unwrap().as_str().trim().into(),
287                        text: String::new(),
288                        full_text: String::new(),
289                        start_page: *page_no,
290                        parent: None,
291                    });
292                }
293                continue;
294            }
295            if let Some(m) = re_part.captures(s) {
296                if is_heading_title(m.get(3).map(|x| x.as_str()).unwrap_or("")) {
297                    flush(&mut cur, &mut buf, &mut provisions);
298                    let num = m.get(2).unwrap().as_str();
299                    let head = m.get(3).unwrap().as_str().trim();
300                    let base = if let Some(sch) = &current_schedule {
301                        format!("sch-{sch}-part-{}", slugify(num))
302                    } else {
303                        format!("part-{}", slugify(num))
304                    };
305                    provisions.push(Provision {
306                        frag: unique_frag(base),
307                        kind: "part".into(),
308                        number: num.into(),
309                        heading: if head.is_empty() {
310                            format!("Part {num}")
311                        } else {
312                            head.into()
313                        },
314                        text: String::new(),
315                        full_text: String::new(),
316                        start_page: *page_no,
317                        parent: None,
318                    });
319                    continue;
320                }
321            }
322            if let Some(m) = re_div.captures(s) {
323                if is_heading_title(m.get(3).map(|x| x.as_str()).unwrap_or("")) {
324                    flush(&mut cur, &mut buf, &mut provisions);
325                    let num = m.get(2).unwrap().as_str();
326                    let head = m.get(3).unwrap().as_str().trim();
327                    let base = if let Some(sch) = &current_schedule {
328                        format!("sch-{sch}-div-{}", slugify(num))
329                    } else {
330                        format!("div-{}", slugify(num))
331                    };
332                    provisions.push(Provision {
333                        frag: unique_frag(base),
334                        kind: "division".into(),
335                        number: num.into(),
336                        heading: if head.is_empty() {
337                            format!("Division {num}")
338                        } else {
339                            head.into()
340                        },
341                        text: String::new(),
342                        full_text: String::new(),
343                        start_page: *page_no,
344                        parent: None,
345                    });
346                    continue;
347                }
348            }
349            if let Some(m) = re_section.captures(s) {
350                let head = m.get(2).unwrap().as_str();
351                if !s.starts_with('(') && looks_like_section_heading(head) {
352                    flush(&mut cur, &mut buf, &mut provisions);
353                    let num = m.get(1).unwrap().as_str();
354                    let base = if let Some(sch) = &current_schedule {
355                        format!("sch-{sch}-sec-{}", slugify(num))
356                    } else {
357                        format!("sec-{}", slugify(num))
358                    };
359                    cur = Some(Provision {
360                        frag: unique_frag(base),
361                        kind: "section".into(),
362                        number: num.into(),
363                        heading: head.trim().into(),
364                        text: String::new(),
365                        full_text: String::new(),
366                        start_page: *page_no,
367                        parent: None,
368                    });
369                    continue;
370                }
371            }
372            if !has_eu_articles {
373                if let Some(m) = re_hist.captures(s) {
374                    let head = m.get(2).unwrap().as_str();
375                    if !s.starts_with('(')
376                        && looks_like_section_heading(head)
377                        && re_section.captures(s).is_none()
378                    {
379                        flush(&mut cur, &mut buf, &mut provisions);
380                        let num = m.get(1).unwrap().as_str();
381                        let base = if let Some(sch) = &current_schedule {
382                            format!("sch-{sch}-sec-{}", slugify(num))
383                        } else {
384                            format!("sec-{}", slugify(num))
385                        };
386                        let heading = head.trim();
387                        if heading.len() > 90 || heading.starts_with(|c: char| c.is_lowercase()) {
388                            cur = Some(Provision {
389                                frag: unique_frag(base),
390                                kind: "section".into(),
391                                number: num.into(),
392                                heading: format!("Section {num}"),
393                                text: String::new(),
394                                full_text: String::new(),
395                                start_page: *page_no,
396                                parent: None,
397                            });
398                            buf.push(heading.into());
399                        } else {
400                            cur = Some(Provision {
401                                frag: unique_frag(base),
402                                kind: "section".into(),
403                                number: num.into(),
404                                heading: heading.into(),
405                                text: String::new(),
406                                full_text: String::new(),
407                                start_page: *page_no,
408                                parent: None,
409                            });
410                        }
411                        continue;
412                    }
413                }
414            }
415            if let Some(c) = cur.as_mut() {
416                // EU article: first line after "Article N" is often the short title.
417                if c.heading.is_empty()
418                    && s.len() < 120
419                    && !s.chars().next().unwrap_or(' ').is_ascii_digit()
420                {
421                    c.heading = s.to_string();
422                } else {
423                    buf.push(s.to_string());
424                }
425            }
426        }
427    }
428    flush(&mut cur, &mut buf, &mut provisions);
429
430    // Subsection split
431    provisions = decompose_provisions(provisions);
432
433    let slug = {
434        let mut s = slugify(
435            &Regex::new(r"(?i)\s*No\.\s*\d+.*$")
436                .unwrap()
437                .replace(&title, ""),
438        );
439        let parts: Vec<_> = s.split('-').take(12).collect();
440        s = parts.join("-");
441        if s.len() < 3 || s.len() > 90 {
442            s = slugify(title_hint.unwrap_or("instrument"));
443        }
444        s
445    };
446
447    LegislationInstrument {
448        title,
449        slug,
450        jurisdiction: "AU".into(),
451        register_id: None,
452        provisions,
453        pages: pages.len(),
454    }
455}
456
457fn decompose_provisions(mut provisions: Vec<Provision>) -> Vec<Provision> {
458    let re_sub = Regex::new(r"^\((\d+[A-Za-z]?)\)\s+(.+)$").unwrap();
459    let mut out = Vec::new();
460    for mut section in provisions.drain(..) {
461        if section.kind == "section" {
462            if section.full_text.is_empty() {
463                section.full_text = section.text.clone();
464            }
465            let lines: Vec<&str> = section.text.lines().collect();
466            let starts: Vec<usize> = lines
467                .iter()
468                .enumerate()
469                .filter_map(|(i, ln)| re_sub.is_match(ln).then_some(i))
470                .collect();
471            if starts.len() >= 2 {
472                let mut subs = Vec::new();
473                for (k, &start) in starts.iter().enumerate() {
474                    let end = starts.get(k + 1).copied().unwrap_or(lines.len());
475                    let block = lines[start..end].join("\n").trim().to_string();
476                    let num = re_sub
477                        .captures(lines[start])
478                        .and_then(|c| c.get(1).map(|m| m.as_str().to_string()))
479                        .unwrap_or_else(|| format!("{k}"));
480                    let frag = format!("{}-ss-{}", section.frag, slugify(&num));
481                    subs.push(Provision {
482                        frag,
483                        kind: "subsection".into(),
484                        number: format!("{}({num})", section.number),
485                        heading: section.heading.clone(),
486                        text: block.clone(),
487                        full_text: block,
488                        start_page: section.start_page,
489                        parent: Some(section.frag.clone()),
490                    });
491                }
492                let lead = if starts[0] > 0 {
493                    lines[..starts[0]].join("\n").trim().to_string()
494                } else {
495                    String::new()
496                };
497                section.text = lead;
498                out.push(section);
499                out.extend(subs);
500                continue;
501            }
502        }
503        out.push(section);
504    }
505    out
506}
507
508fn infer_title(pages: &[(u32, String)]) -> Option<String> {
509    let joined = pages
510        .iter()
511        .take(5)
512        .map(|(_, r)| r.as_str())
513        .collect::<Vec<_>>()
514        .join(" ");
515    let joined = Regex::new(r"\s+").unwrap().replace_all(&joined, " ");
516    if let Some(c) = Regex::new(r"(?i)may be cited as the\s+(.{2,120}?\bAct\b[^.]{0,40}?\d{4})\b")
517        .unwrap()
518        .captures(&joined)
519    {
520        return Some(c.get(1).unwrap().as_str().trim().to_string());
521    }
522    None
523}
524
525/// Extract text pages from a PDF file.
526pub fn extract_pdf_pages(path: &Path) -> Result<Vec<(u32, String)>, String> {
527    let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
528    extract_pdf_pages_from_bytes(&bytes)
529}
530
531/// Extracts PDF pages from bytes.
532pub fn extract_pdf_pages_from_bytes(bytes: &[u8]) -> Result<Vec<(u32, String)>, String> {
533    // pdf_extract gives whole-doc text; split on form feeds if present, else one page.
534    let text = pdf_extract::extract_text_from_mem(bytes).map_err(|e| e.to_string())?;
535    let mut pages = Vec::new();
536    if text.contains('\u{c}') {
537        for (i, part) in text.split('\u{c}').enumerate() {
538            pages.push((i as u32 + 1, part.to_string()));
539        }
540    } else {
541        // Heuristic: split every ~3500 chars for page-ish markers when no form feed.
542        let chunk = 3500usize;
543        if text.len() <= chunk {
544            pages.push((1, text));
545        } else {
546            let mut i = 0usize;
547            let mut page = 1u32;
548            while i < text.len() {
549                let end = (i + chunk).min(text.len());
550                let mut cut = end;
551                if end < text.len() {
552                    if let Some(rel) = text[i..end].rfind('\n') {
553                        cut = i + rel + 1;
554                    }
555                }
556                if cut <= i {
557                    cut = end;
558                }
559                pages.push((page, text[i..cut].to_string()));
560                i = cut;
561                page += 1;
562            }
563        }
564    }
565    if pages.is_empty() {
566        pages.push((1, String::new()));
567    }
568    Ok(pages)
569}
570
571fn fnv60(bytes: &[u8]) -> u64 {
572    const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
573    const FNV_PRIME: u64 = 0x100_0000_01b3;
574    let mut h = FNV_OFFSET;
575    for b in bytes {
576        h ^= u64::from(*b);
577        h = h.wrapping_mul(FNV_PRIME);
578    }
579    h & 0x0FFF_FFFF_FFFF_FFFF
580}
581
582/// Seed a parsed instrument into the hypermedia library (instrument + every provision).
583pub fn seed_instrument_into_library(
584    store: &HypermediaStore,
585    inst: &LegislationInstrument,
586    now: u64,
587) -> std::io::Result<LegislationIngestReport> {
588    let mut entries = store.load()?;
589    let mut by_uri: std::collections::HashMap<String, usize> = entries
590        .iter()
591        .enumerate()
592        .map(|(i, e)| (e.asset_uri.clone(), i))
593        .collect();
594
595    let base = format!(
596        "legislation://{}/{}",
597        inst.jurisdiction.to_ascii_lowercase(),
598        inst.register_id.as_deref().unwrap_or(inst.slug.as_str())
599    );
600
601    let mut written = 0usize;
602    let mut upsert = |entry: LibraryEntry| {
603        if let Some(&idx) = by_uri.get(&entry.asset_uri) {
604            let mut e = entry;
605            e.ingested_unix = entries[idx].ingested_unix;
606            entries[idx] = e;
607        } else {
608            by_uri.insert(entry.asset_uri.clone(), entries.len());
609            entries.push(entry);
610        }
611        written += 1;
612    };
613
614    // Build instrument-wide CML context from all provisions (structure + deontic/privacy/rights).
615    let cml_units: Vec<ContextUnit> = inst
616        .provisions
617        .iter()
618        .map(|p| ContextUnit {
619            frag: p.frag.clone(),
620            kind: p.kind.clone(),
621            label: format!("{} {}", p.number, p.heading).trim().to_string(),
622            text: p.source_text().to_string(),
623            page: Some(p.start_page),
624            parent: p.parent.clone(),
625        })
626        .collect();
627    let instrument_graph = build_document_context(&base, &inst.title, &cml_units);
628    // Agent-lean COF package: index + section-aligned body segments under a char budget.
629    let cof_pkg = build_cof_package(
630        &base,
631        &inst.title,
632        &cml_units,
633        DEFAULT_SEGMENT_MAX_CHARS,
634        CofStyle::AgentLean,
635    );
636
637    // Instrument root.
638    let root_uri = base.clone();
639    let root_excerpt = format!(
640        "{} — {} provision(s), {} page(s). CML (cml:Proposed): {} concepts, {} deontic, {} privacy. COF: {} segment(s), ~{} tokens total (load index + one body for agent turns).",
641        inst.title,
642        inst.provisions.len(),
643        inst.pages,
644        instrument_graph.concepts.len(),
645        instrument_graph.deontic_norms,
646        instrument_graph.privacy_hits,
647        cof_pkg.segments.len(),
648        cof_pkg.total_approx_tokens,
649    );
650    let mut root_topics = vec![
651        "legislation".into(),
652        "statute".into(),
653        "cml".into(),
654        inst.jurisdiction.to_ascii_lowercase(),
655        inst.slug.clone(),
656    ];
657    root_topics.extend(instrument_graph.topics.iter().cloned());
658    root_topics.sort();
659    root_topics.dedup();
660    let mut root_purposes = vec!["legislation".into(), "legal".into(), "work".into()];
661    root_purposes.extend(instrument_graph.purposes.iter().cloned());
662    root_purposes.sort();
663    root_purposes.dedup();
664    let root_n3 = if instrument_graph.n3.len() > 64_000 {
665        format!(
666            "{}…\n# [instrument cml_n3 truncated — per-provision entries hold local graphs]",
667            &instrument_graph.n3[..64_000]
668        )
669    } else {
670        instrument_graph.n3.clone()
671    };
672    let mut root = LibraryEntry {
673        asset_uri: root_uri.clone(),
674        primary_subject: fnv60(root_uri.as_bytes()),
675        media_type: LEGISLATION_INSTRUMENT_MEDIA.into(),
676        quins: instrument_graph.quins.clone(),
677        topics: root_topics,
678        projects: vec![format!("legislation:{}", inst.slug)],
679        purposes: root_purposes,
680        place: None,
681        occurred_at: None,
682        lat: None,
683        lon: None,
684        flags: Vec::new(),
685        ingested_unix: now,
686        excerpt: root_excerpt.chars().take(400).collect(),
687        sensitivity: "public".into(),
688        section: LibrarySection::Work.as_str().into(),
689        commons_visibility: CommonsVisibility::None,
690        cml_signals: instrument_graph.signal_tags.clone(),
691        cml_concept_count: instrument_graph.concepts.len() as u32,
692        cml_n3: root_n3,
693        cof_html: cof_pkg
694            .segments
695            .iter()
696            .find(|s| s.is_index)
697            .map(|s| s.html.clone())
698            .unwrap_or_default(),
699        cof_segment_count: cof_pkg.segments.len() as u32,
700        cof_segment_index: 0,
701        cof_profile: cof_pkg.profile.clone(),
702    };
703    root.recompute_section();
704    upsert(root);
705
706    // Persist each COF body segment as its own library row (token-selective load).
707    for seg in cof_pkg.segments.iter().filter(|s| !s.is_index) {
708        let seg_uri = format!("{base}#cof-seg-{}", seg.index);
709        let mut se = LibraryEntry {
710            asset_uri: seg_uri.clone(),
711            primary_subject: fnv60(seg_uri.as_bytes()),
712            media_type: MEDIA_TYPE_COF.into(),
713            quins: Vec::new(),
714            topics: vec![
715                "legislation".into(),
716                "cof".into(),
717                "cml".into(),
718                inst.slug.clone(),
719                format!("cof-seg-{}", seg.index),
720            ],
721            projects: vec![format!("legislation:{}", inst.slug)],
722            purposes: vec!["legislation".into(), "legal".into(), "work".into()],
723            place: None,
724            occurred_at: None,
725            lat: None,
726            lon: None,
727            flags: Vec::new(),
728            ingested_unix: now,
729            excerpt: format!(
730                "COF body segment {}/{} · ~{} tokens · frags: {}",
731                seg.index + 1,
732                seg.total,
733                seg.approx_tokens,
734                seg.unit_frags
735                    .iter()
736                    .take(12)
737                    .cloned()
738                    .collect::<Vec<_>>()
739                    .join(", ")
740            ),
741            sensitivity: "public".into(),
742            section: LibrarySection::Work.as_str().into(),
743            commons_visibility: CommonsVisibility::None,
744            cml_signals: Vec::new(),
745            cml_concept_count: seg.unit_frags.len() as u32,
746            cml_n3: String::new(),
747            cof_html: seg.html.clone(),
748            cof_segment_count: cof_pkg.segments.len() as u32,
749            cof_segment_index: seg.index,
750            cof_profile: cof_pkg.profile.clone(),
751        };
752        se.recompute_section();
753        upsert(se);
754    }
755
756    let mut sections = 0usize;
757    let mut subsections = 0usize;
758    let mut structural = 0usize;
759    let mut with_text = 0usize;
760    let mut empty = 0usize;
761    let mut total_cml_concepts = instrument_graph.concepts.len();
762    let mut total_deontic = instrument_graph.deontic_norms;
763    let mut total_privacy = instrument_graph.privacy_hits;
764    let mut total_rights = instrument_graph.rights_hits;
765
766    for p in &inst.provisions {
767        match p.kind.as_str() {
768            "section" => sections += 1,
769            "subsection" => subsections += 1,
770            "part" | "division" | "schedule" => structural += 1,
771            _ => {}
772        }
773        let body = p.source_text();
774        if body.trim().is_empty() {
775            empty += 1;
776        } else {
777            with_text += 1;
778        }
779        if matches!(
780            p.kind.as_str(),
781            "section" | "subsection" | "part" | "division" | "schedule"
782        ) {
783            let uri = format!("{base}#{}", p.frag);
784            let label = format!("{} {}", p.number, p.heading).trim().to_string();
785            let unit = ContextUnit {
786                frag: p.frag.clone(),
787                kind: p.kind.clone(),
788                label: label.clone(),
789                text: body.to_string(),
790                page: Some(p.start_page),
791                parent: p.parent.clone(),
792            };
793            let g = build_document_context(&uri, &label, &[unit]);
794            total_cml_concepts += g.concepts.len();
795            total_deontic += g.deontic_norms;
796            total_privacy += g.privacy_hits;
797            total_rights += g.rights_hits;
798
799            let mut topics = vec![
800                "legislation".into(),
801                "cml".into(),
802                p.kind.clone(),
803                inst.slug.clone(),
804                format!("s{}", p.number),
805            ];
806            if let Some(parent) = &p.parent {
807                topics.push(parent.clone());
808            }
809            topics.extend(g.topics.iter().cloned());
810            topics.sort();
811            topics.dedup();
812
813            let mut purposes = vec!["legislation".into(), "legal".into(), "work".into()];
814            purposes.extend(g.purposes.iter().cloned());
815            purposes.sort();
816            purposes.dedup();
817
818            let mut excerpt = if body.trim().is_empty() {
819                format!("{label} (no body text extracted)")
820            } else {
821                format!("{label}\n\n{body}")
822            };
823            if excerpt.len() > 12_000 {
824                excerpt = excerpt.chars().take(12_000).collect();
825                excerpt.push_str("\n…[truncated]");
826            }
827            // Prefix with signal chips for UI glance.
828            if !g.signal_tags.is_empty() {
829                let chips: String = g
830                    .signal_tags
831                    .iter()
832                    .take(8)
833                    .cloned()
834                    .collect::<Vec<_>>()
835                    .join(" · ");
836                excerpt = format!("[{chips}]\n{excerpt}");
837            }
838
839            let cml_n3 = if g.n3.len() > 24_000 {
840                format!("{}…\n# [truncated]", &g.n3[..24_000])
841            } else {
842                g.n3
843            };
844
845            let mut entry = LibraryEntry {
846                asset_uri: uri.clone(),
847                primary_subject: fnv60(uri.as_bytes()),
848                media_type: LEGISLATION_MEDIA_TYPE.into(),
849                quins: g.quins,
850                topics,
851                projects: vec![format!("legislation:{}", inst.slug)],
852                purposes,
853                place: None,
854                occurred_at: None,
855                lat: None,
856                lon: None,
857                flags: Vec::new(),
858                ingested_unix: now,
859                excerpt,
860                sensitivity: "public".into(),
861                section: LibrarySection::Work.as_str().into(),
862                commons_visibility: CommonsVisibility::None,
863                cml_signals: g.signal_tags,
864                cml_concept_count: g.concepts.len() as u32,
865                cml_n3,
866                cof_html: String::new(),
867                cof_segment_count: cof_pkg.segments.len() as u32,
868                cof_segment_index: 0,
869                cof_profile: cof_pkg.profile.clone(),
870            };
871            entry.recompute_section();
872            upsert(entry);
873        }
874    }
875
876    store.replace_all(&entries)?;
877
878    let concepts = sections + subsections;
879    let coverage_ok = empty == 0 || (with_text as f64 / concepts.max(1) as f64) >= 0.85;
880
881    Ok(LegislationIngestReport {
882        title: inst.title.clone(),
883        slug: inst.slug.clone(),
884        sections,
885        subsections,
886        structural,
887        concepts_with_text: with_text,
888        empty_text: empty,
889        library_entries_written: written,
890        coverage_ok,
891        cml_concepts: total_cml_concepts,
892        cml_deontic_norms: total_deontic,
893        cml_privacy_hits: total_privacy,
894        cml_rights_hits: total_rights,
895        cof_segments: cof_pkg.segments.len(),
896        cof_approx_tokens: cof_pkg.total_approx_tokens,
897        cof_profile: cof_pkg.profile,
898    })
899}
900
901/// Parse PDF bytes and seed the library.
902pub fn ingest_legislation_pdf_bytes(
903    store: &HypermediaStore,
904    bytes: &[u8],
905    register_id: Option<&str>,
906    jurisdiction: &str,
907    title_hint: Option<&str>,
908) -> Result<LegislationIngestReport, String> {
909    let pages = extract_pdf_pages_from_bytes(bytes)?;
910    let mut inst = parse_pages(&pages, title_hint);
911    inst.jurisdiction = jurisdiction.to_string();
912    if let Some(id) = register_id {
913        inst.register_id = Some(id.to_string());
914        if inst.slug.len() < 3 {
915            inst.slug = slugify(id);
916        }
917    }
918    let now = std::time::SystemTime::now()
919        .duration_since(std::time::UNIX_EPOCH)
920        .map(|d| d.as_secs())
921        .unwrap_or(0);
922    seed_instrument_into_library(store, &inst, now).map_err(|e| e.to_string())
923}
924
925/// Parse plain text (already extracted) as a single-page instrument and seed.
926pub fn ingest_legislation_text(
927    store: &HypermediaStore,
928    text: &str,
929    register_id: Option<&str>,
930    jurisdiction: &str,
931    title_hint: Option<&str>,
932) -> Result<LegislationIngestReport, String> {
933    let pages = vec![(1u32, text.to_string())];
934    let mut inst = parse_pages(&pages, title_hint);
935    inst.jurisdiction = jurisdiction.to_string();
936    if let Some(id) = register_id {
937        inst.register_id = Some(id.to_string());
938    }
939    let now = std::time::SystemTime::now()
940        .duration_since(std::time::UNIX_EPOCH)
941        .map(|d| d.as_secs())
942        .unwrap_or(0);
943    seed_instrument_into_library(store, &inst, now).map_err(|e| e.to_string())
944}
945
946#[cfg(test)]
947mod tests {
948    use super::*;
949
950    #[test]
951    fn enacts_skips_contents_and_keeps_body_text() {
952        let pages = vec![(
953            1u32,
954            "Example Amendment Act 2004\nNo. 5, 2004\n\
955             Contents\n1  Short title\n2  Commencement\nPart 2—Widgets\n3  Widget duty\n\
956             The Parliament of Australia enacts:\n\
957             1  Short title\nThis Act may be cited as the Example Amendment Act 2004.\n\
958             2  Commencement\nThis Act commences on Royal Assent.\n\
959             3  Widget duty\nA person must not widget.\n"
960                .into(),
961        )];
962        let inst = parse_pages(&pages, Some("Example Amendment Act 2004"));
963        let sections: Vec<_> = inst
964            .provisions
965            .iter()
966            .filter(|p| p.kind == "section")
967            .collect();
968        assert_eq!(sections.len(), 3);
969        assert!(sections[0].source_text().contains("may be cited"));
970        assert!(sections[1].source_text().contains("commences"));
971        assert!(sections[2].source_text().contains("must not widget"));
972        // No phantom Part from contents.
973        assert!(inst.provisions.iter().all(|p| p.kind != "part"));
974    }
975
976    #[test]
977    fn seed_writes_library_entries_with_text() {
978        let dir = tempfile::tempdir().unwrap();
979        let store = HypermediaStore::open(dir.path()).unwrap();
980        let text = "\
981The Parliament of Australia enacts:\n\
9821  Short title\nThis Act may be cited as the Demo Act 2020.\n\
9832  Commencement\nThis Act commences on the day after Royal Assent.\n";
984        let report = ingest_legislation_text(
985            &store,
986            text,
987            Some("C2020A00001"),
988            "AU",
989            Some("Demo Act 2020"),
990        )
991        .unwrap();
992        assert!(report.sections >= 2);
993        assert!(report.concepts_with_text >= 2);
994        assert!(report.coverage_ok);
995        let work = store.by_section(LibrarySection::Work).unwrap();
996        assert!(work.len() >= 3); // instrument + 2 sections
997        assert!(work.iter().any(|e| e.excerpt.contains("may be cited")));
998        assert!(work
999            .iter()
1000            .any(|e| e.purposes.iter().any(|p| p == "legislation")));
1001        // CML context graph attached (deontic/privacy cues from body text when present).
1002        assert!(report.cml_concepts > 0);
1003        assert!(work
1004            .iter()
1005            .any(|e| e.cml_concept_count > 0 || !e.cml_n3.is_empty()));
1006    }
1007
1008    #[test]
1009    fn subsections_preserve_full_text_on_parent() {
1010        let pages = vec![(
1011            1u32,
1012            "The Parliament of Australia enacts:\n\
1013             5  Offence\n\
1014             A person commits an offence if:\n\
1015             (1) the person does X; and\n\
1016             (2) the person does Y.\n"
1017                .into(),
1018        )];
1019        let inst = parse_pages(&pages, Some("Offence Act"));
1020        let sec = inst
1021            .provisions
1022            .iter()
1023            .find(|p| p.kind == "section")
1024            .unwrap();
1025        let subs: Vec<_> = inst
1026            .provisions
1027            .iter()
1028            .filter(|p| p.kind == "subsection")
1029            .collect();
1030        assert_eq!(subs.len(), 2);
1031        assert!(sec.full_text.contains("(1)"));
1032        assert!(sec.full_text.contains("(2)"));
1033        assert!(subs[0].source_text().contains("does X"));
1034    }
1035}