1use std::collections::BTreeMap;
12use std::path::Path;
13
14use regex::Regex;
15use serde::{Deserialize, Serialize};
16
17use super::cml_context::{
18 build_cof_package, build_document_context, CofStyle, ContextUnit, DEFAULT_SEGMENT_MAX_CHARS,
19 MEDIA_TYPE_COF,
20};
21use super::hypermedia_store::{CommonsVisibility, HypermediaStore, LibraryEntry, LibrarySection};
22
23pub const LEGISLATION_MEDIA_TYPE: &str = "text/x-legislation-provision";
25pub const LEGISLATION_INSTRUMENT_MEDIA: &str = "text/x-legislation-instrument";
26
27#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
28pub struct Provision {
29 pub frag: String,
30 pub kind: String,
32 pub number: String,
33 pub heading: String,
34 pub text: String,
35 pub full_text: String,
37 pub start_page: u32,
38 pub parent: Option<String>,
39}
40
41impl Provision {
42 pub fn source_text(&self) -> &str {
44 if !self.full_text.trim().is_empty() {
45 &self.full_text
46 } else {
47 &self.text
48 }
49 }
50}
51
52#[derive(Debug, Clone, Serialize, Deserialize)]
53pub struct LegislationInstrument {
54 pub title: String,
55 pub slug: String,
56 pub jurisdiction: String,
57 pub register_id: Option<String>,
58 pub provisions: Vec<Provision>,
59 pub pages: usize,
60}
61
62#[derive(Debug, Clone, Serialize, Deserialize)]
63pub struct LegislationIngestReport {
64 pub title: String,
65 pub slug: String,
66 pub sections: usize,
67 pub subsections: usize,
68 pub structural: usize,
69 pub concepts_with_text: usize,
70 pub empty_text: usize,
71 pub library_entries_written: usize,
72 pub coverage_ok: bool,
73 pub cml_concepts: usize,
75 pub cml_deontic_norms: usize,
76 pub cml_privacy_hits: usize,
77 pub cml_rights_hits: usize,
78 pub cof_segments: usize,
80 pub cof_approx_tokens: usize,
81 pub cof_profile: String,
82}
83
84fn slugify(s: &str) -> String {
85 let lower = s.to_ascii_lowercase();
86 let mut out = String::with_capacity(lower.len());
87 let mut prev_dash = false;
88 for c in lower.chars() {
89 if c.is_ascii_alphanumeric() {
90 out.push(c);
91 prev_dash = false;
92 } else if !prev_dash {
93 out.push('-');
94 prev_dash = true;
95 }
96 }
97 out.trim_matches('-').to_string()
98}
99
100fn looks_like_section_heading(title: &str) -> bool {
101 let t = title.trim();
102 if t.len() < 2 {
103 return false;
104 }
105 let first = t.chars().next().unwrap_or(' ');
106 if first.is_lowercase() {
107 return false;
108 }
109 let lower = t.to_ascii_lowercase();
110 for bad in [
111 "of ",
112 "or ",
113 "and ",
114 "to ",
115 "in ",
116 "for ",
117 "as ",
118 "under ",
119 "made by",
120 "has no effect",
121 ] {
122 if lower.starts_with(bad) {
123 return false;
124 }
125 }
126 if Regex::new(r"^\d{4}$").unwrap().is_match(t) {
127 return false;
128 }
129 let words: Vec<_> = t.split_whitespace().collect();
130 if words.len() > 12 {
131 return false;
132 }
133 if t.ends_with('.') && words.len() > 4 {
134 return false;
135 }
136 if Regex::new(r"^(If|Subject|This|When|Where|Unless|Despite|For the purposes)\b")
137 .unwrap()
138 .is_match(t)
139 {
140 return false;
141 }
142 true
143}
144
145fn clean_page(raw: &str) -> String {
147 raw.replace('\r', "\n")
148 .replace('\u{00a0}', " ")
149 .lines()
150 .map(|l| l.trim_end())
151 .collect::<Vec<_>>()
152 .join("\n")
153}
154
155pub fn parse_pages(pages: &[(u32, String)], title_hint: Option<&str>) -> LegislationInstrument {
157 let re_part = Regex::new(r"^(Part|PART)\s+([0-9IVXLC]+[A-Za-z]?)\b[\s—\-:.]*(.*)$").unwrap();
158 let re_div = Regex::new(r"^(Division|DIVISION)\s+([0-9]+[A-Za-z]?)\b[\s—\-:.]*(.*)$").unwrap();
159 let re_schedule =
160 Regex::new(r"^(Schedule|SCHEDULE)\s+([0-9]+[A-Za-z]?)\s*[—–]\s*(.+)$").unwrap();
161 let re_section = Regex::new(r"^(\d+[A-Z]{0,2})\s+([A-Z0-9][^\n]{1,160})$").unwrap();
162 let re_hist = Regex::new(r"^(\d+[A-Z]{0,2})\.\s*[—–\-]?\s*([A-Z].{0,100})$").unwrap();
163 let re_eu_chapter = Regex::new(r"(?i)^CHAPTER\s+([IVXLC]+)$").unwrap();
164 let re_eu_article = Regex::new(r"(?i)^Article\s+(\d+[A-Z]?)$").unwrap();
165 let re_enacting = Regex::new(r"(?i)Parliament of Australia enacts|BE IT ENACTED").unwrap();
166
167 let mut frag_counts: BTreeMap<String, u32> = BTreeMap::new();
168 let mut unique_frag = |base: String| -> String {
169 let n = frag_counts.entry(base.clone()).or_insert(0);
170 *n += 1;
171 if *n == 1 {
172 base
173 } else {
174 format!("{base}-{n}")
175 }
176 };
177
178 let cleaned_all: Vec<String> = pages
179 .iter()
180 .flat_map(|(_, raw)| {
181 clean_page(raw)
182 .lines()
183 .map(|s| s.trim().to_string())
184 .collect::<Vec<_>>()
185 })
186 .collect();
187 let has_enacting = cleaned_all.iter().any(|ln| re_enacting.is_match(ln));
188 let has_eu_articles = cleaned_all.iter().any(|ln| re_eu_article.is_match(ln));
189 let mut in_body = !(has_enacting || has_eu_articles);
190
191 let mut title = title_hint.unwrap_or("").to_string();
192 if title.is_empty() {
193 title = infer_title(pages).unwrap_or_else(|| "Untitled Legislative Instrument".into());
194 }
195
196 let mut provisions: Vec<Provision> = Vec::new();
197 let mut cur: Option<Provision> = None;
198 let mut buf: Vec<String> = Vec::new();
199 let mut current_schedule: Option<String> = None;
200
201 let flush =
202 |cur: &mut Option<Provision>, buf: &mut Vec<String>, provisions: &mut Vec<Provision>| {
203 if let Some(mut p) = cur.take() {
204 p.text = buf.join("\n").trim().to_string();
205 p.full_text = p.text.clone();
206 provisions.push(p);
207 }
208 buf.clear();
209 };
210
211 let is_heading_title = |head: &str| {
212 let h = head.trim();
213 h.is_empty() || !h.chars().next().map(|c| c.is_lowercase()).unwrap_or(false)
214 };
215
216 for (page_no, raw) in pages {
217 for ln in clean_page(raw).lines() {
218 let s = ln.trim();
219 if title.is_empty()
220 && !Regex::new(r"^[\d(]").unwrap().is_match(s)
221 && Regex::new(r"(?i)\bAct\s+(No\.\s*\d+\s+of\s+)?\d{4}\b")
222 .unwrap()
223 .is_match(s)
224 {
225 title = s.to_string();
226 }
227 if !in_body {
228 if re_enacting.is_match(s) {
229 in_body = true;
230 continue;
231 }
232 if re_eu_chapter.is_match(s) || re_eu_article.is_match(s) {
233 in_body = true;
234 } else {
235 continue;
236 }
237 }
238 if s.is_empty() {
239 if cur.is_some() {
240 buf.push(String::new());
241 }
242 continue;
243 }
244
245 if let Some(m) = re_eu_chapter.captures(s) {
246 flush(&mut cur, &mut buf, &mut provisions);
247 current_schedule = None;
248 let number = m.get(1).unwrap().as_str().to_ascii_uppercase();
249 provisions.push(Provision {
250 frag: unique_frag(format!("chapter-{}", slugify(&number))),
251 kind: "part".into(),
252 number: number.clone(),
253 heading: format!("Chapter {number}"),
254 text: String::new(),
255 full_text: String::new(),
256 start_page: *page_no,
257 parent: None,
258 });
259 continue;
260 }
261 if let Some(m) = re_eu_article.captures(s) {
262 flush(&mut cur, &mut buf, &mut provisions);
263 let number = m.get(1).unwrap().as_str().to_ascii_uppercase();
264 cur = Some(Provision {
265 frag: unique_frag(format!("article-{}", slugify(&number))),
266 kind: "section".into(),
267 number,
268 heading: String::new(),
269 text: String::new(),
270 full_text: String::new(),
271 start_page: *page_no,
272 parent: None,
273 });
274 continue;
276 }
277 if let Some(m) = re_schedule.captures(s) {
278 let sched = slugify(m.get(2).unwrap().as_str());
279 if current_schedule.as_deref() != Some(&sched) {
280 flush(&mut cur, &mut buf, &mut provisions);
281 current_schedule = Some(sched.clone());
282 provisions.push(Provision {
283 frag: unique_frag(format!("sch-{sched}")),
284 kind: "schedule".into(),
285 number: m.get(2).unwrap().as_str().into(),
286 heading: m.get(3).unwrap().as_str().trim().into(),
287 text: String::new(),
288 full_text: String::new(),
289 start_page: *page_no,
290 parent: None,
291 });
292 }
293 continue;
294 }
295 if let Some(m) = re_part.captures(s) {
296 if is_heading_title(m.get(3).map(|x| x.as_str()).unwrap_or("")) {
297 flush(&mut cur, &mut buf, &mut provisions);
298 let num = m.get(2).unwrap().as_str();
299 let head = m.get(3).unwrap().as_str().trim();
300 let base = if let Some(sch) = ¤t_schedule {
301 format!("sch-{sch}-part-{}", slugify(num))
302 } else {
303 format!("part-{}", slugify(num))
304 };
305 provisions.push(Provision {
306 frag: unique_frag(base),
307 kind: "part".into(),
308 number: num.into(),
309 heading: if head.is_empty() {
310 format!("Part {num}")
311 } else {
312 head.into()
313 },
314 text: String::new(),
315 full_text: String::new(),
316 start_page: *page_no,
317 parent: None,
318 });
319 continue;
320 }
321 }
322 if let Some(m) = re_div.captures(s) {
323 if is_heading_title(m.get(3).map(|x| x.as_str()).unwrap_or("")) {
324 flush(&mut cur, &mut buf, &mut provisions);
325 let num = m.get(2).unwrap().as_str();
326 let head = m.get(3).unwrap().as_str().trim();
327 let base = if let Some(sch) = ¤t_schedule {
328 format!("sch-{sch}-div-{}", slugify(num))
329 } else {
330 format!("div-{}", slugify(num))
331 };
332 provisions.push(Provision {
333 frag: unique_frag(base),
334 kind: "division".into(),
335 number: num.into(),
336 heading: if head.is_empty() {
337 format!("Division {num}")
338 } else {
339 head.into()
340 },
341 text: String::new(),
342 full_text: String::new(),
343 start_page: *page_no,
344 parent: None,
345 });
346 continue;
347 }
348 }
349 if let Some(m) = re_section.captures(s) {
350 let head = m.get(2).unwrap().as_str();
351 if !s.starts_with('(') && looks_like_section_heading(head) {
352 flush(&mut cur, &mut buf, &mut provisions);
353 let num = m.get(1).unwrap().as_str();
354 let base = if let Some(sch) = ¤t_schedule {
355 format!("sch-{sch}-sec-{}", slugify(num))
356 } else {
357 format!("sec-{}", slugify(num))
358 };
359 cur = Some(Provision {
360 frag: unique_frag(base),
361 kind: "section".into(),
362 number: num.into(),
363 heading: head.trim().into(),
364 text: String::new(),
365 full_text: String::new(),
366 start_page: *page_no,
367 parent: None,
368 });
369 continue;
370 }
371 }
372 if !has_eu_articles {
373 if let Some(m) = re_hist.captures(s) {
374 let head = m.get(2).unwrap().as_str();
375 if !s.starts_with('(')
376 && looks_like_section_heading(head)
377 && re_section.captures(s).is_none()
378 {
379 flush(&mut cur, &mut buf, &mut provisions);
380 let num = m.get(1).unwrap().as_str();
381 let base = if let Some(sch) = ¤t_schedule {
382 format!("sch-{sch}-sec-{}", slugify(num))
383 } else {
384 format!("sec-{}", slugify(num))
385 };
386 let heading = head.trim();
387 if heading.len() > 90 || heading.starts_with(|c: char| c.is_lowercase()) {
388 cur = Some(Provision {
389 frag: unique_frag(base),
390 kind: "section".into(),
391 number: num.into(),
392 heading: format!("Section {num}"),
393 text: String::new(),
394 full_text: String::new(),
395 start_page: *page_no,
396 parent: None,
397 });
398 buf.push(heading.into());
399 } else {
400 cur = Some(Provision {
401 frag: unique_frag(base),
402 kind: "section".into(),
403 number: num.into(),
404 heading: heading.into(),
405 text: String::new(),
406 full_text: String::new(),
407 start_page: *page_no,
408 parent: None,
409 });
410 }
411 continue;
412 }
413 }
414 }
415 if let Some(c) = cur.as_mut() {
416 if c.heading.is_empty()
418 && s.len() < 120
419 && !s.chars().next().unwrap_or(' ').is_ascii_digit()
420 {
421 c.heading = s.to_string();
422 } else {
423 buf.push(s.to_string());
424 }
425 }
426 }
427 }
428 flush(&mut cur, &mut buf, &mut provisions);
429
430 provisions = decompose_provisions(provisions);
432
433 let slug = {
434 let mut s = slugify(
435 &Regex::new(r"(?i)\s*No\.\s*\d+.*$")
436 .unwrap()
437 .replace(&title, ""),
438 );
439 let parts: Vec<_> = s.split('-').take(12).collect();
440 s = parts.join("-");
441 if s.len() < 3 || s.len() > 90 {
442 s = slugify(title_hint.unwrap_or("instrument"));
443 }
444 s
445 };
446
447 LegislationInstrument {
448 title,
449 slug,
450 jurisdiction: "AU".into(),
451 register_id: None,
452 provisions,
453 pages: pages.len(),
454 }
455}
456
457fn decompose_provisions(mut provisions: Vec<Provision>) -> Vec<Provision> {
458 let re_sub = Regex::new(r"^\((\d+[A-Za-z]?)\)\s+(.+)$").unwrap();
459 let mut out = Vec::new();
460 for mut section in provisions.drain(..) {
461 if section.kind == "section" {
462 if section.full_text.is_empty() {
463 section.full_text = section.text.clone();
464 }
465 let lines: Vec<&str> = section.text.lines().collect();
466 let starts: Vec<usize> = lines
467 .iter()
468 .enumerate()
469 .filter_map(|(i, ln)| re_sub.is_match(ln).then_some(i))
470 .collect();
471 if starts.len() >= 2 {
472 let mut subs = Vec::new();
473 for (k, &start) in starts.iter().enumerate() {
474 let end = starts.get(k + 1).copied().unwrap_or(lines.len());
475 let block = lines[start..end].join("\n").trim().to_string();
476 let num = re_sub
477 .captures(lines[start])
478 .and_then(|c| c.get(1).map(|m| m.as_str().to_string()))
479 .unwrap_or_else(|| format!("{k}"));
480 let frag = format!("{}-ss-{}", section.frag, slugify(&num));
481 subs.push(Provision {
482 frag,
483 kind: "subsection".into(),
484 number: format!("{}({num})", section.number),
485 heading: section.heading.clone(),
486 text: block.clone(),
487 full_text: block,
488 start_page: section.start_page,
489 parent: Some(section.frag.clone()),
490 });
491 }
492 let lead = if starts[0] > 0 {
493 lines[..starts[0]].join("\n").trim().to_string()
494 } else {
495 String::new()
496 };
497 section.text = lead;
498 out.push(section);
499 out.extend(subs);
500 continue;
501 }
502 }
503 out.push(section);
504 }
505 out
506}
507
508fn infer_title(pages: &[(u32, String)]) -> Option<String> {
509 let joined = pages
510 .iter()
511 .take(5)
512 .map(|(_, r)| r.as_str())
513 .collect::<Vec<_>>()
514 .join(" ");
515 let joined = Regex::new(r"\s+").unwrap().replace_all(&joined, " ");
516 if let Some(c) = Regex::new(r"(?i)may be cited as the\s+(.{2,120}?\bAct\b[^.]{0,40}?\d{4})\b")
517 .unwrap()
518 .captures(&joined)
519 {
520 return Some(c.get(1).unwrap().as_str().trim().to_string());
521 }
522 None
523}
524
525pub fn extract_pdf_pages(path: &Path) -> Result<Vec<(u32, String)>, String> {
527 let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
528 extract_pdf_pages_from_bytes(&bytes)
529}
530
531pub fn extract_pdf_pages_from_bytes(bytes: &[u8]) -> Result<Vec<(u32, String)>, String> {
533 let text = pdf_extract::extract_text_from_mem(bytes).map_err(|e| e.to_string())?;
535 let mut pages = Vec::new();
536 if text.contains('\u{c}') {
537 for (i, part) in text.split('\u{c}').enumerate() {
538 pages.push((i as u32 + 1, part.to_string()));
539 }
540 } else {
541 let chunk = 3500usize;
543 if text.len() <= chunk {
544 pages.push((1, text));
545 } else {
546 let mut i = 0usize;
547 let mut page = 1u32;
548 while i < text.len() {
549 let end = (i + chunk).min(text.len());
550 let mut cut = end;
551 if end < text.len() {
552 if let Some(rel) = text[i..end].rfind('\n') {
553 cut = i + rel + 1;
554 }
555 }
556 if cut <= i {
557 cut = end;
558 }
559 pages.push((page, text[i..cut].to_string()));
560 i = cut;
561 page += 1;
562 }
563 }
564 }
565 if pages.is_empty() {
566 pages.push((1, String::new()));
567 }
568 Ok(pages)
569}
570
571fn fnv60(bytes: &[u8]) -> u64 {
572 const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
573 const FNV_PRIME: u64 = 0x100_0000_01b3;
574 let mut h = FNV_OFFSET;
575 for b in bytes {
576 h ^= u64::from(*b);
577 h = h.wrapping_mul(FNV_PRIME);
578 }
579 h & 0x0FFF_FFFF_FFFF_FFFF
580}
581
582pub fn seed_instrument_into_library(
584 store: &HypermediaStore,
585 inst: &LegislationInstrument,
586 now: u64,
587) -> std::io::Result<LegislationIngestReport> {
588 let mut entries = store.load()?;
589 let mut by_uri: std::collections::HashMap<String, usize> = entries
590 .iter()
591 .enumerate()
592 .map(|(i, e)| (e.asset_uri.clone(), i))
593 .collect();
594
595 let base = format!(
596 "legislation://{}/{}",
597 inst.jurisdiction.to_ascii_lowercase(),
598 inst.register_id.as_deref().unwrap_or(inst.slug.as_str())
599 );
600
601 let mut written = 0usize;
602 let mut upsert = |entry: LibraryEntry| {
603 if let Some(&idx) = by_uri.get(&entry.asset_uri) {
604 let mut e = entry;
605 e.ingested_unix = entries[idx].ingested_unix;
606 entries[idx] = e;
607 } else {
608 by_uri.insert(entry.asset_uri.clone(), entries.len());
609 entries.push(entry);
610 }
611 written += 1;
612 };
613
614 let cml_units: Vec<ContextUnit> = inst
616 .provisions
617 .iter()
618 .map(|p| ContextUnit {
619 frag: p.frag.clone(),
620 kind: p.kind.clone(),
621 label: format!("{} {}", p.number, p.heading).trim().to_string(),
622 text: p.source_text().to_string(),
623 page: Some(p.start_page),
624 parent: p.parent.clone(),
625 })
626 .collect();
627 let instrument_graph = build_document_context(&base, &inst.title, &cml_units);
628 let cof_pkg = build_cof_package(
630 &base,
631 &inst.title,
632 &cml_units,
633 DEFAULT_SEGMENT_MAX_CHARS,
634 CofStyle::AgentLean,
635 );
636
637 let root_uri = base.clone();
639 let root_excerpt = format!(
640 "{} — {} provision(s), {} page(s). CML (cml:Proposed): {} concepts, {} deontic, {} privacy. COF: {} segment(s), ~{} tokens total (load index + one body for agent turns).",
641 inst.title,
642 inst.provisions.len(),
643 inst.pages,
644 instrument_graph.concepts.len(),
645 instrument_graph.deontic_norms,
646 instrument_graph.privacy_hits,
647 cof_pkg.segments.len(),
648 cof_pkg.total_approx_tokens,
649 );
650 let mut root_topics = vec![
651 "legislation".into(),
652 "statute".into(),
653 "cml".into(),
654 inst.jurisdiction.to_ascii_lowercase(),
655 inst.slug.clone(),
656 ];
657 root_topics.extend(instrument_graph.topics.iter().cloned());
658 root_topics.sort();
659 root_topics.dedup();
660 let mut root_purposes = vec!["legislation".into(), "legal".into(), "work".into()];
661 root_purposes.extend(instrument_graph.purposes.iter().cloned());
662 root_purposes.sort();
663 root_purposes.dedup();
664 let root_n3 = if instrument_graph.n3.len() > 64_000 {
665 format!(
666 "{}…\n# [instrument cml_n3 truncated — per-provision entries hold local graphs]",
667 &instrument_graph.n3[..64_000]
668 )
669 } else {
670 instrument_graph.n3.clone()
671 };
672 let mut root = LibraryEntry {
673 asset_uri: root_uri.clone(),
674 primary_subject: fnv60(root_uri.as_bytes()),
675 media_type: LEGISLATION_INSTRUMENT_MEDIA.into(),
676 quins: instrument_graph.quins.clone(),
677 topics: root_topics,
678 projects: vec![format!("legislation:{}", inst.slug)],
679 purposes: root_purposes,
680 place: None,
681 occurred_at: None,
682 lat: None,
683 lon: None,
684 flags: Vec::new(),
685 ingested_unix: now,
686 excerpt: root_excerpt.chars().take(400).collect(),
687 sensitivity: "public".into(),
688 section: LibrarySection::Work.as_str().into(),
689 commons_visibility: CommonsVisibility::None,
690 cml_signals: instrument_graph.signal_tags.clone(),
691 cml_concept_count: instrument_graph.concepts.len() as u32,
692 cml_n3: root_n3,
693 cof_html: cof_pkg
694 .segments
695 .iter()
696 .find(|s| s.is_index)
697 .map(|s| s.html.clone())
698 .unwrap_or_default(),
699 cof_segment_count: cof_pkg.segments.len() as u32,
700 cof_segment_index: 0,
701 cof_profile: cof_pkg.profile.clone(),
702 };
703 root.recompute_section();
704 upsert(root);
705
706 for seg in cof_pkg.segments.iter().filter(|s| !s.is_index) {
708 let seg_uri = format!("{base}#cof-seg-{}", seg.index);
709 let mut se = LibraryEntry {
710 asset_uri: seg_uri.clone(),
711 primary_subject: fnv60(seg_uri.as_bytes()),
712 media_type: MEDIA_TYPE_COF.into(),
713 quins: Vec::new(),
714 topics: vec![
715 "legislation".into(),
716 "cof".into(),
717 "cml".into(),
718 inst.slug.clone(),
719 format!("cof-seg-{}", seg.index),
720 ],
721 projects: vec![format!("legislation:{}", inst.slug)],
722 purposes: vec!["legislation".into(), "legal".into(), "work".into()],
723 place: None,
724 occurred_at: None,
725 lat: None,
726 lon: None,
727 flags: Vec::new(),
728 ingested_unix: now,
729 excerpt: format!(
730 "COF body segment {}/{} · ~{} tokens · frags: {}",
731 seg.index + 1,
732 seg.total,
733 seg.approx_tokens,
734 seg.unit_frags
735 .iter()
736 .take(12)
737 .cloned()
738 .collect::<Vec<_>>()
739 .join(", ")
740 ),
741 sensitivity: "public".into(),
742 section: LibrarySection::Work.as_str().into(),
743 commons_visibility: CommonsVisibility::None,
744 cml_signals: Vec::new(),
745 cml_concept_count: seg.unit_frags.len() as u32,
746 cml_n3: String::new(),
747 cof_html: seg.html.clone(),
748 cof_segment_count: cof_pkg.segments.len() as u32,
749 cof_segment_index: seg.index,
750 cof_profile: cof_pkg.profile.clone(),
751 };
752 se.recompute_section();
753 upsert(se);
754 }
755
756 let mut sections = 0usize;
757 let mut subsections = 0usize;
758 let mut structural = 0usize;
759 let mut with_text = 0usize;
760 let mut empty = 0usize;
761 let mut total_cml_concepts = instrument_graph.concepts.len();
762 let mut total_deontic = instrument_graph.deontic_norms;
763 let mut total_privacy = instrument_graph.privacy_hits;
764 let mut total_rights = instrument_graph.rights_hits;
765
766 for p in &inst.provisions {
767 match p.kind.as_str() {
768 "section" => sections += 1,
769 "subsection" => subsections += 1,
770 "part" | "division" | "schedule" => structural += 1,
771 _ => {}
772 }
773 let body = p.source_text();
774 if body.trim().is_empty() {
775 empty += 1;
776 } else {
777 with_text += 1;
778 }
779 if matches!(
780 p.kind.as_str(),
781 "section" | "subsection" | "part" | "division" | "schedule"
782 ) {
783 let uri = format!("{base}#{}", p.frag);
784 let label = format!("{} {}", p.number, p.heading).trim().to_string();
785 let unit = ContextUnit {
786 frag: p.frag.clone(),
787 kind: p.kind.clone(),
788 label: label.clone(),
789 text: body.to_string(),
790 page: Some(p.start_page),
791 parent: p.parent.clone(),
792 };
793 let g = build_document_context(&uri, &label, &[unit]);
794 total_cml_concepts += g.concepts.len();
795 total_deontic += g.deontic_norms;
796 total_privacy += g.privacy_hits;
797 total_rights += g.rights_hits;
798
799 let mut topics = vec![
800 "legislation".into(),
801 "cml".into(),
802 p.kind.clone(),
803 inst.slug.clone(),
804 format!("s{}", p.number),
805 ];
806 if let Some(parent) = &p.parent {
807 topics.push(parent.clone());
808 }
809 topics.extend(g.topics.iter().cloned());
810 topics.sort();
811 topics.dedup();
812
813 let mut purposes = vec!["legislation".into(), "legal".into(), "work".into()];
814 purposes.extend(g.purposes.iter().cloned());
815 purposes.sort();
816 purposes.dedup();
817
818 let mut excerpt = if body.trim().is_empty() {
819 format!("{label} (no body text extracted)")
820 } else {
821 format!("{label}\n\n{body}")
822 };
823 if excerpt.len() > 12_000 {
824 excerpt = excerpt.chars().take(12_000).collect();
825 excerpt.push_str("\n…[truncated]");
826 }
827 if !g.signal_tags.is_empty() {
829 let chips: String = g
830 .signal_tags
831 .iter()
832 .take(8)
833 .cloned()
834 .collect::<Vec<_>>()
835 .join(" · ");
836 excerpt = format!("[{chips}]\n{excerpt}");
837 }
838
839 let cml_n3 = if g.n3.len() > 24_000 {
840 format!("{}…\n# [truncated]", &g.n3[..24_000])
841 } else {
842 g.n3
843 };
844
845 let mut entry = LibraryEntry {
846 asset_uri: uri.clone(),
847 primary_subject: fnv60(uri.as_bytes()),
848 media_type: LEGISLATION_MEDIA_TYPE.into(),
849 quins: g.quins,
850 topics,
851 projects: vec![format!("legislation:{}", inst.slug)],
852 purposes,
853 place: None,
854 occurred_at: None,
855 lat: None,
856 lon: None,
857 flags: Vec::new(),
858 ingested_unix: now,
859 excerpt,
860 sensitivity: "public".into(),
861 section: LibrarySection::Work.as_str().into(),
862 commons_visibility: CommonsVisibility::None,
863 cml_signals: g.signal_tags,
864 cml_concept_count: g.concepts.len() as u32,
865 cml_n3,
866 cof_html: String::new(),
867 cof_segment_count: cof_pkg.segments.len() as u32,
868 cof_segment_index: 0,
869 cof_profile: cof_pkg.profile.clone(),
870 };
871 entry.recompute_section();
872 upsert(entry);
873 }
874 }
875
876 store.replace_all(&entries)?;
877
878 let concepts = sections + subsections;
879 let coverage_ok = empty == 0 || (with_text as f64 / concepts.max(1) as f64) >= 0.85;
880
881 Ok(LegislationIngestReport {
882 title: inst.title.clone(),
883 slug: inst.slug.clone(),
884 sections,
885 subsections,
886 structural,
887 concepts_with_text: with_text,
888 empty_text: empty,
889 library_entries_written: written,
890 coverage_ok,
891 cml_concepts: total_cml_concepts,
892 cml_deontic_norms: total_deontic,
893 cml_privacy_hits: total_privacy,
894 cml_rights_hits: total_rights,
895 cof_segments: cof_pkg.segments.len(),
896 cof_approx_tokens: cof_pkg.total_approx_tokens,
897 cof_profile: cof_pkg.profile,
898 })
899}
900
901pub fn ingest_legislation_pdf_bytes(
903 store: &HypermediaStore,
904 bytes: &[u8],
905 register_id: Option<&str>,
906 jurisdiction: &str,
907 title_hint: Option<&str>,
908) -> Result<LegislationIngestReport, String> {
909 let pages = extract_pdf_pages_from_bytes(bytes)?;
910 let mut inst = parse_pages(&pages, title_hint);
911 inst.jurisdiction = jurisdiction.to_string();
912 if let Some(id) = register_id {
913 inst.register_id = Some(id.to_string());
914 if inst.slug.len() < 3 {
915 inst.slug = slugify(id);
916 }
917 }
918 let now = std::time::SystemTime::now()
919 .duration_since(std::time::UNIX_EPOCH)
920 .map(|d| d.as_secs())
921 .unwrap_or(0);
922 seed_instrument_into_library(store, &inst, now).map_err(|e| e.to_string())
923}
924
925pub fn ingest_legislation_text(
927 store: &HypermediaStore,
928 text: &str,
929 register_id: Option<&str>,
930 jurisdiction: &str,
931 title_hint: Option<&str>,
932) -> Result<LegislationIngestReport, String> {
933 let pages = vec![(1u32, text.to_string())];
934 let mut inst = parse_pages(&pages, title_hint);
935 inst.jurisdiction = jurisdiction.to_string();
936 if let Some(id) = register_id {
937 inst.register_id = Some(id.to_string());
938 }
939 let now = std::time::SystemTime::now()
940 .duration_since(std::time::UNIX_EPOCH)
941 .map(|d| d.as_secs())
942 .unwrap_or(0);
943 seed_instrument_into_library(store, &inst, now).map_err(|e| e.to_string())
944}
945
946#[cfg(test)]
947mod tests {
948 use super::*;
949
950 #[test]
951 fn enacts_skips_contents_and_keeps_body_text() {
952 let pages = vec![(
953 1u32,
954 "Example Amendment Act 2004\nNo. 5, 2004\n\
955 Contents\n1 Short title\n2 Commencement\nPart 2—Widgets\n3 Widget duty\n\
956 The Parliament of Australia enacts:\n\
957 1 Short title\nThis Act may be cited as the Example Amendment Act 2004.\n\
958 2 Commencement\nThis Act commences on Royal Assent.\n\
959 3 Widget duty\nA person must not widget.\n"
960 .into(),
961 )];
962 let inst = parse_pages(&pages, Some("Example Amendment Act 2004"));
963 let sections: Vec<_> = inst
964 .provisions
965 .iter()
966 .filter(|p| p.kind == "section")
967 .collect();
968 assert_eq!(sections.len(), 3);
969 assert!(sections[0].source_text().contains("may be cited"));
970 assert!(sections[1].source_text().contains("commences"));
971 assert!(sections[2].source_text().contains("must not widget"));
972 assert!(inst.provisions.iter().all(|p| p.kind != "part"));
974 }
975
976 #[test]
977 fn seed_writes_library_entries_with_text() {
978 let dir = tempfile::tempdir().unwrap();
979 let store = HypermediaStore::open(dir.path()).unwrap();
980 let text = "\
981The Parliament of Australia enacts:\n\
9821 Short title\nThis Act may be cited as the Demo Act 2020.\n\
9832 Commencement\nThis Act commences on the day after Royal Assent.\n";
984 let report = ingest_legislation_text(
985 &store,
986 text,
987 Some("C2020A00001"),
988 "AU",
989 Some("Demo Act 2020"),
990 )
991 .unwrap();
992 assert!(report.sections >= 2);
993 assert!(report.concepts_with_text >= 2);
994 assert!(report.coverage_ok);
995 let work = store.by_section(LibrarySection::Work).unwrap();
996 assert!(work.len() >= 3); assert!(work.iter().any(|e| e.excerpt.contains("may be cited")));
998 assert!(work
999 .iter()
1000 .any(|e| e.purposes.iter().any(|p| p == "legislation")));
1001 assert!(report.cml_concepts > 0);
1003 assert!(work
1004 .iter()
1005 .any(|e| e.cml_concept_count > 0 || !e.cml_n3.is_empty()));
1006 }
1007
1008 #[test]
1009 fn subsections_preserve_full_text_on_parent() {
1010 let pages = vec![(
1011 1u32,
1012 "The Parliament of Australia enacts:\n\
1013 5 Offence\n\
1014 A person commits an offence if:\n\
1015 (1) the person does X; and\n\
1016 (2) the person does Y.\n"
1017 .into(),
1018 )];
1019 let inst = parse_pages(&pages, Some("Offence Act"));
1020 let sec = inst
1021 .provisions
1022 .iter()
1023 .find(|p| p.kind == "section")
1024 .unwrap();
1025 let subs: Vec<_> = inst
1026 .provisions
1027 .iter()
1028 .filter(|p| p.kind == "subsection")
1029 .collect();
1030 assert_eq!(subs.len(), 2);
1031 assert!(sec.full_text.contains("(1)"));
1032 assert!(sec.full_text.contains("(2)"));
1033 assert!(subs[0].source_text().contains("does X"));
1034 }
1035}