Skip to main content

qualia_client_core/wellfair/cml_context/
extract.rs

1//! Deterministic signal extractors over plain text (legislation, policy, general prose).
2//!
3//! These are **heuristic proposals** for the CML layer — not legal advice and not attested.
4
5use regex::Regex;
6use serde::{Deserialize, Serialize};
7
8/// Proposed deontic class for a provision / paragraph.
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
10#[serde(rename_all = "snake_case")]
11pub enum DeonticClass {
12    Obligation,
13    Permission,
14    Prohibition,
15    Right,
16    /// Neutral / descriptive / machinery — no clear duty.
17    Undertaking,
18}
19
20impl DeonticClass {
21    /// `str` form.
22    pub fn as_str(self) -> &'static str {
23        match self {
24            Self::Obligation => "obligation",
25            Self::Permission => "permission",
26            Self::Prohibition => "prohibition",
27            Self::Right => "right",
28            Self::Undertaking => "undertaking",
29        }
30    }
31
32    /// CML type.
33    pub fn cml_type(self) -> &'static str {
34        match self {
35            Self::Obligation => "values:Obligation",
36            Self::Permission => "values:Permission",
37            Self::Prohibition => "values:Prohibition",
38            Self::Right => "values:Right",
39            Self::Undertaking => "values:Undertaking",
40        }
41    }
42}
43
44/// A named signal hit (privacy family, rights, etc.).
45#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
46pub struct SignalHit {
47    pub family: String,
48    pub signal: String,
49    pub confidence: u8,
50}
51
52/// Privacy / data-protection family (GDPR-like and cognates).
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum PrivacySignal {
55    PersonalData,
56    SpecialCategory,
57    DataSubject,
58    Controller,
59    Processor,
60    Consent,
61    LawfulBasis,
62    PurposeLimitation,
63    DataMinimisation,
64    StorageLimitation,
65    IntegrityConfidentiality,
66    Accountability,
67    Erasure,
68    AccessRight,
69    Portability,
70    Objection,
71    Rectification,
72    Restriction,
73    AutomatedDecision,
74    Dpia,
75    CrossBorderTransfer,
76    BreachNotification,
77    Children,
78    Surveillance,
79}
80
81impl PrivacySignal {
82    /// `str` form.
83    pub fn as_str(self) -> &'static str {
84        match self {
85            Self::PersonalData => "personal-data",
86            Self::SpecialCategory => "special-category-data",
87            Self::DataSubject => "data-subject",
88            Self::Controller => "controller",
89            Self::Processor => "processor",
90            Self::Consent => "consent",
91            Self::LawfulBasis => "lawful-basis",
92            Self::PurposeLimitation => "purpose-limitation",
93            Self::DataMinimisation => "data-minimisation",
94            Self::StorageLimitation => "storage-limitation",
95            Self::IntegrityConfidentiality => "integrity-confidentiality",
96            Self::Accountability => "accountability",
97            Self::Erasure => "erasure-right-to-be-forgotten",
98            Self::AccessRight => "access-right",
99            Self::Portability => "data-portability",
100            Self::Objection => "right-to-object",
101            Self::Rectification => "rectification",
102            Self::Restriction => "restriction-of-processing",
103            Self::AutomatedDecision => "automated-decision-making",
104            Self::Dpia => "dpia",
105            Self::CrossBorderTransfer => "cross-border-transfer",
106            Self::BreachNotification => "breach-notification",
107            Self::Children => "children-data",
108            Self::Surveillance => "surveillance",
109        }
110    }
111}
112
113/// Classify deontic force from English legislative / policy phrasing.
114pub fn classify_deontic(text: &str) -> (DeonticClass, u8) {
115    let t = text.to_ascii_lowercase();
116    // Order matters: stronger / more specific first.
117    let forbid = [
118        "must not",
119        "shall not",
120        "may not",
121        "is prohibited",
122        "are prohibited",
123        "it is an offence",
124        "commits an offence",
125        "is guilty of an offence",
126        "must never",
127        "shall never",
128        "is forbidden",
129        "not permitted to",
130        "unlawful to",
131    ];
132    if forbid.iter().any(|p| t.contains(p)) {
133        return (DeonticClass::Prohibition, 88);
134    }
135    let right = [
136        "has a right to",
137        "have a right to",
138        "is entitled to",
139        "are entitled to",
140        "right of access",
141        "right to erasure",
142        "right to be forgotten",
143        "right to object",
144        "right to data portability",
145        "right to rectification",
146        "right to restriction",
147        "fundamental right",
148        "human right",
149    ];
150    if right.iter().any(|p| t.contains(p)) {
151        return (DeonticClass::Right, 86);
152    }
153    let oblig = [
154        " must ",
155        " shall ",
156        "is required to",
157        "are required to",
158        "is obliged to",
159        "duty to",
160        "obligation to",
161        "must ensure",
162        "shall ensure",
163        "must provide",
164        "shall provide",
165        "must notify",
166        "shall notify",
167    ];
168    // Leading "Must" / "Shall" at sentence start
169    if t.starts_with("must ") || t.starts_with("shall ") {
170        return (DeonticClass::Obligation, 84);
171    }
172    if oblig.iter().any(|p| t.contains(p)) {
173        return (DeonticClass::Obligation, 82);
174    }
175    let permit = [
176        " may ",
177        "is permitted to",
178        "are permitted to",
179        "is authorised to",
180        "is authorized to",
181        "has power to",
182        "may elect",
183        "at the discretion",
184    ];
185    if t.starts_with("may ") {
186        return (DeonticClass::Permission, 78);
187    }
188    if permit.iter().any(|p| t.contains(p)) {
189        return (DeonticClass::Permission, 76);
190    }
191    (DeonticClass::Undertaking, 40)
192}
193
194/// Extract GDPR-like / privacy-family signals.
195pub fn extract_privacy_signals(text: &str) -> Vec<SignalHit> {
196    let t = text.to_ascii_lowercase();
197    let mut hits = Vec::new();
198    let mut push = |sig: PrivacySignal, conf: u8, needles: &[&str]| {
199        if needles.iter().any(|n| t.contains(n)) {
200            hits.push(SignalHit {
201                family: "privacy".into(),
202                signal: sig.as_str().into(),
203                confidence: conf,
204            });
205        }
206    };
207    push(
208        PrivacySignal::PersonalData,
209        90,
210        &[
211            "personal data",
212            "personal information",
213            "personally identifiable",
214            "pii",
215            "information about an individual",
216            "identifiable natural person",
217        ],
218    );
219    push(
220        PrivacySignal::SpecialCategory,
221        92,
222        &[
223            "special category",
224            "sensitive personal",
225            "racial or ethnic",
226            "political opinion",
227            "religious belief",
228            "trade union membership",
229            "genetic data",
230            "biometric data",
231            "health data",
232            "sex life",
233            "sexual orientation",
234        ],
235    );
236    push(
237        PrivacySignal::DataSubject,
238        88,
239        &["data subject", "individual concerned", "person to whom"],
240    );
241    push(
242        PrivacySignal::Controller,
243        88,
244        &[
245            "data controller",
246            "controller shall",
247            "controller must",
248            "as controller",
249        ],
250    );
251    push(
252        PrivacySignal::Processor,
253        88,
254        &[
255            "data processor",
256            "processor shall",
257            "processor must",
258            "as processor",
259        ],
260    );
261    push(
262        PrivacySignal::Consent,
263        85,
264        &[
265            "consent",
266            "freely given",
267            "informed consent",
268            "withdraw consent",
269        ],
270    );
271    push(
272        PrivacySignal::LawfulBasis,
273        87,
274        &[
275            "lawful basis",
276            "lawful ground",
277            "legitimate interest",
278            "legal obligation",
279            "vital interest",
280            "public task",
281            "contractual necessity",
282        ],
283    );
284    push(
285        PrivacySignal::PurposeLimitation,
286        84,
287        &[
288            "purpose limitation",
289            "specified purpose",
290            "compatible purpose",
291            "further processing",
292        ],
293    );
294    push(
295        PrivacySignal::DataMinimisation,
296        84,
297        &[
298            "data minimisation",
299            "data minimization",
300            "not excessive",
301            "adequate, relevant",
302        ],
303    );
304    push(
305        PrivacySignal::StorageLimitation,
306        84,
307        &[
308            "storage limitation",
309            "no longer than necessary",
310            "retention period",
311            "kept no longer",
312        ],
313    );
314    push(
315        PrivacySignal::IntegrityConfidentiality,
316        83,
317        &[
318            "integrity and confidentiality",
319            "appropriate security",
320            "technical and organisational",
321            "technical and organizational",
322            "security of processing",
323            "encryption",
324            "pseudonymisation",
325            "pseudonymization",
326        ],
327    );
328    push(
329        PrivacySignal::Accountability,
330        82,
331        &[
332            "accountability",
333            "demonstrate compliance",
334            "records of processing",
335        ],
336    );
337    push(
338        PrivacySignal::Erasure,
339        90,
340        &[
341            "right to erasure",
342            "right to be forgotten",
343            "erase personal data",
344            "delete the personal",
345            "destruction of personal",
346        ],
347    );
348    push(
349        PrivacySignal::AccessRight,
350        88,
351        &[
352            "right of access",
353            "subject access",
354            "access to personal data",
355            "copy of the personal",
356        ],
357    );
358    push(
359        PrivacySignal::Portability,
360        88,
361        &[
362            "data portability",
363            "structured, commonly used",
364            "machine-readable format",
365        ],
366    );
367    push(
368        PrivacySignal::Objection,
369        86,
370        &["right to object", "object to processing", "opt out"],
371    );
372    push(
373        PrivacySignal::Rectification,
374        86,
375        &[
376            "right to rectification",
377            "rectify",
378            "inaccurate personal data",
379        ],
380    );
381    push(
382        PrivacySignal::Restriction,
383        85,
384        &["restriction of processing", "restrict processing"],
385    );
386    push(
387        PrivacySignal::AutomatedDecision,
388        90,
389        &[
390            "automated decision",
391            "automated processing",
392            "profiling",
393            "solely automated",
394            "algorithmic decision",
395        ],
396    );
397    push(
398        PrivacySignal::Dpia,
399        91,
400        &[
401            "data protection impact assessment",
402            "dpia",
403            "privacy impact assessment",
404            "pia ",
405        ],
406    );
407    push(
408        PrivacySignal::CrossBorderTransfer,
409        89,
410        &[
411            "transfer to a third country",
412            "cross-border",
413            "cross border",
414            "adequacy decision",
415            "standard contractual clauses",
416            "binding corporate rules",
417            "overseas disclosure",
418            "overseas recipient",
419        ],
420    );
421    push(
422        PrivacySignal::BreachNotification,
423        90,
424        &[
425            "personal data breach",
426            "data breach",
427            "notify the supervisory",
428            "notify the commissioner",
429            "breach notification",
430            "security incident",
431        ],
432    );
433    push(
434        PrivacySignal::Children,
435        87,
436        &[
437            "child's personal data",
438            "children's data",
439            "under the age of 16",
440            "parental consent",
441        ],
442    );
443    push(
444        PrivacySignal::Surveillance,
445        80,
446        &[
447            "surveillance",
448            "intercept",
449            "tracking",
450            "cctv",
451            "location data",
452            "metadata retention",
453        ],
454    );
455    // Dedup by signal name (keep highest conf).
456    hits.sort_by(|a, b| a.signal.cmp(&b.signal));
457    hits.dedup_by(|a, b| {
458        if a.signal == b.signal {
459            if a.confidence < b.confidence {
460                *a = b.clone();
461            }
462            true
463        } else {
464            false
465        }
466    });
467    hits
468}
469
470/// Human-rights / civil-rights cues (broader than privacy).
471pub fn extract_rights_signals(text: &str) -> Vec<SignalHit> {
472    let t = text.to_ascii_lowercase();
473    let mut hits = Vec::new();
474    let pairs: &[(&str, &str, u8)] = &[
475        ("human-rights", "human rights", 88),
476        ("human-rights", "charter of rights", 90),
477        ("human-rights", "bill of rights", 88),
478        ("non-discrimination", "discrimination", 80),
479        ("non-discrimination", "equal treatment", 82),
480        ("due-process", "due process", 85),
481        ("due-process", "natural justice", 84),
482        ("freedom-of-expression", "freedom of expression", 88),
483        ("freedom-of-expression", "freedom of speech", 86),
484        ("privacy-as-right", "right to privacy", 90),
485        ("privacy-as-right", "respect for private life", 88),
486        ("liberty", "personal liberty", 82),
487        ("fair-trial", "fair trial", 88),
488        ("refugees", "non-refoulement", 90),
489        ("indigenous", "indigenous", 75),
490        ("indigenous", "aboriginal", 75),
491        ("disability", "disability rights", 85),
492        ("labour", "workplace right", 80),
493        ("labour", "industrial relations", 78),
494    ];
495    for (sig, needle, conf) in pairs {
496        if t.contains(needle) {
497            hits.push(SignalHit {
498                family: "rights".into(),
499                signal: (*sig).into(),
500                confidence: *conf,
501            });
502        }
503    }
504    hits.sort_by(|a, b| a.signal.cmp(&b.signal));
505    hits.dedup_by(|a, b| a.signal == b.signal);
506    hits
507}
508
509/// Temporal / LTL-ish cues (commencement, deadlines).
510pub fn extract_temporal_signals(text: &str) -> Vec<SignalHit> {
511    let t = text.to_ascii_lowercase();
512    let mut hits = Vec::new();
513    let pairs: &[(&str, &str, u8)] = &[
514        ("commencement", "commences", 80),
515        ("commencement", "comes into force", 85),
516        ("commencement", "comes into operation", 85),
517        ("royal-assent", "royal assent", 88),
518        ("proclamation", "proclamation", 82),
519        ("deadline-days", "within ", 70),
520        ("deadline-days", " not later than ", 78),
521        ("deadline-days", "no later than", 80),
522        ("sunset", "ceases to have effect", 85),
523        ("sunset", "sunsets", 80),
524        ("retrospective", "is taken to have", 82),
525        ("retrospective", "deemed to have commenced", 85),
526    ];
527    for (sig, needle, conf) in pairs {
528        if t.contains(needle) {
529            hits.push(SignalHit {
530                family: "temporal".into(),
531                signal: (*sig).into(),
532                confidence: *conf,
533            });
534        }
535    }
536    // Numeric day windows: "within 30 days"
537    if Regex::new(r"(?i)within\s+\d+\s+days")
538        .unwrap()
539        .is_match(text)
540    {
541        hits.push(SignalHit {
542            family: "temporal".into(),
543            signal: "within-n-days".into(),
544            confidence: 86,
545        });
546    }
547    hits.sort_by(|a, b| a.signal.cmp(&b.signal));
548    hits.dedup_by(|a, b| a.signal == b.signal);
549    hits
550}
551
552/// Cross-references to other provisions / instruments.
553pub fn extract_cross_refs(text: &str) -> Vec<String> {
554    let mut out = Vec::new();
555    let patterns = [
556        r"(?i)\bsection\s+(\d+[A-Za-z]{0,2}(?:\(\d+[A-Za-z]?\))?)",
557        r"(?i)\bsubsection\s+\((\d+[A-Za-z]?)\)",
558        r"(?i)\barticle\s+(\d+[A-Za-z]?)",
559        r"(?i)\bschedule\s+(\d+[A-Za-z]?)",
560        r"(?i)\bpart\s+([0-9IVXLC]+[A-Za-z]?)",
561        r"(?i)\bdivision\s+(\d+[A-Za-z]?)",
562        r"(?i)\bregulation\s+(\d+[A-Za-z]?)",
563    ];
564    for pat in patterns {
565        let re = Regex::new(pat).unwrap();
566        for cap in re.captures_iter(text) {
567            if let Some(m) = cap.get(0) {
568                let s = m.as_str().trim().to_string();
569                if !out.contains(&s) {
570                    out.push(s);
571                }
572            }
573        }
574    }
575    out.truncate(32);
576    out
577}
578
579#[cfg(test)]
580mod tests {
581    use super::*;
582
583    #[test]
584    fn deontic_prohibition_and_obligation() {
585        assert_eq!(
586            classify_deontic("A person must not disclose personal data.").0,
587            DeonticClass::Prohibition
588        );
589        assert_eq!(
590            classify_deontic("The controller shall ensure appropriate security.").0,
591            DeonticClass::Obligation
592        );
593        assert_eq!(
594            classify_deontic("The individual has a right to access their data.").0,
595            DeonticClass::Right
596        );
597        assert_eq!(
598            classify_deontic("The Commissioner may issue guidelines.").0,
599            DeonticClass::Permission
600        );
601    }
602
603    #[test]
604    fn privacy_gdpr_family_hits() {
605        let text = "The data controller must obtain consent before processing personal data \
606                    and honour the right to erasure. A DPIA is required for profiling.";
607        let hits = extract_privacy_signals(text);
608        let sigs: Vec<_> = hits.iter().map(|h| h.signal.as_str()).collect();
609        assert!(sigs.contains(&"controller"));
610        assert!(sigs.contains(&"consent"));
611        assert!(sigs.contains(&"personal-data"));
612        assert!(sigs.contains(&"erasure-right-to-be-forgotten"));
613        assert!(sigs.contains(&"dpia") || sigs.contains(&"automated-decision-making"));
614    }
615
616    #[test]
617    fn cross_refs_captured() {
618        let refs = extract_cross_refs("Subject to section 12(1) and Schedule 2, see Article 6.");
619        assert!(refs
620            .iter()
621            .any(|r| r.to_ascii_lowercase().contains("section")));
622        assert!(refs
623            .iter()
624            .any(|r| r.to_ascii_lowercase().contains("schedule")));
625    }
626}