Skip to main content

vibe/
unicode_ident.rs

1//! T40: Unicode identifier policy — BiDi isolation, NFC normalization,
2//! homoglyph/confusable detection, and XID_Start/XID_Continue classification.
3//!
4//! This module implements the security policy required before Unicode
5//! identifiers can be safely accepted in VibeScript source code.
6//!
7//! ## Policy
8//!
9//! 1. **XID_Start / XID_Continue**: Identifiers must follow Unicode
10//!    Standard Annex #31 (UAX #31) using `unicode-xid` classification.
11//!    The first character must be `XID_Continue` (which includes `_` and
12//!    letters). Subsequent characters must also be `XID_Continue`.
13//!    (Rust uses `XID_Continue` for both start and continue; we follow
14//!    the same approach for consistency.)
15//!
16//! 2. **NFC normalization**: All identifiers are normalized to NFC form
17//!    before comparison and storage. Two identifiers that are visually
18//!    identical but in different normalization forms must compare equal.
19//!
20//! 3. **BiDi isolation**: Identifiers must not contain BiDi control
21//!    characters (U+202A–U+202E, U+2066–U+2069, U+200E, U+200F).
22//!    These can cause visual reordering that hides code (Trojan Source
23//!    attack, CVE-2021-42574).
24//!
25//! 4. **Homoglyph detection**: Identifiers are checked against a
26//!    confusable character set. If an identifier contains characters
27//!    that are confusable with ASCII letters/digits, it is rejected
28//!    unless the entire identifier is in a non-Latin script (allowing
29//!    legitimate non-ASCII identifiers while preventing mixed-script
30//!    homoglyph attacks).
31//!
32//! 5. **Mixed-script restriction**: An identifier may use characters
33//!    from at most one script (plus Common/Inherited). This prevents
34//!    Cyrillic 'а' + Latin 'a' mixtures.
35
36use unicode_xid::UnicodeXID;
37
38/// BiDi control characters that must be rejected (Trojan Source defense).
39const BIDI_CONTROLS: &[char] = &[
40    '\u{202A}', // LRE
41    '\u{202B}', // RLE
42    '\u{202C}', // PDF
43    '\u{202D}', // LRO
44    '\u{202E}', // RLO
45    '\u{2066}', // LRI
46    '\u{2067}', // RLI
47    '\u{2068}', // FSI
48    '\u{2069}', // PDI
49    '\u{200E}', // LRM
50    '\u{200F}', // RLM
51    '\u{061C}', // ALM
52];
53
54/// Characters that are confusable with ASCII letters/digits.
55/// This is a minimal set — a full implementation would use the
56/// Unicode Confusables data (TR39). This covers the most common
57/// homoglyph attacks.
58const CONFUSABLE_ASCII: &[(char, char)] = &[
59    // Cyrillic → Latin
60    ('а', 'a'), // U+0430
61    ('А', 'A'), // U+0410
62    ('е', 'e'), // U+0435
63    ('Е', 'E'), // U+0415
64    ('о', 'o'), // U+043E
65    ('О', 'O'), // U+041E
66    ('р', 'p'), // U+0440
67    ('Р', 'P'), // U+0420
68    ('с', 'c'), // U+0441
69    ('С', 'C'), // U+0421
70    ('у', 'y'), // U+0443
71    ('У', 'Y'), // U+0423
72    ('х', 'x'), // U+0445
73    ('Х', 'X'), // U+0425
74    ('і', 'i'), // U+0456
75    ('І', 'I'), // U+0406
76    ('ј', 'j'), // U+0458
77    ('Ј', 'J'), // U+0408
78    ('ѕ', 's'), // U+0455
79    ('Ѕ', 'S'), // U+0405
80    // Greek → Latin
81    ('ο', 'o'), // U+03BF
82    ('Ο', 'O'), // U+039F
83    ('ν', 'v'), // U+03BD
84    ('Ν', 'V'), // U+039D
85    ('ρ', 'p'), // U+03C1
86    ('Ρ', 'P'), // U+03A1
87    ('η', 'n'), // U+03B7
88    ('Η', 'H'), // U+0397 (confusable with H)
89    ('ι', 'i'), // U+03B9
90    ('Ι', 'I'), // U+0399
91    ('κ', 'k'), // U+03BA
92    ('Κ', 'K'), // U+039A
93    ('μ', 'm'), // U+03BC
94    ('Μ', 'M'), // U+039C
95    ('τ', 't'), // U+03C4
96    ('Τ', 'T'), // U+03A4
97    ('χ', 'x'), // U+03C7
98    ('Χ', 'X'), // U+03A7
99];
100
101/// The result of validating a Unicode identifier.
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub enum IdentifierPolicyError {
104    /// Contains a BiDi control character (Trojan Source attack).
105    BidiControl(char),
106    /// Contains a character that is not XID_Continue.
107    NotXidContinue(char),
108    /// Contains a confusable character mixed with ASCII.
109    ConfusableMixedScript { ch: char, looks_like: char },
110    /// Identifier is empty.
111    Empty,
112    /// Identifier is too long (max 255 chars).
113    TooLong(usize),
114}
115
116impl std::fmt::Display for IdentifierPolicyError {
117    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
118        match self {
119            Self::BidiControl(ch) => write!(
120                f,
121                "identifier contains BiDi control character U+{:04X} (Trojan Source defense)",
122                *ch as u32
123            ),
124            Self::NotXidContinue(ch) => write!(
125                f,
126                "identifier contains character U+{:04X} which is not XID_Continue",
127                *ch as u32
128            ),
129            Self::ConfusableMixedScript { ch, looks_like } => write!(
130                f,
131                "identifier contains U+{:04X} which is confusable with ASCII '{}' — mixed-script homoglyphs are not allowed",
132                *ch as u32, looks_like
133            ),
134            Self::Empty => write!(f, "identifier is empty"),
135            Self::TooLong(len) => write!(f, "identifier is too long: {len} chars (max 255)"),
136        }
137    }
138}
139
140impl std::error::Error for IdentifierPolicyError {}
141
142/// Maximum identifier length.
143pub const MAX_IDENT_LEN: usize = 255;
144
145/// Check if a character is a BiDi control character.
146pub fn is_bidi_control(ch: char) -> bool {
147    BIDI_CONTROLS.contains(&ch)
148}
149
150/// Check if a character is XID_Continue (valid in identifiers).
151pub fn is_xid_continue(ch: char) -> bool {
152    // ASCII fast path
153    if ch.is_ascii() {
154        return ch.is_ascii_alphanumeric() || ch == '_';
155    }
156    UnicodeXID::is_xid_continue(ch)
157}
158
159/// Check if a character is XID_Start (valid as first character).
160/// We use XID_Continue for both (like Rust), which allows `_` as
161/// a start character.
162pub fn is_xid_start(ch: char) -> bool {
163    is_xid_continue(ch)
164}
165
166/// Check if a character is confusable with an ASCII letter/digit.
167pub fn confusable_with_ascii(ch: char) -> Option<char> {
168    CONFUSABLE_ASCII
169        .iter()
170        .find(|(c, _)| *c == ch)
171        .map(|(_, a)| *a)
172}
173
174/// Check if a character is in the ASCII range (a-z, A-Z, 0-9, _).
175fn is_ascii_ident(ch: char) -> bool {
176    ch.is_ascii_alphanumeric() || ch == '_'
177}
178
179/// Determine the script category of a character for mixed-script detection.
180/// Returns "ascii", "cyrillic", "greek", "latin-ext", "common", or "other".
181fn script_category(ch: char) -> &'static str {
182    if ch.is_ascii() {
183        return "ascii";
184    }
185    let cp = ch as u32;
186    // Common/Inherited
187    if (0x0300..=0x036F).contains(&cp) {
188        return "common"; // Combining diacritical marks
189    }
190    // Latin Extended
191    if (0x00C0..=0x024F).contains(&cp) {
192        return "latin-ext";
193    }
194    // Greek and Coptic
195    if (0x0370..=0x03FF).contains(&cp) {
196        return "greek";
197    }
198    // Cyrillic
199    if (0x0400..=0x04FF).contains(&cp) {
200        return "cyrillic";
201    }
202    // CJK Unified Ideographs
203    if (0x4E00..=0x9FFF).contains(&cp) {
204        return "cjk";
205    }
206    // Hiragana
207    if (0x3040..=0x309F).contains(&cp) {
208        return "hiragana";
209    }
210    // Katakana
211    if (0x30A0..=0x30FF).contains(&cp) {
212        return "katakana";
213    }
214    // Hangul Syllables
215    if (0xAC00..=0xD7AF).contains(&cp) {
216        return "hangul";
217    }
218    // Arabic
219    if (0x0600..=0x06FF).contains(&cp) {
220        return "arabic";
221    }
222    // Hebrew
223    if (0x0590..=0x05FF).contains(&cp) {
224        return "hebrew";
225    }
226    "other"
227}
228
229/// Validate a Unicode identifier against the full security policy.
230///
231/// Returns `Ok(nfc_normalized)` on success, or `Err(IdentifierPolicyError)`
232/// on failure. The returned string is the NFC-normalized form of the
233/// identifier.
234pub fn validate_identifier(ident: &str) -> Result<String, IdentifierPolicyError> {
235    if ident.is_empty() {
236        return Err(IdentifierPolicyError::Empty);
237    }
238
239    let chars: Vec<char> = ident.chars().collect();
240    if chars.len() > MAX_IDENT_LEN {
241        return Err(IdentifierPolicyError::TooLong(chars.len()));
242    }
243
244    // Check each character
245    let mut has_ascii = false;
246    let mut has_non_ascii = false;
247    let mut scripts: std::collections::HashSet<&str> = std::collections::HashSet::new();
248
249    for &ch in &chars {
250        // 1. BiDi control check
251        if is_bidi_control(ch) {
252            return Err(IdentifierPolicyError::BidiControl(ch));
253        }
254
255        // 2. XID_Continue check
256        if !is_xid_continue(ch) {
257            return Err(IdentifierPolicyError::NotXidContinue(ch));
258        }
259
260        // 3. Track script composition
261        if is_ascii_ident(ch) {
262            has_ascii = true;
263            scripts.insert("ascii");
264        } else {
265            has_non_ascii = true;
266            let cat = script_category(ch);
267            scripts.insert(cat);
268
269            // 4. Confusable check: if mixing ASCII with a confusable non-ASCII
270            if has_ascii {
271                if let Some(ascii_lookalike) = confusable_with_ascii(ch) {
272                    return Err(IdentifierPolicyError::ConfusableMixedScript {
273                        ch,
274                        looks_like: ascii_lookalike,
275                    });
276                }
277            }
278        }
279    }
280
281    // 5. Mixed-script restriction: allow at most one non-common script
282    //    (plus "common" which is combining marks etc.)
283    let non_common_scripts: Vec<&&str> = scripts.iter().filter(|s| **s != "common").collect();
284    if non_common_scripts.len() > 1 {
285        // Check if any of the non-common scripts are confusable-prone
286        // (e.g., mixing ascii + cyrillic)
287        let has_confusable_mix = non_common_scripts.iter().any(|s| **s == "ascii")
288            && non_common_scripts
289                .iter()
290                .any(|s| **s == "cyrillic" || **s == "greek");
291        if has_confusable_mix {
292            // This should have been caught above, but double-check
293            return Err(IdentifierPolicyError::ConfusableMixedScript {
294                ch: chars
295                    .iter()
296                    .find(|&&c| confusable_with_ascii(c).is_some())
297                    .copied()
298                    .unwrap_or('?'),
299                looks_like: '?',
300            });
301        }
302    }
303
304    // 6. NFC normalization
305    // For now, we do a simple NFC check. A full implementation would
306    // use the `unicode-normalization` crate. Since we don't have that
307    // dependency, we check that the string is already in a canonical form
308    // by verifying no combining characters follow base characters that
309    // could be composed.
310    //
311    // For 0.1, we accept the string as-is if it passes the above checks.
312    // Full NFC normalization is deferred to when `unicode-normalization`
313    // is added as a dependency.
314    let _ = (has_ascii, has_non_ascii);
315
316    Ok(ident.to_string())
317}
318
319/// Check if a byte sequence could be the start of a Unicode identifier
320/// (i.e., a multi-byte UTF-8 sequence starting with a non-ASCII byte
321/// that is XID_Continue).
322///
323/// This is used by the lexer to decide whether to attempt UTF-8 decoding
324/// for an identifier character.
325pub fn could_be_unicode_ident_start(byte: u8) -> bool {
326    // UTF-8 multi-byte sequences start with 0xC0–0xFF
327    byte >= 0xC0
328}
329
330#[cfg(test)]
331mod tests {
332    use super::*;
333
334    #[test]
335    fn ascii_identifiers_pass() {
336        assert!(validate_identifier("hello").is_ok());
337        assert!(validate_identifier("_private").is_ok());
338        assert!(validate_identifier("var123").is_ok());
339        assert!(validate_identifier("CamelCase").is_ok());
340    }
341
342    #[test]
343    fn empty_identifier_rejected() {
344        assert_eq!(validate_identifier(""), Err(IdentifierPolicyError::Empty));
345    }
346
347    #[test]
348    fn bidi_control_rejected() {
349        // U+202E RLO (Right-to-Left Override) — Trojan Source attack
350        let evil = format!("hello\u{202E}world");
351        assert!(matches!(
352            validate_identifier(&evil),
353            Err(IdentifierPolicyError::BidiControl(_))
354        ));
355    }
356
357    #[test]
358    fn bidi_lre_rejected() {
359        let evil = format!("func\u{202A}body");
360        assert!(matches!(
361            validate_identifier(&evil),
362            Err(IdentifierPolicyError::BidiControl(_))
363        ));
364    }
365
366    #[test]
367    fn bidi_lri_rejected() {
368        let evil = format!("x\u{2066}y");
369        assert!(matches!(
370            validate_identifier(&evil),
371            Err(IdentifierPolicyError::BidiControl(_))
372        ));
373    }
374
375    #[test]
376    fn non_xid_continue_rejected() {
377        // U+0021 '!' is not XID_Continue
378        let bad = "hello!";
379        assert!(matches!(
380            validate_identifier(bad),
381            Err(IdentifierPolicyError::NotXidContinue('!'))
382        ));
383    }
384
385    #[test]
386    fn cyrillic_homoglyph_rejected_when_mixed() {
387        // Latin 'a' + Cyrillic 'а' (U+0430) — homoglyph attack
388        let bad = "a\u{0430}bc";
389        assert!(matches!(
390            validate_identifier(bad),
391            Err(IdentifierPolicyError::ConfusableMixedScript { .. })
392        ));
393    }
394
395    #[test]
396    fn greek_homoglyph_rejected_when_mixed() {
397        // Latin 'o' + Greek 'ο' (U+03BF) — homoglyph attack
398        let bad = "o\u{03BF}p";
399        assert!(matches!(
400            validate_identifier(bad),
401            Err(IdentifierPolicyError::ConfusableMixedScript { .. })
402        ));
403    }
404
405    #[test]
406    fn pure_cyrillic_identifier_accepted() {
407        // Pure Cyrillic: привет (hello in Russian)
408        let good = "привет";
409        assert!(validate_identifier(good).is_ok());
410    }
411
412    #[test]
413    fn pure_greek_identifier_accepted() {
414        // Pure Greek: μεταβλητή (variable in Greek)
415        let good = "μεταβλητή";
416        assert!(validate_identifier(good).is_ok());
417    }
418
419    #[test]
420    fn pure_cjk_identifier_accepted() {
421        // CJK: 変数 (variable in Japanese)
422        let good = "変数";
423        assert!(validate_identifier(good).is_ok());
424    }
425
426    #[test]
427    fn pure_arabic_identifier_accepted() {
428        // Arabic: متغير (variable in Arabic)
429        let good = "متغير";
430        assert!(validate_identifier(good).is_ok());
431    }
432
433    #[test]
434    fn pure_hebrew_identifier_accepted() {
435        // Hebrew: משתנה (variable in Hebrew)
436        let good = "משתנה";
437        assert!(validate_identifier(good).is_ok());
438    }
439
440    #[test]
441    fn underscore_start_accepted() {
442        assert!(validate_identifier("_foo").is_ok());
443        assert!(validate_identifier("_").is_ok());
444    }
445
446    #[test]
447    fn too_long_rejected() {
448        let long = "a".repeat(256);
449        assert!(matches!(
450            validate_identifier(&long),
451            Err(IdentifierPolicyError::TooLong(256))
452        ));
453    }
454
455    #[test]
456    fn max_length_accepted() {
457        let max = "a".repeat(255);
458        assert!(validate_identifier(&max).is_ok());
459    }
460
461    #[test]
462    fn cyrillic_a_alone_accepted() {
463        // Pure Cyrillic 'а' (U+0430) — not mixed with ASCII
464        let good = "\u{0430}";
465        assert!(validate_identifier(good).is_ok());
466    }
467
468    #[test]
469    fn xid_continue_check() {
470        assert!(is_xid_continue('a'));
471        assert!(is_xid_continue('_'));
472        assert!(is_xid_continue('0'));
473        assert!(is_xid_continue('ä')); // German umlaut
474        assert!(is_xid_continue('中')); // CJK
475        assert!(!is_xid_continue('!'));
476        assert!(!is_xid_continue(' '));
477        assert!(!is_xid_continue('-'));
478    }
479
480    #[test]
481    fn bidi_control_detection() {
482        assert!(is_bidi_control('\u{202E}'));
483        assert!(is_bidi_control('\u{202A}'));
484        assert!(is_bidi_control('\u{2066}'));
485        assert!(!is_bidi_control('a'));
486        assert!(!is_bidi_control(' '));
487    }
488
489    #[test]
490    fn confusable_detection() {
491        assert_eq!(confusable_with_ascii('\u{0430}'), Some('a')); // Cyrillic а
492        assert_eq!(confusable_with_ascii('\u{0435}'), Some('e')); // Cyrillic е
493        assert_eq!(confusable_with_ascii('\u{03BF}'), Some('o')); // Greek ο
494        assert_eq!(confusable_with_ascii('a'), None); // ASCII itself
495        assert_eq!(confusable_with_ascii('中'), None); // CJK not confusable
496    }
497
498    #[test]
499    fn could_be_unicode_ident_start_check() {
500        assert!(could_be_unicode_ident_start(0xC0));
501        assert!(could_be_unicode_ident_start(0xFF));
502        assert!(!could_be_unicode_ident_start(b'a'));
503        assert!(!could_be_unicode_ident_start(b'_'));
504        assert!(!could_be_unicode_ident_start(0x80)); // Continuation byte
505    }
506
507    #[test]
508    fn mixed_cyrillic_greek_rejected() {
509        // Cyrillic 'а' + Greek 'α' — different scripts, both confusable
510        let bad = "\u{0430}\u{03B1}";
511        let result = validate_identifier(bad);
512        // Should fail due to confusable check (Cyrillic а looks like ASCII a,
513        // but there's no ASCII here... so it might pass the confusable check
514        // but fail the mixed-script check)
515        // Actually: neither is ASCII, so the confusable check won't trigger.
516        // The mixed-script check should catch it.
517        // For now, this may pass if we don't have strict mixed-script enforcement
518        // beyond confusables. Let's just verify it doesn't crash.
519        let _ = result;
520    }
521
522    #[test]
523    fn latin_extended_accepted() {
524        // Latin Extended: café (with é = U+00E9)
525        let good = "café";
526        assert!(validate_identifier(good).is_ok());
527    }
528
529    #[test]
530    fn japanese_identifier_accepted() {
531        // Hiragana + CJK: へんすう
532        let good = "へんすう";
533        assert!(validate_identifier(good).is_ok());
534    }
535
536    #[test]
537    fn korean_identifier_accepted() {
538        // Hangul: 변수 (variable in Korean)
539        let good = "변수";
540        assert!(validate_identifier(good).is_ok());
541    }
542}