Skip to main content

vibe/stdlib_v2/text_and_collections/
grapheme_str.rs

1//! Unicode UAX #29 Extended Grapheme Cluster String Operations (`str.*`).
2//!
3//! Eliminates JavaScript's UTF-16 surrogate-pair corruption (`"๐Ÿš€".length === 2`,
4//! `"๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ".length === 11`, and `$&`/`$'` replacement traps) by operating on extended
5//! grapheme clusters, explicit scalar code points, and UTF-8 bytes.
6
7use crate::error::Diagnostic;
8use crate::span::Span;
9use crate::value::Value;
10
11use super::{
12    extract_i64, extract_list, extract_opt_i64, extract_str, resolve_element_index,
13    resolve_slice_bound,
14};
15
16/// Segment a UTF-8 string into Unicode UAX #29 Extended Grapheme Clusters.
17///
18/// Keeps together:
19/// - CRLF (`\r\n`)
20/// - Base characters + combining marks, variation selectors, and enclosing marks
21/// - Emoji skin-tone modifiers (`\u{1F3FB}..=\u{1F3FF}`)
22/// - Zero-Width Joiner (`\u{200D}`) emoji sequences (e.g. `"๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ"`, `"๐Ÿ‘ฉโ€๐Ÿš€"`)
23/// - Regional Indicator flag pairs (`\u{1F1E6}..=\u{1F1FF}` in pairs, e.g. `"๐Ÿ‡ฆ๐Ÿ‡บ"`)
24pub fn split_graphemes(s: &str) -> Vec<String> {
25    let chars: Vec<char> = s.chars().collect();
26    let mut clusters = Vec::new();
27    let mut i = 0;
28
29    while i < chars.len() {
30        let first = chars[i];
31        let mut cluster = String::new();
32        cluster.push(first);
33        i += 1;
34
35        if first == '\r' {
36            if i < chars.len() && chars[i] == '\n' {
37                cluster.push('\n');
38                i += 1;
39            }
40            clusters.push(cluster);
41            continue;
42        }
43        if first == '\n' {
44            clusters.push(cluster);
45            continue;
46        }
47
48        let mut ri_count = usize::from(is_regional_indicator(first));
49
50        while i < chars.len() {
51            let c = chars[i];
52            if c == '\r' || c == '\n' {
53                break;
54            }
55            if is_grapheme_extend_or_modifier(c) {
56                cluster.push(c);
57                i += 1;
58                continue;
59            }
60            if c == '\u{200D}' {
61                cluster.push(c);
62                i += 1;
63                if i < chars.len() && chars[i] != '\r' && chars[i] != '\n' {
64                    cluster.push(chars[i]);
65                    i += 1;
66                }
67                continue;
68            }
69            if ri_count == 1 && is_regional_indicator(c) {
70                cluster.push(c);
71                ri_count += 1;
72                i += 1;
73                continue;
74            }
75            break;
76        }
77
78        clusters.push(cluster);
79    }
80
81    clusters
82}
83
84fn is_regional_indicator(c: char) -> bool {
85    matches!(c as u32, 0x1F1E6..=0x1F1FF)
86}
87
88fn is_grapheme_extend_or_modifier(c: char) -> bool {
89    let cp = c as u32;
90    matches!(
91        cp,
92        0x0300..=0x036F
93            | 0x0483..=0x0489
94            | 0x0591..=0x05BD
95            | 0x0610..=0x061A
96            | 0x064B..=0x065F
97            | 0x0670
98            | 0x06D6..=0x06ED
99            | 0x0900..=0x0903
100            | 0x093A..=0x094F
101            | 0x0E31
102            | 0x0E34..=0x0E3A
103            | 0x0E47..=0x0E4E
104            | 0x1AB0..=0x1AFF
105            | 0x1DC0..=0x1DFF
106            | 0x200C
107            | 0x20D0..=0x20FF
108            | 0xFE00..=0xFE0F
109            | 0xFE20..=0xFE2F
110            | 0x1F3FB..=0x1F3FF
111            | 0xE0020..=0xE007F
112            | 0xE0100..=0xE01EF
113    )
114}
115
116/// Dispatch `str.*` bindings.
117pub(crate) fn call_grapheme_str(
118    path: &str,
119    args: &[Value],
120    span: Span,
121) -> Result<Option<Value>, Diagnostic> {
122    match path {
123        "str.graphemes" => {
124            let s = extract_str(args, 0, "str.graphemes(s)", span)?;
125            let list = split_graphemes(&s)
126                .into_iter()
127                .map(Value::String)
128                .collect();
129            Ok(Some(Value::List(list)))
130        }
131        "str.grapheme_len" => {
132            let s = extract_str(args, 0, "str.grapheme_len(s)", span)?;
133            let count = split_graphemes(&s).len() as i64;
134            Ok(Some(Value::I64(count)))
135        }
136        "str.grapheme_at" => {
137            let s = extract_str(args, 0, "str.grapheme_at(s, idx)", span)?;
138            let idx = extract_i64(args, 1, "str.grapheme_at(s, idx)", span)?;
139            let graphemes = split_graphemes(&s);
140            let out = resolve_element_index(idx, graphemes.len())
141                .map(|i| Value::String(graphemes[i].clone()))
142                .unwrap_or(Value::Null);
143            Ok(Some(out))
144        }
145        "str.grapheme_slice" => {
146            let s = extract_str(args, 0, "str.grapheme_slice(s, start, end_opt)", span)?;
147            let start = extract_i64(args, 1, "str.grapheme_slice(s, start, end_opt)", span)?;
148            let graphemes = split_graphemes(&s);
149            let len = graphemes.len();
150            let s_idx = resolve_slice_bound(start, len);
151            let e_idx = extract_opt_i64(args, 2)
152                .map(|e| resolve_slice_bound(e, len))
153                .unwrap_or(len);
154            let sliced = if s_idx < e_idx {
155                graphemes[s_idx..e_idx].concat()
156            } else {
157                String::new()
158            };
159            Ok(Some(Value::String(sliced)))
160        }
161        "str.code_points" => {
162            let s = extract_str(args, 0, "str.code_points(s)", span)?;
163            let cps = s.chars().map(|c| Value::I64(c as u32 as i64)).collect();
164            Ok(Some(Value::List(cps)))
165        }
166        "str.bytes" => {
167            let s = extract_str(args, 0, "str.bytes(s)", span)?;
168            let bs = s.as_bytes().iter().map(|&b| Value::I64(b as i64)).collect();
169            Ok(Some(Value::List(bs)))
170        }
171        "str.trim" => {
172            let s = extract_str(args, 0, "str.trim(s)", span)?;
173            Ok(Some(Value::String(s.trim().to_string())))
174        }
175        "str.trim_start" => {
176            let s = extract_str(args, 0, "str.trim_start(s)", span)?;
177            Ok(Some(Value::String(s.trim_start().to_string())))
178        }
179        "str.trim_end" => {
180            let s = extract_str(args, 0, "str.trim_end(s)", span)?;
181            Ok(Some(Value::String(s.trim_end().to_string())))
182        }
183        "str.pad_start" => {
184            let s = extract_str(args, 0, "str.pad_start(s, target_len, fill)", span)?;
185            let target_len =
186                extract_i64(args, 1, "str.pad_start(s, target_len, fill)", span)?.max(0) as usize;
187            let fill = args
188                .get(2)
189                .and_then(|v| match v {
190                    Value::String(f) if !f.is_empty() => Some(f.clone()),
191                    _ => None,
192                })
193                .unwrap_or_else(|| " ".to_string());
194            Ok(Some(Value::String(pad_graphemes(&s, target_len, &fill, true))))
195        }
196        "str.pad_end" => {
197            let s = extract_str(args, 0, "str.pad_end(s, target_len, fill)", span)?;
198            let target_len =
199                extract_i64(args, 1, "str.pad_end(s, target_len, fill)", span)?.max(0) as usize;
200            let fill = args
201                .get(2)
202                .and_then(|v| match v {
203                    Value::String(f) if !f.is_empty() => Some(f.clone()),
204                    _ => None,
205                })
206                .unwrap_or_else(|| " ".to_string());
207            Ok(Some(Value::String(pad_graphemes(&s, target_len, &fill, false))))
208        }
209        "str.split" => {
210            let s = extract_str(args, 0, "str.split(s, sep)", span)?;
211            let sep = extract_str(args, 1, "str.split(s, sep)", span)?;
212            let parts: Vec<Value> = if sep.is_empty() {
213                split_graphemes(&s).into_iter().map(Value::String).collect()
214            } else {
215                s.split(&sep).map(|p| Value::String(p.to_string())).collect()
216            };
217            Ok(Some(Value::List(parts)))
218        }
219        "str.join" => {
220            let list = extract_list(args, 0, "str.join(list, sep)", span)?;
221            let sep = extract_str(args, 1, "str.join(list, sep)", span)?;
222            let mut strs = Vec::with_capacity(list.len());
223            for item in &list {
224                match item {
225                    Value::String(s) | Value::Iri(s) => strs.push(s.clone()),
226                    other => strs.push(format!("{other}")),
227                }
228            }
229            Ok(Some(Value::String(strs.join(&sep))))
230        }
231        "str.replace_literal" => {
232            let s = extract_str(args, 0, "str.replace_literal(s, pat, rep)", span)?;
233            let pat = extract_str(args, 1, "str.replace_literal(s, pat, rep)", span)?;
234            let rep = extract_str(args, 2, "str.replace_literal(s, pat, rep)", span)?;
235            Ok(Some(Value::String(s.replace(&pat, &rep))))
236        }
237        "str.starts_with" => {
238            let s = extract_str(args, 0, "str.starts_with(s, prefix)", span)?;
239            let prefix = extract_str(args, 1, "str.starts_with(s, prefix)", span)?;
240            Ok(Some(Value::Bool(s.starts_with(&prefix))))
241        }
242        "str.ends_with" => {
243            let s = extract_str(args, 0, "str.ends_with(s, suffix)", span)?;
244            let suffix = extract_str(args, 1, "str.ends_with(s, suffix)", span)?;
245            Ok(Some(Value::Bool(s.ends_with(&suffix))))
246        }
247        "str.contains" => {
248            let s = extract_str(args, 0, "str.contains(s, sub)", span)?;
249            let sub = extract_str(args, 1, "str.contains(s, sub)", span)?;
250            Ok(Some(Value::Bool(s.contains(&sub))))
251        }
252        "str.to_upper" => {
253            let s = extract_str(args, 0, "str.to_upper(s)", span)?;
254            Ok(Some(Value::String(s.to_uppercase())))
255        }
256        "str.to_lower" => {
257            let s = extract_str(args, 0, "str.to_lower(s)", span)?;
258            Ok(Some(Value::String(s.to_lowercase())))
259        }
260        _ => Ok(None),
261    }
262}
263
264fn pad_graphemes(s: &str, target_len: usize, fill: &str, pad_left: bool) -> String {
265    let gs = split_graphemes(s);
266    if gs.len() >= target_len {
267        return s.to_string();
268    }
269    let fill_gs = split_graphemes(fill);
270    if fill_gs.is_empty() {
271        return s.to_string();
272    }
273    let needed = target_len - gs.len();
274    let mut pad_str = String::new();
275    for i in 0..needed {
276        pad_str.push_str(&fill_gs[i % fill_gs.len()]);
277    }
278    if pad_left {
279        format!("{pad_str}{s}")
280    } else {
281        format!("{s}{pad_str}")
282    }
283}