Skip to main content

vibe/
trivia.rs

1//! CST trivia — comments, whitespace, and exact token text (T36).
2//!
3//! The AST drops comments and whitespace. The CST preserves them so
4//! that `poet translate` and `poet format` can round-trip source text
5//! without destroying commentary.
6//!
7//! ## Design
8//!
9//! - [`Trivia`] represents a comment or whitespace gap.
10//! - [`TriviaSink`] collects trivia as the lexer advances.
11//! - [`CstNode`] wraps an AST node with its leading and trailing trivia.
12//!
13//! The trivia sink is optional — the lexer can run in "AST mode"
14//! (trivia discarded, current behaviour) or "CST mode" (trivia
15//! collected). This avoids changing existing behaviour.
16//!
17//! Reference: `docs/vibescript-full-impl-PLAN.md` §8.8 T36.
18
19use crate::span::Span;
20
21/// The kind of trivia.
22#[derive(Debug, Clone, PartialEq, Eq)]
23pub enum TriviaKind {
24    /// A line comment: `// ...`
25    LineComment,
26    /// A block comment: `/* ... */`
27    BlockComment,
28    /// Whitespace (spaces, tabs, newlines).
29    Whitespace,
30}
31
32/// A single trivia element — a comment or whitespace gap.
33#[derive(Debug, Clone, PartialEq, Eq)]
34pub struct Trivia {
35    /// The kind of trivia.
36    pub kind: TriviaKind,
37    /// The exact source text of this trivia (including delimiters).
38    pub text: String,
39    /// The span of this trivia in the source.
40    pub span: Span,
41}
42
43impl Trivia {
44    /// Create a line comment trivia.
45    pub fn line_comment(text: &str, span: Span) -> Self {
46        Self {
47            kind: TriviaKind::LineComment,
48            text: text.to_string(),
49            span,
50        }
51    }
52
53    /// Create a block comment trivia.
54    pub fn block_comment(text: &str, span: Span) -> Self {
55        Self {
56            kind: TriviaKind::BlockComment,
57            text: text.to_string(),
58            span,
59        }
60    }
61
62    /// Create a whitespace trivia.
63    pub fn whitespace(text: &str, span: Span) -> Self {
64        Self {
65            kind: TriviaKind::Whitespace,
66            text: text.to_string(),
67            span,
68        }
69    }
70
71    /// Is this trivia a comment?
72    pub fn is_comment(&self) -> bool {
73        matches!(
74            self.kind,
75            TriviaKind::LineComment | TriviaKind::BlockComment
76        )
77    }
78
79    /// Is this trivia whitespace?
80    pub fn is_whitespace(&self) -> bool {
81        matches!(self.kind, TriviaKind::Whitespace)
82    }
83}
84
85/// A sink that collects trivia as the lexer advances.
86/// Use [`TriviaSink::new`] to create a collecting sink, or
87/// [`TriviaSink::disabled`] to create a discarding sink.
88#[derive(Debug, Clone)]
89pub struct TriviaSink {
90    entries: Vec<Trivia>,
91    enabled: bool,
92}
93
94impl Default for TriviaSink {
95    fn default() -> Self {
96        Self::disabled()
97    }
98}
99
100impl TriviaSink {
101    /// Create a trivia sink that collects trivia.
102    pub fn new() -> Self {
103        Self {
104            entries: Vec::new(),
105            enabled: true,
106        }
107    }
108
109    /// Create a trivia sink that discards trivia (AST mode).
110    pub fn disabled() -> Self {
111        Self {
112            entries: Vec::new(),
113            enabled: false,
114        }
115    }
116
117    /// Is this sink collecting trivia?
118    pub fn is_enabled(&self) -> bool {
119        self.enabled
120    }
121
122    /// Record a trivia element. If the sink is disabled, this is a no-op.
123    pub fn record(&mut self, trivia: Trivia) {
124        if self.enabled {
125            self.entries.push(trivia);
126        }
127    }
128
129    /// Record a line comment.
130    pub fn record_line_comment(&mut self, text: &str, span: Span) {
131        if self.enabled {
132            self.record(Trivia::line_comment(text, span));
133        }
134    }
135
136    /// Record a block comment.
137    pub fn record_block_comment(&mut self, text: &str, span: Span) {
138        if self.enabled {
139            self.record(Trivia::block_comment(text, span));
140        }
141    }
142
143    /// Record whitespace.
144    pub fn record_whitespace(&mut self, text: &str, span: Span) {
145        if self.enabled {
146            self.record(Trivia::whitespace(text, span));
147        }
148    }
149
150    /// Get all collected trivia.
151    pub fn entries(&self) -> &[Trivia] {
152        &self.entries
153    }
154
155    /// Take all collected trivia, leaving the sink empty.
156    pub fn take(&mut self) -> Vec<Trivia> {
157        std::mem::take(&mut self.entries)
158    }
159
160    /// Number of trivia entries collected.
161    pub fn len(&self) -> usize {
162        self.entries.len()
163    }
164
165    /// Is the sink empty?
166    pub fn is_empty(&self) -> bool {
167        self.entries.is_empty()
168    }
169
170    /// Get only the comment trivia (line + block).
171    pub fn comments(&self) -> Vec<&Trivia> {
172        self.entries.iter().filter(|t| t.is_comment()).collect()
173    }
174
175    /// Get only the whitespace trivia.
176    pub fn whitespace(&self) -> Vec<&Trivia> {
177        self.entries.iter().filter(|t| t.is_whitespace()).collect()
178    }
179}
180
181/// A CST node — an AST node with its leading and trailing trivia.
182///
183/// This is a generic wrapper that preserves the trivia around a node
184/// so that the source can be reconstructed.
185#[derive(Debug, Clone)]
186pub struct CstNode<T> {
187    /// The AST node itself.
188    pub node: T,
189    /// Trivia before this node (comments + whitespace).
190    pub leading: Vec<Trivia>,
191    /// Trivia after this node (comments + whitespace).
192    pub trailing: Vec<Trivia>,
193}
194
195impl<T> CstNode<T> {
196    /// Creates a new `CstNode`.
197    pub fn new(node: T, leading: Vec<Trivia>, trailing: Vec<Trivia>) -> Self {
198        Self {
199            node,
200            leading,
201            trailing,
202        }
203    }
204
205    /// Get the leading comments.
206    pub fn leading_comments(&self) -> Vec<&Trivia> {
207        self.leading.iter().filter(|t| t.is_comment()).collect()
208    }
209
210    /// Get the trailing comments.
211    pub fn trailing_comments(&self) -> Vec<&Trivia> {
212        self.trailing.iter().filter(|t| t.is_comment()).collect()
213    }
214}
215
216/// Extract trivia from source text by scanning for comments.
217///
218/// This is a standalone function that doesn't require the lexer to be
219/// in CST mode. It scans the source text and returns all comments
220/// found, with their spans. This is useful for tools that need to
221/// extract comments from already-parsed source.
222pub fn extract_comments(source: &str) -> Vec<Trivia> {
223    let bytes = source.as_bytes();
224    let mut comments = Vec::new();
225    let mut pos = 0;
226
227    while pos < bytes.len() {
228        // Line comment
229        if pos + 1 < bytes.len() && bytes[pos] == b'/' && bytes[pos + 1] == b'/' {
230            let start = pos;
231            pos += 2;
232            while pos < bytes.len() && bytes[pos] != b'\n' && bytes[pos] != b'\r' {
233                pos += 1;
234            }
235            let text = &source[start..pos];
236            comments.push(Trivia::line_comment(
237                text,
238                Span::new(start as u32, pos as u32),
239            ));
240            continue;
241        }
242
243        // Block comment
244        if pos + 1 < bytes.len() && bytes[pos] == b'/' && bytes[pos + 1] == b'*' {
245            let start = pos;
246            pos += 2;
247            let mut closed = false;
248            while pos + 1 < bytes.len() {
249                if bytes[pos] == b'*' && bytes[pos + 1] == b'/' {
250                    pos += 2;
251                    closed = true;
252                    break;
253                }
254                pos += 1;
255            }
256            if !closed {
257                // Unterminated block comment — include to end.
258                pos = bytes.len();
259            }
260            let text = &source[start..pos];
261            comments.push(Trivia::block_comment(
262                text,
263                Span::new(start as u32, pos as u32),
264            ));
265            continue;
266        }
267
268        // Skip string literals (don't look for comments inside strings)
269        if bytes[pos] == b'"' {
270            pos += 1;
271            while pos < bytes.len() && bytes[pos] != b'"' {
272                if bytes[pos] == b'\\' && pos + 1 < bytes.len() {
273                    pos += 2;
274                } else {
275                    pos += 1;
276                }
277            }
278            if pos < bytes.len() {
279                pos += 1; // skip closing quote
280            }
281            continue;
282        }
283
284        pos += 1;
285    }
286
287    comments
288}
289
290#[cfg(test)]
291mod tests {
292    use super::*;
293
294    #[test]
295    fn trivia_line_comment() {
296        let t = Trivia::line_comment("// hello", Span::new(0, 8));
297        assert!(t.is_comment());
298        assert!(!t.is_whitespace());
299        assert_eq!(t.kind, TriviaKind::LineComment);
300        assert_eq!(t.text, "// hello");
301    }
302
303    #[test]
304    fn trivia_block_comment() {
305        let t = Trivia::block_comment("/* hi */", Span::new(0, 8));
306        assert!(t.is_comment());
307        assert_eq!(t.kind, TriviaKind::BlockComment);
308    }
309
310    #[test]
311    fn trivia_whitespace() {
312        let t = Trivia::whitespace("  \n  ", Span::new(0, 5));
313        assert!(t.is_whitespace());
314        assert!(!t.is_comment());
315        assert_eq!(t.kind, TriviaKind::Whitespace);
316    }
317
318    #[test]
319    fn trivia_sink_disabled_discards() {
320        let mut sink = TriviaSink::disabled();
321        sink.record_line_comment("// test", Span::point(0));
322        assert!(sink.is_empty());
323        assert!(!sink.is_enabled());
324    }
325
326    #[test]
327    fn trivia_sink_enabled_collects() {
328        let mut sink = TriviaSink::new();
329        sink.record_line_comment("// hello", Span::new(0, 8));
330        sink.record_block_comment("/* world */", Span::new(10, 21));
331        sink.record_whitespace("  ", Span::new(22, 24));
332        assert_eq!(sink.len(), 3);
333        assert!(sink.is_enabled());
334    }
335
336    #[test]
337    fn trivia_sink_comments_filter() {
338        let mut sink = TriviaSink::new();
339        sink.record_line_comment("// c1", Span::point(0));
340        sink.record_whitespace("  ", Span::point(5));
341        sink.record_block_comment("/* c2 */", Span::point(7));
342        let comments = sink.comments();
343        assert_eq!(comments.len(), 2);
344        assert_eq!(comments[0].text, "// c1");
345        assert_eq!(comments[1].text, "/* c2 */");
346    }
347
348    #[test]
349    fn trivia_sink_whitespace_filter() {
350        let mut sink = TriviaSink::new();
351        sink.record_line_comment("// c1", Span::point(0));
352        sink.record_whitespace("  ", Span::point(5));
353        sink.record_whitespace("\n", Span::point(7));
354        let ws = sink.whitespace();
355        assert_eq!(ws.len(), 2);
356    }
357
358    #[test]
359    fn trivia_sink_take() {
360        let mut sink = TriviaSink::new();
361        sink.record_line_comment("// c1", Span::point(0));
362        let taken = sink.take();
363        assert_eq!(taken.len(), 1);
364        assert!(sink.is_empty());
365    }
366
367    #[test]
368    fn cst_node_construction() {
369        let node = 42i32;
370        let leading = vec![Trivia::line_comment("// pre", Span::point(0))];
371        let trailing = vec![Trivia::line_comment("// post", Span::point(10))];
372        let cst = CstNode::new(node, leading, trailing);
373        assert_eq!(cst.node, 42);
374        assert_eq!(cst.leading.len(), 1);
375        assert_eq!(cst.trailing.len(), 1);
376        assert_eq!(cst.leading_comments().len(), 1);
377        assert_eq!(cst.trailing_comments().len(), 1);
378    }
379
380    #[test]
381    fn extract_comments_line_comments() {
382        let src = "// first\nlet x = 1; // second\n";
383        let comments = extract_comments(src);
384        assert_eq!(comments.len(), 2);
385        assert_eq!(comments[0].text, "// first");
386        assert_eq!(comments[1].text, "// second");
387    }
388
389    #[test]
390    fn extract_comments_block_comments() {
391        let src = "/* a */ let x = 1; /* b */";
392        let comments = extract_comments(src);
393        assert_eq!(comments.len(), 2);
394        assert_eq!(comments[0].text, "/* a */");
395        assert_eq!(comments[1].text, "/* b */");
396    }
397
398    #[test]
399    fn extract_comments_mixed() {
400        let src = "// line\n/* block */\nlet x = 1; // trailing\n";
401        let comments = extract_comments(src);
402        assert_eq!(comments.len(), 3);
403        assert!(comments[0].is_comment());
404        assert!(comments[1].is_comment());
405        assert!(comments[2].is_comment());
406    }
407
408    #[test]
409    fn extract_comments_ignores_comments_in_strings() {
410        let src = "let s = \"// not a comment\"; // real comment";
411        let comments = extract_comments(src);
412        assert_eq!(comments.len(), 1);
413        assert_eq!(comments[0].text, "// real comment");
414    }
415
416    #[test]
417    fn extract_comments_empty_source() {
418        let comments = extract_comments("");
419        assert!(comments.is_empty());
420    }
421
422    #[test]
423    fn extract_comments_no_comments() {
424        let comments = extract_comments("let x = 1; let y = 2;");
425        assert!(comments.is_empty());
426    }
427
428    #[test]
429    fn extract_comments_unterminated_block() {
430        let comments = extract_comments("/* never closed");
431        assert_eq!(comments.len(), 1);
432        assert_eq!(comments[0].text, "/* never closed");
433    }
434
435    #[test]
436    fn extract_comments_spans_correct() {
437        let src = "// hi\nlet x = 1;";
438        let comments = extract_comments(src);
439        assert_eq!(comments.len(), 1);
440        assert_eq!(comments[0].span.start, 0);
441        assert_eq!(comments[0].span.end, 5); // "// hi"
442    }
443}