fix(parser): remove all duplicated comments in trivia builder (#2689)

crates/oxc_parser/src/lexer/mod.rs

@@ -37,7 +37,7 @@ use oxc_span::{SourceType, Span};
 use self::{
     byte_handlers::handle_byte,
     source::{Source, SourcePosition},
-    trivia_builder::{TriviaBuilder, TriviasCheckpoint},
+    trivia_builder::TriviaBuilder,
 };
 pub use self::{
     kind::Kind,
@@ -54,8 +54,6 @@ pub struct LexerCheckpoint<'a> {
     token: Token,
 
     errors_pos: usize,
-
-    trivias: TriviasCheckpoint,
 }
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
@@ -155,14 +153,12 @@ impl<'a> Lexer<'a> {
             position: self.source.position(),
             token: self.token,
             errors_pos: self.errors.len(),
-            trivias: self.trivia_builder.checkpoint(),
         }
     }
 
     /// Rewinds the lexer to the same state as when the passed in `checkpoint` was created.
     pub fn rewind(&mut self, checkpoint: LexerCheckpoint<'a>) {
         self.errors.truncate(checkpoint.errors_pos);
-        self.trivia_builder.rewind(checkpoint.trivias);
         self.source.set_position(checkpoint.position);
         self.token = checkpoint.token;
         self.lookahead.clear();

crates/oxc_parser/src/lexer/trivia_builder.rs

@@ -1,45 +1,38 @@
+use std::collections::BTreeMap;
+
 use oxc_ast::{CommentKind, Trivias};
 use oxc_span::Span;
 
 #[derive(Debug, Default)]
 pub struct TriviaBuilder {
-    trivias: Trivias,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct TriviasCheckpoint {
-    comments_len: usize,
-    irregular_whitespaces_len: usize,
+    // Duplicated comments can be added from rewind, use `BTreeMap` to ensure uniqueness
+    comments: BTreeMap<u32, (u32, CommentKind)>,
+    irregular_whitespaces: Vec<Span>,
 }
 
 impl TriviaBuilder {
     pub fn build(self) -> Trivias {
-        self.trivias
-    }
-
-    pub fn checkpoint(&self) -> TriviasCheckpoint {
-        TriviasCheckpoint {
-            comments_len: self.trivias.comments.len(),
-            irregular_whitespaces_len: self.trivias.irregular_whitespaces.len(),
+        Trivias {
+            comments: self
+                .comments
+                .into_iter()
+                .map(|(start, (end, kind))| (start, end, kind))
+                .collect(),
+            irregular_whitespaces: self.irregular_whitespaces,
         }
     }
 
-    pub fn rewind(&mut self, checkpoint: TriviasCheckpoint) {
-        self.trivias.comments.truncate(checkpoint.comments_len);
-        self.trivias.irregular_whitespaces.truncate(checkpoint.irregular_whitespaces_len);
-    }
-
-    /// skip leading `//`
     pub fn add_single_line_comment(&mut self, start: u32, end: u32) {
-        self.trivias.comments.push((start + 2, end, CommentKind::SingleLine));
+        // skip leading `//`
+        self.comments.insert(start + 2, (end, CommentKind::SingleLine));
     }
 
-    /// skip leading `/*` and trailing `*/`
     pub fn add_multi_line_comment(&mut self, start: u32, end: u32) {
-        self.trivias.comments.push((start + 2, end - 2, CommentKind::MultiLine));
+        // skip leading `/*` and trailing `*/`
+        self.comments.insert(start + 2, (end - 2, CommentKind::MultiLine));
     }
 
     pub fn add_irregular_whitespace(&mut self, start: u32, end: u32) {
-        self.trivias.irregular_whitespaces.push(Span::new(start, end));
+        self.irregular_whitespaces.push(Span::new(start, end));
     }
 }

tasks/coverage/codegen_misc.snap

@@ -1,3 +1,3 @@
 codegen_misc Summary:
-AST Parsed     : 12/12 (100.00%)
-Positive Passed: 12/12 (100.00%)
+AST Parsed     : 14/14 (100.00%)
+Positive Passed: 14/14 (100.00%)

tasks/coverage/misc/pass/oxc-2592.ts

@@ -0,0 +1,4 @@
+export type Foo = (
+    // comment
+    "bar"
+)

tasks/coverage/misc/pass/oxc-2674.tsx

@@ -0,0 +1,9 @@
+const foo = <div
+    // comment
+/>;
+
+<div-
+    // comment
+/>;
+
+<div-- /* comment */ />;

tasks/coverage/parser_babel.snap

@@ -1,6 +1,6 @@
 parser_babel Summary:
 AST Parsed     : 2090/2096 (99.71%)
-Positive Passed: 2080/2096 (99.24%)
+Positive Passed: 2083/2096 (99.38%)
 Negative Passed: 1362/1500 (90.80%)
 Expect Syntax Error: "annex-b/disabled/1.1-html-comments-close/input.js"
 Expect Syntax Error: "annex-b/disabled/3.1-sloppy-labeled-functions/input.js"
@@ -140,9 +140,6 @@ Expect Syntax Error: "typescript/types/read-only-3/input.ts"
 Expect Syntax Error: "typescript/types/read-only-4/input.ts"
 Expect Syntax Error: "typescript/types/tuple-optional-invalid/input.ts"
 Expect Syntax Error: "typescript/types/tuple-required-after-labeled-optional/input.ts"
-Duplicated comments " 3 ": "comments/basic/nested-parentheses/input.js"
-Duplicated comments " 1 ": "comments/basic/object-method/input.js"
-Duplicated comments " 3 ": "comments/basic/sequence-expression/input.js"
 Expect to Parse: "core/opts/allowNewTargetOutsideFunction-true/input.js"
 
   × Unexpected new.target expression

tasks/coverage/parser_misc.snap

@@ -1,6 +1,6 @@
 parser_misc Summary:
-AST Parsed     : 12/12 (100.00%)
-Positive Passed: 12/12 (100.00%)
+AST Parsed     : 14/14 (100.00%)
+Positive Passed: 14/14 (100.00%)
 Negative Passed: 8/8 (100.00%)
 
   × Unexpected token

tasks/coverage/parser_typescript.snap

@@ -1,6 +1,6 @@
 parser_typescript Summary:
 AST Parsed     : 5240/5243 (99.94%)
-Positive Passed: 5231/5243 (99.77%)
+Positive Passed: 5233/5243 (99.81%)
 Negative Passed: 1062/4879 (21.77%)
 Expect Syntax Error: "compiler/ClassDeclaration10.ts"
 Expect Syntax Error: "compiler/ClassDeclaration11.ts"
@@ -2697,7 +2697,7 @@ Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardOfFormTypeOfEq
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardOfFormTypeOfNotEqualHasNoEffect.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardOfFormTypeOfOther.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsDefeat.ts"
-Duplicated comments " change value of x": "conformance/expressions/typeGuards/typeGuardsInIfStatement.ts"
+Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsInIfStatement.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsWithAny.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsWithInstanceOf.ts"
 Expect Syntax Error: "conformance/expressions/typeGuards/typeGuardsWithInstanceOfByConstructorSignature.ts"
@@ -3819,7 +3819,6 @@ Expect Syntax Error: "conformance/types/unknown/unknownControlFlow.ts"
 Expect Syntax Error: "conformance/types/unknown/unknownType1.ts"
 Expect Syntax Error: "conformance/types/unknown/unknownType2.ts"
 Expect Syntax Error: "conformance/types/witness/witness.ts"
-Duplicated comments "comment2": "compiler/awaitExpressionInnerCommentEmit.ts"
 Expect to Parse: "compiler/bom-utf16be.ts"
 
   × Invalid Character `￾`
@@ -3854,7 +3853,6 @@ Expect to Parse: "compiler/emitBundleWithShebangAndPrologueDirectives1.ts"
    ·  ─
  7 │ "use strict"
    ╰────
-Duplicated comments " must be identifier?": "compiler/jsxNestedWithinTernaryParsesCorrectly.tsx"
 Expect to Parse: "compiler/withStatementInternalComments.ts"
 
   × 'with' statements are not allowed

tasks/coverage/prettier_misc.snap

@@ -1,8 +1,9 @@
 prettier_misc Summary:
-AST Parsed     : 12/12 (100.00%)
-Positive Passed: 7/12 (58.33%)
+AST Parsed     : 14/14 (100.00%)
+Positive Passed: 8/14 (57.14%)
 Expect to Parse: "pass/oxc-1740.tsx"
 Expect to Parse: "pass/oxc-2087.ts"
 Expect to Parse: "pass/oxc-2394.ts"
+Expect to Parse: "pass/oxc-2674.tsx"
 Expect to Parse: "pass/swc-1627.js"
 Expect to Parse: "pass/swc-8243.tsx"

tasks/coverage/prettier_typescript.snap

@@ -1,6 +1,6 @@
 prettier_typescript Summary:
 AST Parsed     : 5243/5243 (100.00%)
-Positive Passed: 2871/5243 (54.76%)
+Positive Passed: 2872/5243 (54.78%)
 Expect to Parse: "compiler/DeclarationErrorsNoEmitOnError.ts"
 Expect to Parse: "compiler/abstractInterfaceIdentifierName.ts"
 Expect to Parse: "compiler/abstractPropertyBasics.ts"
@@ -72,7 +72,6 @@ Expect to Parse: "compiler/augmentedTypesModules3b.ts"
 Expect to Parse: "compiler/augmentedTypesModules4.ts"
 Expect to Parse: "compiler/autonumberingInEnums.ts"
 Expect to Parse: "compiler/avoidCycleWithVoidExpressionReturnedFromArrow.ts"
-Expect to Parse: "compiler/awaitExpressionInnerCommentEmit.ts"
 Expect to Parse: "compiler/awaitUnionPromise.ts"
 Expect to Parse: "compiler/awaitedTypeCrash.ts"
 Expect to Parse: "compiler/awaitedTypeJQuery.ts"