fix(parser): fix duplicated comments during parser rewind (#2600)

closes #2592

crates/oxc_parser/src/lexer/mod.rs

@@ -37,7 +37,7 @@ use oxc_span::{SourceType, Span};
 use self::{
     byte_handlers::handle_byte,
     source::{Source, SourcePosition},
-    trivia_builder::TriviaBuilder,
+    trivia_builder::{TriviaBuilder, TriviasCheckpoint},
 };
 pub use self::{
     kind::Kind,
@@ -54,6 +54,8 @@ pub struct LexerCheckpoint<'a> {
     token: Token,
 
     errors_pos: usize,
+
+    trivias: TriviasCheckpoint,
 }
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
@@ -153,12 +155,14 @@ impl<'a> Lexer<'a> {
             position: self.source.position(),
             token: self.token,
             errors_pos: self.errors.len(),
+            trivias: self.trivia_builder.checkpoint(),
         }
     }
 
     /// Rewinds the lexer to the same state as when the passed in `checkpoint` was created.
     pub fn rewind(&mut self, checkpoint: LexerCheckpoint<'a>) {
         self.errors.truncate(checkpoint.errors_pos);
+        self.trivia_builder.rewind(checkpoint.trivias);
         self.source.set_position(checkpoint.position);
         self.token = checkpoint.token;
         self.lookahead.clear();

crates/oxc_parser/src/lexer/trivia_builder.rs

@@ -6,11 +6,29 @@ pub struct TriviaBuilder {
     trivias: Trivias,
 }
 
+#[derive(Debug, Clone, Copy)]
+pub struct TriviasCheckpoint {
+    comments_len: usize,
+    irregular_whitespaces_len: usize,
+}
+
 impl TriviaBuilder {
     pub fn build(self) -> Trivias {
         self.trivias
     }
 
+    pub fn checkpoint(&self) -> TriviasCheckpoint {
+        TriviasCheckpoint {
+            comments_len: self.trivias.comments.len(),
+            irregular_whitespaces_len: self.trivias.irregular_whitespaces.len(),
+        }
+    }
+
+    pub fn rewind(&mut self, checkpoint: TriviasCheckpoint) {
+        self.trivias.comments.truncate(checkpoint.comments_len);
+        self.trivias.irregular_whitespaces.truncate(checkpoint.irregular_whitespaces_len);
+    }
+
     /// skip leading `//`
     pub fn add_single_line_comment(&mut self, start: u32, end: u32) {
         self.trivias.comments.push((start + 2, end, CommentKind::SingleLine));

crates/oxc_parser/src/lib.rs

@@ -407,6 +407,8 @@ impl<'a> ParserImpl<'a> {
 #[cfg(test)]
 mod test {
 
+    use oxc_ast::CommentKind;
+
     use super::*;
 
     #[test]
@@ -450,6 +452,22 @@ mod test {
         }
     }
 
+    #[test]
+    fn comments() {
+        let allocator = Allocator::default();
+        let source_type = SourceType::default().with_typescript(true);
+        let sources = [
+            ("// line comment", CommentKind::SingleLine),
+            ("/* line comment */", CommentKind::MultiLine),
+            ("type Foo = ( /* Require properties which are not generated automatically. */ 'bar')", CommentKind::MultiLine),
+        ];
+        for (source, kind) in sources {
+            let ret = Parser::new(&allocator, source, source_type).parse();
+            assert_eq!(ret.trivias.comments.len(), 1, "{source}");
+            assert_eq!(ret.trivias.comments[0].2, kind, "{source}");
+        }
+    }
+
     #[test]
     fn memory_leak() {
         let allocator = Allocator::default();

tasks/coverage/prettier_babel.snap

@@ -1,9 +1,8 @@
 prettier_babel Summary:
 AST Parsed     : 2096/2096 (100.00%)
-Positive Passed: 1903/2096 (90.79%)
+Positive Passed: 1904/2096 (90.84%)
 Expect to Parse: "comments/attachComment-false/array-expression-trailing-comma/input.js"
 Expect to Parse: "comments/basic/array-expression-trailing-comma/input.js"
-Expect to Parse: "comments/basic/object-accessor-computed/input.js"
 Expect to Parse: "comments/basic/object-expression-trailing-comma/input.js"
 Expect to Parse: "comments/basic/object-method/input.js"
 Expect to Parse: "comments/basic/object-method-async-generator/input.js"

tasks/coverage/prettier_test262.snap

@@ -1,6 +1,6 @@
 prettier_test262 Summary:
 AST Parsed     : 45836/45836 (100.00%)
-Positive Passed: 42643/45836 (93.03%)
+Positive Passed: 42652/45836 (93.05%)
 Expect to Parse: "annexB/built-ins/RegExp/RegExp-leading-escape-BMP.js"
 Expect to Parse: "annexB/built-ins/RegExp/RegExp-trailing-escape-BMP.js"
 Expect to Parse: "annexB/built-ins/escape/to-primitive-err.js"
@@ -389,7 +389,6 @@ Expect to Parse: "built-ins/AsyncGeneratorPrototype/return/this-val-not-async-ge
 Expect to Parse: "built-ins/AsyncGeneratorPrototype/return/this-val-not-object.js"
 Expect to Parse: "built-ins/AsyncGeneratorPrototype/throw/this-val-not-async-generator.js"
 Expect to Parse: "built-ins/AsyncGeneratorPrototype/throw/this-val-not-object.js"
-Expect to Parse: "built-ins/Atomics/notify/undefined-index-defaults-to-zero.js"
 Expect to Parse: "built-ins/Atomics/waitAsync/bigint/false-for-timeout.js"
 Expect to Parse: "built-ins/Atomics/waitAsync/bigint/null-for-timeout.js"
 Expect to Parse: "built-ins/Atomics/waitAsync/bigint/object-for-timeout.js"
@@ -1824,7 +1823,6 @@ Expect to Parse: "language/expressions/array/S11.1.4_A2.js"
 Expect to Parse: "language/expressions/array/spread-obj-override-immutable.js"
 Expect to Parse: "language/expressions/arrow-function/dstr/dflt-obj-ptrn-prop-eval-err.js"
 Expect to Parse: "language/expressions/arrow-function/dstr/obj-ptrn-prop-eval-err.js"
-Expect to Parse: "language/expressions/arrow-function/forbidden-ext/b2/arrow-function-forbidden-ext-indirect-access-own-prop-caller-get.js"
 Expect to Parse: "language/expressions/arrow-function/prototype-rules.js"
 Expect to Parse: "language/expressions/assignment/8.12.5-3-b_2.js"
 Expect to Parse: "language/expressions/assignment/S11.13.1_A2.1_T1.js"
@@ -1842,15 +1840,10 @@ Expect to Parse: "language/expressions/assignment/dstr/array-iteration.js"
 Expect to Parse: "language/expressions/assignment/dstr/obj-prop-name-evaluation-error.js"
 Expect to Parse: "language/expressions/assignment/dstr/obj-prop-name-evaluation.js"
 Expect to Parse: "language/expressions/assignment/dstr/obj-rest-descriptors.js"
-Expect to Parse: "language/expressions/async-arrow-function/forbidden-ext/b2/async-arrow-function-forbidden-ext-indirect-access-own-prop-caller-get.js"
-Expect to Parse: "language/expressions/async-function/forbidden-ext/b2/async-func-expr-named-forbidden-ext-indirect-access-own-prop-caller-get.js"
-Expect to Parse: "language/expressions/async-function/forbidden-ext/b2/async-func-expr-nameless-forbidden-ext-indirect-access-own-prop-caller-get.js"
 Expect to Parse: "language/expressions/async-generator/dstr/dflt-obj-ptrn-prop-eval-err.js"
 Expect to Parse: "language/expressions/async-generator/dstr/named-dflt-obj-ptrn-prop-eval-err.js"
 Expect to Parse: "language/expressions/async-generator/dstr/named-obj-ptrn-prop-eval-err.js"
 Expect to Parse: "language/expressions/async-generator/dstr/obj-ptrn-prop-eval-err.js"
-Expect to Parse: "language/expressions/async-generator/forbidden-ext/b2/async-gen-func-expr-forbidden-ext-indirect-access-own-prop-caller-get.js"
-Expect to Parse: "language/expressions/async-generator/forbidden-ext/b2/async-gen-named-func-expr-forbidden-ext-indirect-access-own-prop-caller-get.js"
 Expect to Parse: "language/expressions/async-generator/named-yield-star-async-next.js"
 Expect to Parse: "language/expressions/async-generator/named-yield-star-async-return.js"
 Expect to Parse: "language/expressions/async-generator/named-yield-star-async-throw.js"
@@ -2289,10 +2282,8 @@ Expect to Parse: "language/expressions/equals/S9.1_A1_T3.js"
 Expect to Parse: "language/expressions/equals/to-prim-hint.js"
 Expect to Parse: "language/expressions/function/dstr/dflt-obj-ptrn-prop-eval-err.js"
 Expect to Parse: "language/expressions/function/dstr/obj-ptrn-prop-eval-err.js"
-Expect to Parse: "language/expressions/function/forbidden-ext/b2/func-expr-forbidden-ext-indirect-access-own-prop-caller-get.js"
 Expect to Parse: "language/expressions/generators/dstr/dflt-obj-ptrn-prop-eval-err.js"
 Expect to Parse: "language/expressions/generators/dstr/obj-ptrn-prop-eval-err.js"
-Expect to Parse: "language/expressions/generators/forbidden-ext/b2/gen-func-expr-forbidden-ext-indirect-access-own-prop-caller-get.js"
 Expect to Parse: "language/expressions/greater-than/S11.8.2_A2.2_T1.js"
 Expect to Parse: "language/expressions/greater-than/S11.8.2_A2.4_T2.js"
 Expect to Parse: "language/expressions/greater-than/S11.8.2_A3.2_T1.2.js"

tasks/coverage/prettier_typescript.snap

@@ -1,6 +1,6 @@
 prettier_typescript Summary:
 AST Parsed     : 5243/5243 (100.00%)
-Positive Passed: 2864/5243 (54.63%)
+Positive Passed: 2871/5243 (54.76%)
 Expect to Parse: "compiler/DeclarationErrorsNoEmitOnError.ts"
 Expect to Parse: "compiler/abstractInterfaceIdentifierName.ts"
 Expect to Parse: "compiler/abstractPropertyBasics.ts"
@@ -43,7 +43,6 @@ Expect to Parse: "compiler/arrayLiteralContextualType.ts"
 Expect to Parse: "compiler/arrayTypeInSignatureOfInterfaceAndClass.ts"
 Expect to Parse: "compiler/arrayconcat.ts"
 Expect to Parse: "compiler/arrowFunctionInExpressionStatement2.ts"
-Expect to Parse: "compiler/arrowFunctionParsingDoesNotConfuseParenthesizedObjectForArrowHead.ts"
 Expect to Parse: "compiler/arrowFunctionWithObjectLiteralBody5.ts"
 Expect to Parse: "compiler/arrowFunctionWithObjectLiteralBody6.ts"
 Expect to Parse: "compiler/assertionFunctionsCanNarrowByDiscriminant.ts"
@@ -1762,7 +1761,6 @@ Expect to Parse: "conformance/expressions/unaryOperators/voidOperator/voidOperat
 Expect to Parse: "conformance/expressions/unaryOperators/voidOperator/voidOperatorWithEnumType.ts"
 Expect to Parse: "conformance/expressions/unaryOperators/voidOperator/voidOperatorWithNumberType.ts"
 Expect to Parse: "conformance/expressions/unaryOperators/voidOperator/voidOperatorWithStringType.ts"
-Expect to Parse: "conformance/externalModules/asiPreventsParsingAsAmbientExternalModule01.ts"
 Expect to Parse: "conformance/externalModules/asiPreventsParsingAsAmbientExternalModule02.ts"
 Expect to Parse: "conformance/externalModules/es6/es6modulekindWithES5Target12.ts"
 Expect to Parse: "conformance/externalModules/es6/es6modulekindWithES5Target5.ts"
@@ -1789,10 +1787,7 @@ Expect to Parse: "conformance/interfaces/declarationMerging/mergedInterfacesWith
 Expect to Parse: "conformance/interfaces/declarationMerging/mergedInterfacesWithMultipleBases3.ts"
 Expect to Parse: "conformance/interfaces/declarationMerging/twoMergedInterfacesWithDifferingOverloads.ts"
 Expect to Parse: "conformance/interfaces/declarationMerging/twoMergedInterfacesWithDifferingOverloads2.ts"
-Expect to Parse: "conformance/interfaces/interfaceDeclarations/asiPreventsParsingAsInterface01.ts"
-Expect to Parse: "conformance/interfaces/interfaceDeclarations/asiPreventsParsingAsInterface02.ts"
 Expect to Parse: "conformance/interfaces/interfaceDeclarations/asiPreventsParsingAsInterface03.ts"
-Expect to Parse: "conformance/interfaces/interfaceDeclarations/asiPreventsParsingAsInterface04.ts"
 Expect to Parse: "conformance/interfaces/interfaceDeclarations/derivedInterfaceDoesNotHideBaseSignatures.ts"
 Expect to Parse: "conformance/interfaces/interfaceDeclarations/interfaceExtendsObjectIntersection.ts"
 Expect to Parse: "conformance/interfaces/interfaceDeclarations/interfaceThatHidesBaseProperty.ts"
@@ -1835,8 +1830,6 @@ Expect to Parse: "conformance/internalModules/exportDeclarations/ExportVariableW
 Expect to Parse: "conformance/internalModules/importDeclarations/exportImportAlias.ts"
 Expect to Parse: "conformance/internalModules/importDeclarations/importAliasIdentifiers.ts"
 Expect to Parse: "conformance/internalModules/moduleBody/moduleWithStatementsOfEveryKind.ts"
-Expect to Parse: "conformance/internalModules/moduleDeclarations/asiPreventsParsingAsNamespace01.ts"
-Expect to Parse: "conformance/internalModules/moduleDeclarations/asiPreventsParsingAsNamespace02.ts"
 Expect to Parse: "conformance/internalModules/moduleDeclarations/asiPreventsParsingAsNamespace03.ts"
 Expect to Parse: "conformance/internalModules/moduleDeclarations/asiPreventsParsingAsNamespace05.ts"
 Expect to Parse: "conformance/internalModules/moduleDeclarations/instantiatedModule.ts"