test(transformer): handle more exec tests (#8493)

Some of Babel's exec tests contain a return statement at top level, in order to return a Promise.

e.g. ad572fd1a1/packages/babel-plugin-transform-private-methods/test/fixtures/private-method-loose/async/exec.js

These test files could not be parsed due to illegal position of `return`, and the tests were silently excluded.

This PR:

1. Includes those tests by passing `allow_return_outside_function: true` option to parser for exec tests.
2. Includes a "transform error" message in snapshot for any exec tests which produce errors in parser/transformer.

tasks/transform_conformance/snapshots/babel_exec.snap.md

@@ -2,7 +2,7 @@ commit: 54a8389f
 
 node: v22.12.0
 
-Passed: 291 of 374 (77.81%)
+Passed: 318 of 406 (78.33%)
 
 Failures:
 
@@ -181,6 +181,11 @@ ReferenceError: _Foo_brand is not defined
     at new Foo (./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-loose-assignment-exec.test.js:8:38)
     at ./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-loose-assignment-exec.test.js:15:9
 
+./fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-loose-async-exec.test.js
+ReferenceError: _Cl_brand is not defined
+    at new Cl (./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-loose-async-exec.test.js:8:38)
+    at ./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-loose-async-exec.test.js:17:9
+
 ./fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-loose-before-fields-exec.test.js
 ReferenceError: _Cl_brand is not defined
     at new Cl (./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-loose-before-fields-exec.test.js:10:38)
@@ -220,6 +225,11 @@ ReferenceError: _Foo_brand is not defined
     at new Foo (./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsProperties-assignment-exec.test.js:8:38)
     at ./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsProperties-assignment-exec.test.js:15:9
 
+./fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsProperties-async-exec.test.js
+ReferenceError: _Cl_brand is not defined
+    at new Cl (./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsProperties-async-exec.test.js:8:38)
+    at ./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsProperties-async-exec.test.js:17:9
+
 ./fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsProperties-before-fields-exec.test.js
 ReferenceError: _Cl_brand is not defined
     at new Cl (./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsProperties-before-fields-exec.test.js:11:38)
@@ -250,6 +260,11 @@ ReferenceError: _Sub_brand is not defined
     at new Sub (./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsProperties-super-exec.test.js:16:38)
     at ./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsProperties-super-exec.test.js:29:9
 
+./fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsSymbols-async-exec.test.js
+ReferenceError: _Cl_brand is not defined
+    at new Cl (./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsSymbols-async-exec.test.js:8:38)
+    at ./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-method-privateFieldsAsSymbols-async-exec.test.js:17:9
+
 ./fixtures/babel/babel-plugin-transform-private-methods-test-fixtures-private-static-method-loose-basic-exec.test.js
 TypeError: attempted to use private field on non-instance
     at _classPrivateFieldBase (./node_modules/.pnpm/@babel+runtime@7.26.0/node_modules/@babel/runtime/helpers/classPrivateFieldLooseBase.js:2:44)
@@ -436,6 +451,14 @@ AssertionError: expected true to be false // Object.is equality
 AssertionError: expected 2 to be 5 // Object.is equality
     at ./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-private-property-in-object-test-fixtures-to-native-fields-static-shadow-exec.test.js:18:25
 
+./fixtures/babel/babel-plugin-transform-react-jsx-source-test-fixtures-react-source-basic-sample-exec.test.js
+ReferenceError: transformAsync is not defined
+    at ./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-react-jsx-source-test-fixtures-react-source-basic-sample-exec.test.js:4:16
+
+./fixtures/babel/babel-plugin-transform-react-jsx-source-test-fixtures-react-source-with-source-exec.test.js
+ReferenceError: transformAsync is not defined
+    at ./tasks/transform_conformance/fixtures/babel/babel-plugin-transform-react-jsx-source-test-fixtures-react-source-with-source-exec.test.js:4:16
+
 ./fixtures/babel/babel-preset-env-test-fixtures-plugins-integration-issue-15170-exec.test.js
 AssertionError: expected [Function] to not throw an error but 'ReferenceError: x is not defined' was thrown
     at Proxy.<anonymous> (./node_modules/.pnpm/@vitest+expect@2.1.2/node_modules/@vitest/expect/dist/index.js:1438:21)

tasks/transform_conformance/src/driver.rs

@@ -4,6 +4,7 @@ use oxc::{
     ast::ast::Program,
     codegen::{CodegenOptions, CodegenReturn},
     diagnostics::OxcDiagnostic,
+    parser::ParseOptions,
     span::SourceType,
     transformer::{TransformOptions, TransformerReturn},
     CompilerInterface,
@@ -12,12 +13,20 @@ use oxc_tasks_transform_checker::check_semantic_after_transform;
 
 pub struct Driver {
     check_semantic: bool,
+    allow_return_outside_function: bool,
     options: TransformOptions,
     printed: String,
     errors: Vec<OxcDiagnostic>,
 }
 
 impl CompilerInterface for Driver {
+    fn parse_options(&self) -> ParseOptions {
+        ParseOptions {
+            allow_return_outside_function: self.allow_return_outside_function,
+            ..Default::default()
+        }
+    }
+
     fn transform_options(&self) -> Option<&TransformOptions> {
         Some(&self.options)
     }
@@ -61,8 +70,18 @@ impl CompilerInterface for Driver {
 }
 
 impl Driver {
-    pub fn new(check_semantic: bool, options: TransformOptions) -> Self {
-        Self { check_semantic, options, printed: String::new(), errors: vec![] }
+    pub fn new(
+        check_semantic: bool,
+        allow_return_outside_function: bool,
+        options: TransformOptions,
+    ) -> Self {
+        Self {
+            check_semantic,
+            allow_return_outside_function,
+            options,
+            printed: String::new(),
+            errors: vec![],
+        }
     }
 
     pub fn errors(&mut self) -> Vec<OxcDiagnostic> {

tasks/transform_conformance/src/test_case.rs

@@ -207,7 +207,15 @@ impl TestCase {
         false
     }
 
-    fn transform(&self, mode: HelperLoaderMode) -> Result<String, String> {
+    /// Transform test case source.
+    ///
+    /// `allow_return_outside_function` is for exec tests which sometimes include `return` at top level.
+    /// This option is passed to parser to prevent it failing to pass those exec tests.
+    fn transform(
+        &self,
+        mode: HelperLoaderMode,
+        allow_return_outside_function: bool,
+    ) -> Result<String, String> {
         let path = &self.path;
         let transform_options = match &self.transform_options {
             Ok(transform_options) => transform_options,
@@ -227,8 +235,11 @@ impl TestCase {
             .as_ref()
             .and_then(|cwd| path.strip_prefix(cwd).ok().map(|p| Path::new("<CWD>").join(p)))
             .unwrap_or(path.clone());
-        let mut driver =
-            Driver::new(false, options).execute(&source_text, self.source_type, cwd_path.as_path());
+        let mut driver = Driver::new(false, allow_return_outside_function, options).execute(
+            &source_text,
+            self.source_type,
+            cwd_path.as_path(),
+        );
         let errors = driver.errors();
         if !errors.is_empty() {
             let source = NamedSource::new(
@@ -275,7 +286,7 @@ impl TestCase {
         let mut actual_errors = None;
         let mut transform_options = None;
 
-        match self.transform(HelperLoaderMode::External) {
+        match self.transform(HelperLoaderMode::External, false) {
             Err(error) => {
                 actual_errors.replace(get_babel_error(&error));
             }
@@ -365,7 +376,7 @@ impl TestCase {
         if passed {
             if let Some(options) = transform_options {
                 let mismatch_errors =
-                    Driver::new(/* check transform mismatch */ true, options)
+                    Driver::new(/* check transform mismatch */ true, false, options)
                         .execute(&input, self.source_type, &self.path)
                         .errors();
                 self.errors.extend(mismatch_errors);
@@ -381,13 +392,14 @@ impl TestCase {
             println!("Input:\n{}\n", fs::read_to_string(&self.path).unwrap());
         }
 
-        let result = match self.transform(HelperLoaderMode::Runtime) {
+        let result = match self.transform(HelperLoaderMode::Runtime, true) {
             Ok(code) => code,
             Err(error) => {
                 if filtered {
-                    println!("Transform Errors:\n{error:?}\n",);
+                    println!("Transform Errors:\n{error:?}\n");
+                    return;
                 }
-                return;
+                "throw new Error('Transform error');".to_string()
             }
         };
         self.write_to_test_files(&result);