feat(transform_conformance): show transform conformance errors (#4976)

tasks/transform_conformance/babel_exec.snap.md

@@ -1,24 +1,24 @@
 commit: 12619ffe
 
-Passed: 16/23
+Passed: 20/23
 
 # All Passed:
+* babel-plugin-transform-logical-assignment-operators
 * babel-plugin-transform-nullish-coalescing-operator
 * babel-plugin-transform-optional-catch-binding
 * babel-plugin-transform-exponentiation-operator
 * babel-plugin-transform-arrow-functions
+* babel-plugin-transform-react-jsx-source
 
 
-# babel-preset-env (7/11)
+# babel-preset-env (8/11)
 * plugins-integration/class-arrow-super-tagged-expr/exec.js
+exec failed
+
 * plugins-integration/issue-15170/exec.js
+exec failed
+
 * sanity/check-es2015-constants/exec.js
-* sanity/regex-dot-all/exec.js
+exec failed
 
-# babel-plugin-transform-logical-assignment-operators (3/4)
-* logical-assignment/null-coalescing/exec.js
-
-# babel-plugin-transform-react-jsx-source (0/2)
-* react-source/basic-sample/exec.js
-* react-source/with-source/exec.js
 

tasks/transform_conformance/oxc.snap.md

@@ -1,15 +1,149 @@
 commit: 12619ffe
 
-Passed: 31/35
+Passed: 4/35
 
 # All Passed:
-* babel-plugin-transform-optional-catch-binding
-* babel-plugin-transform-typescript
 
 
-# babel-plugin-transform-react-jsx (23/27)
+
+# babel-plugin-transform-optional-catch-binding (0/1)
+* try-catch-shadow/input.js
+Scopes mismatch after transform
+Symbols mismatch after transform
+
+
+# babel-plugin-transform-typescript (1/7)
+* class-constructor-arguments/input.ts
+ReferenceId mismatch after transform
+
+* computed-constant-value/input.ts
+Scopes mismatch after transform
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* elimination-declare/input.ts
+Bindings Mismatch:
+previous scope ScopeId(0): ["A", "ReactiveMarkerSymbol"]
+current  scope ScopeId(0): []
+
+* enum-member-reference/input.ts
+Scopes mismatch after transform
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* export-elimination/input.ts
+Scopes mismatch after transform
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* redeclarations/input.ts
+Symbols mismatch after transform
+
+
+# babel-plugin-transform-react-jsx (3/27)
 * refresh/can-handle-implicit-arrow-returns/input.jsx
-* refresh/registers-identifiers-used-in-jsx-at-definition-site/input.jsx
-* refresh/registers-identifiers-used-in-react-create-element-at-definition-site/input.jsx
-* refresh/supports-typescript-namespace-syntax/input.tsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/does-not-consider-require-like-methods-to-be-hocs/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/does-not-get-tripped-by-iifes/input.jsx
+Bindings Mismatch:
+previous scope ScopeId(0): []
+current  scope ScopeId(0): ["_s"]
+Bindings Mismatch:
+previous scope ScopeId(1): ["_s"]
+current  scope ScopeId(1): []
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/generates-signatures-for-function-declarations-calling-hooks/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/generates-signatures-for-function-expressions-calling-hooks/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/generates-valid-signature-for-exotic-ways-to-call-hooks/input.jsx
+Scopes mismatch after transform
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/ignores-complex-definitions/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/ignores-hoc-definitions/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/includes-custom-hooks-into-the-signatures/input.jsx
+Scopes mismatch after transform
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-capitalized-identifiers-in-hoc-calls/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-identifiers-used-in-jsx-at-definition-site/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-identifiers-used-in-react-create-element-at-definition-site/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-likely-hocs-with-inline-functions-1/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-likely-hocs-with-inline-functions-2/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-likely-hocs-with-inline-functions-3/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-top-level-exported-function-declarations/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-top-level-exported-named-arrow-functions/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-top-level-function-declarations/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-top-level-variable-declarations-with-arrow-functions/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/registers-top-level-variable-declarations-with-function-expressions/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/supports-typescript-namespace-syntax/input.tsx
+Scopes mismatch after transform
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/uses-custom-identifiers-for-refresh-reg-and-refresh-sig/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* refresh/uses-original-function-declaration-if-it-get-reassigned/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
+* unicode/input.jsx
+Symbols mismatch after transform
+ReferenceId mismatch after transform
+
 

tasks/transform_conformance/src/driver.rs

@@ -73,7 +73,7 @@ impl Driver {
             options,
             printed: String::new(),
             errors: vec![],
-            check_semantic: false,
+            check_semantic: true,
             checker: PostTransformChecker::default(),
         }
     }

tasks/transform_conformance/src/lib.rs

@@ -153,7 +153,11 @@ impl TestRunner {
             // Run the test
             let (passed, failed): (Vec<TestCaseKind>, Vec<TestCaseKind>) = test_cases
                 .into_iter()
-                .partition(|test_case| test_case.test(self.options.filter.is_some()));
+                .map(|mut test_case| {
+                    test_case.test(self.options.filter.is_some());
+                    test_case
+                })
+                .partition(|test_case| test_case.errors().is_empty());
             all_passed_count += passed.len();
 
             // Snapshot
@@ -168,6 +172,14 @@ impl TestRunner {
                     snapshot.push_str(&normalize_path(
                         test_case.path().strip_prefix(&case_root).unwrap(),
                     ));
+                    let errors = test_case.errors();
+                    if !errors.is_empty() {
+                        snapshot.push('\n');
+                        for error in test_case.errors() {
+                            snapshot.push_str(&error.message);
+                        }
+                        snapshot.push('\n');
+                    }
                     snapshot.push('\n');
                 }
                 snapshot.push('\n');

tasks/transform_conformance/src/test_case.rs

@@ -65,12 +65,19 @@ impl TestCaseKind {
         }
     }
 
-    pub fn test(&self, filter: bool) -> bool {
+    pub fn test(&mut self, filter: bool) {
         match self {
             Self::Transform(test_case) => test_case.test(filter),
             Self::Exec(test_case) => test_case.test(filter),
         }
     }
+
+    pub fn errors(&self) -> &Vec<OxcDiagnostic> {
+        match self {
+            Self::Transform(test_case) => test_case.errors(),
+            Self::Exec(test_case) => test_case.errors(),
+        }
+    }
 }
 
 fn transform_options(options: &BabelOptions) -> Result<TransformOptions, Vec<Error>> {
@@ -84,7 +91,9 @@ pub trait TestCase {
 
     fn transform_options(&self) -> &Result<TransformOptions, Vec<Error>>;
 
-    fn test(&self, filtered: bool) -> bool;
+    fn test(&mut self, filtered: bool);
+
+    fn errors(&self) -> &Vec<OxcDiagnostic>;
 
     fn path(&self) -> &Path;
 
@@ -178,6 +187,7 @@ pub struct ConformanceTestCase {
     path: PathBuf,
     options: BabelOptions,
     transform_options: Result<TransformOptions, Vec<Error>>,
+    errors: Vec<OxcDiagnostic>,
 }
 
 impl TestCase for ConformanceTestCase {
@@ -185,7 +195,7 @@ impl TestCase for ConformanceTestCase {
         let mut options = BabelOptions::from_test_path(path.parent().unwrap());
         options.cwd.replace(cwd.to_path_buf());
         let transform_options = transform_options(&options);
-        Self { path: path.to_path_buf(), options, transform_options }
+        Self { path: path.to_path_buf(), options, transform_options, errors: vec![] }
     }
 
     fn options(&self) -> &BabelOptions {
@@ -200,8 +210,12 @@ impl TestCase for ConformanceTestCase {
         &self.path
     }
 
+    fn errors(&self) -> &Vec<OxcDiagnostic> {
+        &self.errors
+    }
+
     /// Test conformance by comparing the parsed babel code and transformed code.
-    fn test(&self, filtered: bool) -> bool {
+    fn test(&mut self, filtered: bool) {
         let output_path = self.path.parent().unwrap().read_dir().unwrap().find_map(|entry| {
             let path = entry.ok()?.path();
             let file_stem = path.file_stem()?;
@@ -237,14 +251,12 @@ impl TestCase for ConformanceTestCase {
 
         let mut transformed_code = String::new();
         let mut actual_errors = String::new();
+        let mut transform_options = None;
 
-        let transform_options = match self.transform_options() {
-            Ok(transform_options) => {
-                match Driver::new(transform_options.clone()).execute(
-                    &input,
-                    source_type,
-                    &self.path,
-                ) {
+        match self.transform_options() {
+            Ok(options) => {
+                transform_options.replace(options.clone());
+                match Driver::new(options.clone()).execute(&input, source_type, &self.path) {
                     Ok(printed) => {
                         transformed_code = printed;
                     }
@@ -257,14 +269,12 @@ impl TestCase for ConformanceTestCase {
                         actual_errors = get_babel_error(&error);
                     }
                 }
-                Some(transform_options.clone())
             }
             Err(json_err) => {
                 let error = json_err.iter().map(ToString::to_string).collect::<Vec<_>>().join("\n");
                 actual_errors = get_babel_error(&error);
-                None
             }
-        };
+        }
 
         let babel_options = self.options();
 
@@ -315,7 +325,10 @@ impl TestCase for ConformanceTestCase {
 
             println!("Passed: {passed}");
         }
-        passed
+
+        if !passed {
+            self.errors.push(OxcDiagnostic::error(actual_errors));
+        }
     }
 }
 
@@ -324,6 +337,7 @@ pub struct ExecTestCase {
     path: PathBuf,
     options: BabelOptions,
     transform_options: Result<TransformOptions, Vec<Error>>,
+    errors: Vec<OxcDiagnostic>,
 }
 
 impl ExecTestCase {
@@ -357,7 +371,7 @@ impl TestCase for ExecTestCase {
         let mut options = BabelOptions::from_test_path(path.parent().unwrap());
         options.cwd.replace(cwd.to_path_buf());
         let transform_options = transform_options(&options);
-        Self { path: path.to_path_buf(), options, transform_options }
+        Self { path: path.to_path_buf(), options, transform_options, errors: vec![] }
     }
 
     fn options(&self) -> &BabelOptions {
@@ -372,7 +386,11 @@ impl TestCase for ExecTestCase {
         &self.path
     }
 
-    fn test(&self, filtered: bool) -> bool {
+    fn errors(&self) -> &Vec<OxcDiagnostic> {
+        &self.errors
+    }
+
+    fn test(&mut self, filtered: bool) {
         if filtered {
             println!("input_path: {:?}", &self.path);
             println!("Input:\n{}\n", fs::read_to_string(&self.path).unwrap());
@@ -387,7 +405,7 @@ impl TestCase for ExecTestCase {
                         error.iter().map(ToString::to_string).collect::<Vec<_>>().join("\n")
                     );
                 }
-                return false;
+                return;
             }
         };
         let target_path = self.write_to_test_files(&result);
@@ -398,7 +416,9 @@ impl TestCase for ExecTestCase {
             println!("Test Result:\n{}\n", TestRunnerEnv::get_test_result(&target_path));
         }
 
-        passed
+        if !passed {
+            self.errors.push(OxcDiagnostic::error("exec failed"));
+        }
     }
 }