feat(ast_codegen): process AST-related syntax types. (#4694)

.github/workflows/ci.yml

@@ -276,7 +276,10 @@ jobs:
         with:
           filters: |
             src:
-              - 'crates/oxc_ast/src/ast/**'
+              - 'crates/oxc_ast/src/ast/*' # single star is intentional
+              - 'crates/oxc_ast/src/generated/**' # to potentially cause CI error if generated files are edited manually
+              - 'crates/oxc_syntax/src/number.rs'
+              - 'crates/oxc_syntax/src/operator.rs'
               - 'tasks/ast_codegen/src/**'
 
       - uses: Boshen/setup-rust@main

Cargo.lock

@@ -1768,6 +1768,7 @@ dependencies = [
  "bitflags 2.6.0",
  "dashmap 6.0.1",
  "nonmax",
+ "oxc_ast_macros",
  "oxc_index",
  "oxc_span",
  "phf",

crates/oxc_ast/src/generated/assert_layouts.rs

@@ -4,6 +4,7 @@
 use std::mem::{align_of, offset_of, size_of};
 
 use crate::ast::*;
+use oxc_syntax::{number::*, operator::*};
 
 #[cfg(target_pointer_width = "64")]
 const _: () = {
@@ -1124,6 +1125,20 @@ const _: () = {
     assert!(align_of::<JSXText>() == 8usize);
     assert!(offset_of!(JSXText, span) == 0usize);
     assert!(offset_of!(JSXText, value) == 8usize);
+    assert!(size_of::<NumberBase>() == 1usize);
+    assert!(align_of::<NumberBase>() == 1usize);
+    assert!(size_of::<BigintBase>() == 1usize);
+    assert!(align_of::<BigintBase>() == 1usize);
+    assert!(size_of::<AssignmentOperator>() == 1usize);
+    assert!(align_of::<AssignmentOperator>() == 1usize);
+    assert!(size_of::<BinaryOperator>() == 1usize);
+    assert!(align_of::<BinaryOperator>() == 1usize);
+    assert!(size_of::<LogicalOperator>() == 1usize);
+    assert!(align_of::<LogicalOperator>() == 1usize);
+    assert!(size_of::<UnaryOperator>() == 1usize);
+    assert!(align_of::<UnaryOperator>() == 1usize);
+    assert!(size_of::<UpdateOperator>() == 1usize);
+    assert!(align_of::<UpdateOperator>() == 1usize);
 };
 
 #[cfg(target_pointer_width = "32")]
@@ -2245,6 +2260,20 @@ const _: () = {
     assert!(align_of::<JSXText>() == 4usize);
     assert!(offset_of!(JSXText, span) == 0usize);
     assert!(offset_of!(JSXText, value) == 8usize);
+    assert!(size_of::<NumberBase>() == 1usize);
+    assert!(align_of::<NumberBase>() == 1usize);
+    assert!(size_of::<BigintBase>() == 1usize);
+    assert!(align_of::<BigintBase>() == 1usize);
+    assert!(size_of::<AssignmentOperator>() == 1usize);
+    assert!(align_of::<AssignmentOperator>() == 1usize);
+    assert!(size_of::<BinaryOperator>() == 1usize);
+    assert!(align_of::<BinaryOperator>() == 1usize);
+    assert!(size_of::<LogicalOperator>() == 1usize);
+    assert!(align_of::<LogicalOperator>() == 1usize);
+    assert!(size_of::<UnaryOperator>() == 1usize);
+    assert!(align_of::<UnaryOperator>() == 1usize);
+    assert!(size_of::<UpdateOperator>() == 1usize);
+    assert!(align_of::<UpdateOperator>() == 1usize);
 };
 
 #[cfg(not(any(target_pointer_width = "64", target_pointer_width = "32")))]

crates/oxc_syntax/Cargo.toml

@@ -20,8 +20,9 @@ workspace = true
 doctest = false
 
 [dependencies]
-oxc_index = { workspace = true }
-oxc_span  = { workspace = true }
+oxc_index      = { workspace = true }
+oxc_span       = { workspace = true }
+oxc_ast_macros = { workspace = true }
 
 unicode-id-start = { workspace = true }
 bitflags         = { workspace = true }

crates/oxc_syntax/src/number.rs

@@ -1,4 +1,6 @@
-#[repr(u8)]
+use oxc_ast_macros::ast;
+
+#[ast]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum NumberBase {
     Float = 0,
@@ -14,7 +16,7 @@ impl NumberBase {
     }
 }
 
-#[repr(u8)]
+#[ast]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum BigintBase {
     Decimal = 0,

crates/oxc_syntax/src/operator.rs

@@ -6,9 +6,11 @@ use serde::Serialize;
 #[cfg(feature = "serialize")]
 use tsify::Tsify;
 
+use oxc_ast_macros::ast;
+
 use crate::precedence::{GetPrecedence, Precedence};
 
-#[repr(u8)]
+#[ast]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serialize", derive(Serialize, Tsify))]
 pub enum AssignmentOperator {
@@ -87,7 +89,7 @@ impl AssignmentOperator {
     }
 }
 
-#[repr(u8)]
+#[ast]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serialize", derive(Serialize, Tsify))]
 pub enum BinaryOperator {
@@ -275,7 +277,7 @@ impl GetPrecedence for BinaryOperator {
     }
 }
 
-#[repr(u8)]
+#[ast]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serialize", derive(Serialize))]
 #[cfg_attr(feature = "serialize", derive(Tsify))]
@@ -316,7 +318,7 @@ impl GetPrecedence for LogicalOperator {
     }
 }
 
-#[repr(u8)]
+#[ast]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serialize", derive(Serialize))]
 #[cfg_attr(feature = "serialize", derive(Tsify))]
@@ -370,7 +372,7 @@ impl UnaryOperator {
     }
 }
 
-#[repr(u8)]
+#[ast]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serialize", derive(Serialize))]
 #[cfg_attr(feature = "serialize", derive(Tsify))]

tasks/ast_codegen/src/generators/assert_layouts.rs

@@ -42,6 +42,8 @@ impl Generator for AssertLayouts {
                 endl!();
 
                 use crate::ast::*;
+                use oxc_syntax::{number::*, operator::*};
+
 
                 endl!();
 

tasks/ast_codegen/src/main.rs

@@ -1,4 +1,5 @@
 const AST_CRATE: &str = "crates/oxc_ast";
+const AST_SYNTAX: &str = "crates/oxc_syntax";
 #[allow(dead_code)]
 const AST_MACROS_CRATE: &str = "crates/oxc_ast_macros";
 
@@ -218,9 +219,8 @@ impl AstCodegen {
             .map(rust_ast::Module::from)
             .map(rust_ast::Module::load)
             .map_ok(rust_ast::Module::expand)
-            .flatten()
-            .map_ok(rust_ast::Module::analyze)
-            .collect::<Result<Result<Vec<_>>>>()??;
+            .map_ok(|it| it.map(rust_ast::Module::analyze))
+            .collect::<Result<Result<Result<Vec<_>>>>>()???;
 
         // early passes
         let ctx = {
@@ -244,11 +244,16 @@ impl AstCodegen {
 }
 
 fn files() -> impl std::iter::Iterator<Item = String> {
-    fn path(path: &str) -> String {
+    fn ast(path: &str) -> String {
         format!("{AST_CRATE}/src/ast/{path}.rs")
     }
 
-    vec![path("literal"), path("js"), path("ts"), path("jsx")].into_iter()
+    fn syntax(path: &str) -> String {
+        format!("{AST_SYNTAX}/src/{path}.rs")
+    }
+
+    vec![ast("literal"), ast("js"), ast("ts"), ast("jsx"), syntax("number"), syntax("operator")]
+        .into_iter()
 }
 
 fn write_generated_streams(

tasks/ast_codegen/src/passes/calc_layout.rs

@@ -351,22 +351,19 @@ lazy_static! {
             64 => Layout::wide_ptr_64(),
             32 => Layout::wide_ptr_32(),
         },
+        // External Bumpalo type
         Vec: {
             64 => Layout::known(32, 8, 1),
             32 => Layout::known(16, 4, 1),
         },
+        // Unsupported: we don't analyze `Cell` types
         Cell<Option<ScopeId>>: { _ => Layout::known(4, 4, 0), },
         Cell<Option<SymbolId>>: { _ => Layout::known(4, 4, 0), },
         Cell<Option<ReferenceId>>: { _ => Layout::known(4, 4, 0), },
-        ReferenceFlag: { _ => Layout::known(1, 1, 0), },
-        AssignmentOperator: { _ => Layout::known(1, 1, 1), },
-        LogicalOperator: { _ => Layout::known(1, 1, 1), },
-        UnaryOperator: { _ => Layout::known(1, 1, 1), },
-        BinaryOperator: { _ => Layout::known(1, 1, 1), },
-        UpdateOperator: { _ => Layout::known(1, 1, 1), },
         SourceType: { _ => Layout::known(4, 1, 1), },
+        // Unsupported: this is a `bitflags` generated type, we don't expand macros
+        ReferenceFlag: { _ => Layout::known(1, 1, 0), },
+        // Unsupported: this is a `bitflags` generated type, we don't expand macros
         RegExpFlags: { _ => Layout::known(1, 1, 0), },
-        BigintBase: { _ => Layout::known(1, 1, 1), },
-        NumberBase: { _ => Layout::known(1, 1, 1), },
     };
 }