dan/oxc
1
0
Fork 0
mirror of https://github.com/danbulant/oxc synced 2026-05-19 04:08:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(ast_tools): add dedicated Derive trait. (#5278)

Browse source 
In an effort toward the implementation of #5256, this PR allows us to have a separately generated "derive" file for each crate.
This also eliminates a bunch of boilerplate when writing new "derive" generators and generally makes it more approachable.
This commit is contained in:
rzvxa 2024-09-03 02:36:36 +00:00
parent be1a6d4c84
commit 68a1c01f4e
18 changed files with 459 additions and 310 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/.generated_ast_watch_list.yml vendored
View file

@ -13,9 +13,9 @@ src:
- 'crates/oxc_ast/src/generated/assert_layouts.rs'
- 'crates/oxc_ast/src/generated/ast_kind.rs'
- 'crates/oxc_ast/src/generated/ast_builder.rs'
- 'crates/oxc_ast/src/generated/visit.rs'
- 'crates/oxc_ast/src/generated/visit_mut.rs'
- 'crates/oxc_ast/src/generated/derive_clone_in.rs'
- 'crates/oxc_ast/src/generated/derive_get_span.rs'
- 'crates/oxc_ast/src/generated/derive_get_span_mut.rs'
- 'crates/oxc_ast/src/generated/visit.rs'
- 'crates/oxc_ast/src/generated/visit_mut.rs'
- 'tasks/ast_codegen/src/**'

10
crates/oxc_ast/src/ast/mod.rs
View file

@ -175,11 +175,11 @@
//!
//! If you are seeing compile-time errors in `src/ast/macros.rs`, this will be the cause.
mod js;
mod jsx;
mod literal;
mod macros;
mod ts;
pub(crate) mod js;
pub(crate) mod jsx;
pub(crate) mod literal;
pub(crate) mod macros;
pub(crate) mod ts;
use macros::inherit_variants;

13
crates/oxc_ast/src/generated/derive_clone_in.rs
View file

@ -1,12 +1,21 @@
// Auto-generated code, DO NOT EDIT DIRECTLY!
// To edit this generated file you have to edit `tasks/ast_tools/src/generators/derive_clone_in.rs`
// To edit this generated file you have to edit `tasks/ast_tools/src/derives/clone_in.rs`
#![allow(clippy::default_trait_access)]
use oxc_allocator::{Allocator, CloneIn};
#[allow(clippy::wildcard_imports)]
use crate::ast::*;
use crate::ast::js::*;
#[allow(clippy::wildcard_imports)]
use crate::ast::jsx::*;
#[allow(clippy::wildcard_imports)]
use crate::ast::literal::*;
#[allow(clippy::wildcard_imports)]
use crate::ast::ts::*;
impl<'alloc> CloneIn<'alloc> for BooleanLiteral {
type Cloned = BooleanLiteral;

15
crates/oxc_ast/src/generated/derive_get_span.rs
View file

@ -1,12 +1,21 @@
// Auto-generated code, DO NOT EDIT DIRECTLY!
// To edit this generated file you have to edit `tasks/ast_tools/src/generators/derive_get_span.rs`
// To edit this generated file you have to edit `tasks/ast_tools/src/derives/get_span.rs`
#![allow(clippy::match_same_arms)]
use oxc_span::GetSpan;
use oxc_span::{GetSpan, Span};
#[allow(clippy::wildcard_imports)]
use crate::ast::*;
use crate::ast::js::*;
#[allow(clippy::wildcard_imports)]
use crate::ast::jsx::*;
#[allow(clippy::wildcard_imports)]
use crate::ast::literal::*;
#[allow(clippy::wildcard_imports)]
use crate::ast::ts::*;
impl GetSpan for BooleanLiteral {
#[inline]

15
crates/oxc_ast/src/generated/derive_get_span_mut.rs
View file

@ -1,12 +1,21 @@
// Auto-generated code, DO NOT EDIT DIRECTLY!
// To edit this generated file you have to edit `tasks/ast_tools/src/generators/derive_get_span.rs`
// To edit this generated file you have to edit `tasks/ast_tools/src/derives/get_span.rs`
#![allow(clippy::match_same_arms)]
use oxc_span::GetSpanMut;
use oxc_span::{GetSpanMut, Span};
#[allow(clippy::wildcard_imports)]
use crate::ast::*;
use crate::ast::js::*;
#[allow(clippy::wildcard_imports)]
use crate::ast::jsx::*;
#[allow(clippy::wildcard_imports)]
use crate::ast::literal::*;
#[allow(clippy::wildcard_imports)]
use crate::ast::ts::*;
impl GetSpanMut for BooleanLiteral {
#[inline]

83
tasks/ast_tools/src/codegen.rs
View file

@ -1,12 +1,16 @@
use std::{cell::RefCell, collections::HashMap, path::PathBuf};
use itertools::Itertools;
use proc_macro2::TokenStream;
use crate::{
derives::{Derive, DeriveOutput},
fmt::pretty_print,
generators::{Generator, GeneratorOutput},
passes::Pass,
rust_ast::{self, AstRef},
schema::{lower_ast_types, Schema, TypeDef},
util::write_all_to,
Result, TypeId,
};
@ -15,16 +19,51 @@ pub struct AstCodegen {
files: Vec<PathBuf>,
passes: Vec<Box<dyn Runner<Output = (), Context = EarlyCtx>>>,
generators: Vec<Box<dyn Runner<Output = GeneratorOutput, Context = LateCtx>>>,
derives: Vec<Box<dyn Runner<Output = DeriveOutput, Context = LateCtx>>>,
}
pub struct AstCodegenResult {
pub schema: Schema,
pub outputs: Vec<(/* generator name */ &'static str, /* output */ GeneratorOutput)>,
pub outputs: Vec<SideEffect>,
}
pub struct SideEffect(/* path */ pub PathBuf, /* output */ pub Vec<u8>);
impl SideEffect {
/// Apply the side-effect
pub fn apply(self) -> std::io::Result<()> {
let Self(path, data) = self;
let path = path.into_os_string();
let path = path.to_str().unwrap();
write_all_to(&data, path)?;
Ok(())
}
#[allow(clippy::unnecessary_wraps)]
pub fn path(&self) -> Option<String> {
let Self(path, _) = self;
let path = path.to_string_lossy();
Some(path.replace('\\', "/"))
}
}
impl From<(PathBuf, TokenStream)> for SideEffect {
fn from((path, stream): (PathBuf, TokenStream)) -> Self {
let content = pretty_print(&stream);
Self(path, content.as_bytes().into())
}
}
impl From<GeneratorOutput> for SideEffect {
fn from(output: GeneratorOutput) -> Self {
Self::from((output.0, output.1))
}
}
pub trait Runner {
type Context;
type Output;
#[allow(dead_code)]
fn name(&self) -> &'static str;
fn run(&mut self, ctx: &Self::Context) -> Result<Self::Output>;
}
@ -116,7 +155,7 @@ impl AstCodegen {
}
#[must_use]
pub fn gen<G>(mut self, generator: G) -> Self
pub fn generate<G>(mut self, generator: G) -> Self
where
G: Generator + Runner<Output = GeneratorOutput, Context = LateCtx> + 'static,
{
@ -124,7 +163,16 @@ impl AstCodegen {
self
}
pub fn generate(self) -> Result<AstCodegenResult> {
#[must_use]
pub fn derive<D>(mut self, derive: D) -> Self
where
D: Derive + Runner<Output = DeriveOutput, Context = LateCtx> + 'static,
{
self.derives.push(Box::new(derive));
self
}
pub fn run(self) -> Result<AstCodegenResult> {
let modules = self
.files
.into_iter()
@ -140,17 +188,42 @@ impl AstCodegen {
_ = self
.passes
.into_iter()
.map(|mut runner| runner.run(&ctx).map(|res| (runner.name(), res)))
.map(|mut runner| runner.run(&ctx))
.collect::<Result<Vec<_>>>()?;
ctx.into_late_ctx()
};
let derives = self
.derives
.into_iter()
.map(|mut runner| runner.run(&ctx))
.map_ok(|output| output.0.into_iter().map(SideEffect::from))
.flatten_ok();
let outputs = self
.generators
.into_iter()
.map(|mut runner| runner.run(&ctx).map(|res| (runner.name(), res)))
.map(|mut runner| runner.run(&ctx))
.map_ok(SideEffect::from)
.chain(derives)
.collect::<Result<Vec<_>>>()?;
Ok(AstCodegenResult { outputs, schema: ctx.schema })
}
}
/// Creates a generated file warning + required information for a generated file.
macro_rules! generated_header {
() => {{
let file = file!().replace("\\", "/");
// TODO add generation date, AST source hash, etc here.
let edit_comment = format!("@ To edit this generated file you have to edit `{file}`");
quote::quote! {
//!@ Auto-generated code, DO NOT EDIT DIRECTLY!
#![doc = #edit_comment]
//!@@line_break
}
}};
}
pub(crate) use generated_header;

53
tasks/ast_tools/src/generators/derive_clone_in.rs → tasks/ast_tools/src/derives/clone_in.rs
View file

@ -6,49 +6,34 @@ use syn::Ident;
use crate::{
codegen::LateCtx,
markers::CloneInAttribute,
output,
schema::{EnumDef, GetIdent, StructDef, TypeDef},
GeneratorOutput,
};
use super::{define_generator, generated_header, Generator};
use super::{define_derive, Derive, DeriveOutput};
define_generator! {
define_derive! {
pub struct DeriveCloneIn;
}
impl Generator for DeriveCloneIn {
fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput {
let impls: Vec<TokenStream> = ctx
.schema()
.into_iter()
.filter(|def| def.generates_derive("CloneIn"))
.map(|def| match &def {
TypeDef::Enum(it) => derive_enum(it),
TypeDef::Struct(it) => derive_struct(it),
})
.collect();
impl Derive for DeriveCloneIn {
fn trait_name() -> &'static str {
"CloneIn"
}
let header = generated_header!();
fn derive(&mut self, def: &TypeDef, _: &LateCtx) -> TokenStream {
match &def {
TypeDef::Enum(it) => derive_enum(it),
TypeDef::Struct(it) => derive_struct(it),
}
}
GeneratorOutput::Stream((
output(crate::AST_CRATE, "derive_clone_in.rs"),
quote! {
#header
fn prelude() -> TokenStream {
quote! {
#![allow(clippy::default_trait_access)]
#![allow(clippy::default_trait_access)]
///@@line_break
use oxc_allocator::{Allocator, CloneIn};
///@@line_break
#[allow(clippy::wildcard_imports)]
use crate::ast::*;
///@@line_break
#(#impls)*
},
))
///@@line_break
use oxc_allocator::{Allocator, CloneIn};
}
}
}
@ -106,7 +91,6 @@ fn impl_clone_in(
) -> TokenStream {
if has_lifetime {
quote! {
///@@line_break
impl <'old_alloc, 'new_alloc> CloneIn<'new_alloc> for #ty_ident<'old_alloc> {
type Cloned = #ty_ident<'new_alloc>;
fn clone_in(&self, #alloc_ident: &'new_alloc Allocator) -> Self::Cloned {
@ -116,7 +100,6 @@ fn impl_clone_in(
}
} else {
quote! {
///@@line_break
impl <'alloc> CloneIn<'alloc> for #ty_ident {
type Cloned = #ty_ident;
fn clone_in(&self, #alloc_ident: &'alloc Allocator) -> Self::Cloned {

87
tasks/ast_tools/src/generators/derive_get_span.rs → tasks/ast_tools/src/derives/get_span.rs
View file

@ -4,41 +4,63 @@ use syn::Ident;
use crate::{
codegen::LateCtx,
output,
schema::{EnumDef, GetGenerics, StructDef, ToType, TypeDef},
util::ToIdent,
Generator, GeneratorOutput,
};
use super::{define_generator, generated_header};
use super::{define_derive, Derive, DeriveOutput};
define_generator! {
define_derive! {
pub struct DeriveGetSpan;
}
impl Generator for DeriveGetSpan {
fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput {
impl Derive for DeriveGetSpan {
fn trait_name() -> &'static str {
"GetSpan"
}
fn derive(&mut self, def: &TypeDef, _: &LateCtx) -> TokenStream {
let self_type = quote!(&self);
let result_type = quote!(Span);
let result_expr = quote!(self.span);
let out = derive("GetSpan", "span", &self_type, &result_type, &result_expr, ctx);
GeneratorOutput::Stream((output(crate::AST_CRATE, "derive_get_span.rs"), out))
derive(Self::trait_name(), "span", &self_type, &result_type, &result_expr, def)
}
fn prelude() -> TokenStream {
quote! {
#![allow(clippy::match_same_arms)]
///@@line_break
use oxc_span::{Span, GetSpan};
}
}
}
define_generator! {
define_derive! {
pub struct DeriveGetSpanMut;
}
impl Generator for DeriveGetSpanMut {
fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput {
impl Derive for DeriveGetSpanMut {
fn trait_name() -> &'static str {
"GetSpanMut"
}
fn derive(&mut self, def: &TypeDef, _: &LateCtx) -> TokenStream {
let self_type = quote!(&mut self);
let result_type = quote!(&mut Span);
let result_expr = quote!(&mut self.span);
let out = derive("GetSpanMut", "span_mut", &self_type, &result_type, &result_expr, ctx);
GeneratorOutput::Stream((output(crate::AST_CRATE, "derive_get_span_mut.rs"), out))
derive(Self::trait_name(), "span_mut", &self_type, &result_type, &result_expr, def)
}
fn prelude() -> TokenStream {
quote! {
#![allow(clippy::match_same_arms)]
///@@line_break
use oxc_span::{Span, GetSpanMut};
}
}
}
@ -48,40 +70,15 @@ fn derive(
self_type: &TokenStream,
result_type: &TokenStream,
result_expr: &TokenStream,
ctx: &LateCtx,
def: &TypeDef,
) -> TokenStream {
let trait_ident = trait_name.to_ident();
let method_ident = method_name.to_ident();
let impls: Vec<TokenStream> = ctx
.schema()
.into_iter()
.filter(|def| def.generates_derive(trait_name))
.map(|def| match &def {
TypeDef::Enum(def) => {
derive_enum(def, &trait_ident, &method_ident, self_type, result_type)
}
TypeDef::Struct(def) => {
derive_struct(def, &trait_ident, &method_ident, self_type, result_type, result_expr)
}
})
.collect();
let header = generated_header!();
quote! {
#header
#![allow(clippy::match_same_arms)]
///@@line_break
use oxc_span::#trait_ident;
///@@line_break
#[allow(clippy::wildcard_imports)]
use crate::ast::*;
///@@line_break
#(#impls)*
match &def {
TypeDef::Enum(def) => derive_enum(def, &trait_ident, &method_ident, self_type, result_type),
TypeDef::Struct(def) => {
derive_struct(def, &trait_ident, &method_ident, self_type, result_type, result_expr)
}
}
}
@ -101,7 +98,6 @@ fn derive_enum(
});
quote! {
///@@line_break
impl #generics #trait_name for #target_type {
fn #method_name(#self_type) -> #result_type {
match self {
@ -132,7 +128,6 @@ fn derive_struct(
};
quote! {
///@@line_break
impl #generics #trait_name for #target_type {
#[inline]
fn #method_name(#self_type) -> #result_type {

129
tasks/ast_tools/src/derives/mod.rs Normal file
View file

@ -0,0 +1,129 @@
use proc_macro2::TokenStream;
use std::path::PathBuf;
use crate::{codegen::LateCtx, schema::TypeDef};
mod clone_in;
mod get_span;
pub use clone_in::DeriveCloneIn;
pub use get_span::{DeriveGetSpan, DeriveGetSpanMut};
pub trait Derive {
fn trait_name() -> &'static str;
fn derive(&mut self, def: &TypeDef, ctx: &LateCtx) -> TokenStream;
fn prelude() -> TokenStream {
TokenStream::default()
}
}
pub trait DeriveTemplate: Derive {
fn template(module_path: Vec<&str>, impls: TokenStream) -> TokenStream;
}
#[derive(Debug, Clone)]
pub struct DeriveOutput(pub Vec<(PathBuf, TokenStream)>);
macro_rules! define_derive {
($vis:vis struct $ident:ident $($lifetime:lifetime)? $($rest:tt)*) => {
$vis struct $ident $($lifetime)? $($rest)*
impl $($lifetime)? $crate::derives::DeriveTemplate for $ident $($lifetime)? {
fn template(module_paths: Vec<&str>, impls: TokenStream) -> TokenStream {
use itertools::Itertools;
let header = $crate::codegen::generated_header!();
let prelude = Self::prelude();
// from `x::y::z` to `crate::y::z::*`
let use_modules = module_paths.into_iter().map(|it|{
let local_path = ["crate"]
.into_iter()
.chain(it.split("::").skip(1))
.chain(["*"])
.join("::");
let use_module: syn::ItemUse =
syn::parse_str(format!("use {local_path};").as_str()).unwrap();
quote::quote! {
///@@line_break
#[allow(clippy::wildcard_imports)]
#use_module
}
});
quote::quote! {
#header
#prelude
#(#use_modules)*
///@@line_break
#impls
}
}
}
impl $($lifetime)? $crate::codegen::Runner for $ident $($lifetime)? {
type Context = $crate::codegen::LateCtx;
type Output = $crate::derives::DeriveOutput;
fn name(&self) -> &'static str {
stringify!($ident)
}
fn run(&mut self, ctx: &$crate::codegen::LateCtx) -> $crate::Result<Self::Output> {
use std::collections::{HashMap, HashSet};
use std::vec::Vec;
use convert_case::{Case, Casing};
use itertools::Itertools;
use $crate::derives::DeriveTemplate;
let trait_name = Self::trait_name();
let output = ctx
.schema()
.into_iter()
.filter(|def| def.generates_derive(trait_name))
.map(|def| (def, self.derive(def, ctx)))
.fold(HashMap::<&str, (HashSet<&str>, Vec<TokenStream>)>::new(), |mut acc, (def, stream)| {
let module_path = def.module_path();
let krate = module_path.split("::").next().unwrap();
if !acc.contains_key(krate) {
acc.insert(krate, Default::default());
}
let streams = acc.get_mut(krate).expect("We checked this right above!");
streams.0.insert(module_path);
streams.1.push(stream);
acc
})
.into_iter()
.sorted_by(|lhs, rhs| lhs.0.cmp(rhs.0))
.fold(Vec::new(), |mut acc, (path, (modules, streams))| {
let mut modules = Vec::from_iter(modules);
modules.sort();
acc.push((
$crate::output(
format!("crates/{}", path.split("::").next().unwrap()).as_str(),
format!("derive_{}.rs", Self::trait_name().to_case(Case::Snake)).as_str()
),
Self::template(
modules,
streams
.into_iter()
.fold(TokenStream::new(), |mut acc, it| {
acc.extend(quote::quote!{
///@@line_break
});
acc.extend(it);
acc
})
)
));
acc
});
Ok(DeriveOutput(output))
}
}
};
}
pub(crate) use define_derive;

8
tasks/ast_tools/src/generators/assert_layouts.rs
View file

@ -3,14 +3,14 @@ use quote::quote;
use syn::Type;
use crate::{
codegen::LateCtx,
codegen::{generated_header, LateCtx},
output,
schema::{FieldDef, ToType, TypeDef},
util::ToIdent,
Generator, GeneratorOutput,
};
use super::{define_generator, generated_header};
use super::define_generator;
define_generator! {
pub struct AssertLayouts;
@ -29,7 +29,7 @@ impl Generator for AssertLayouts {
let header = generated_header!();
GeneratorOutput::Stream((
GeneratorOutput(
output(crate::AST_CRATE, "assert_layouts.rs"),
quote! {
#header
@ -52,7 +52,7 @@ impl Generator for AssertLayouts {
#[cfg(not(any(target_pointer_width = "64", target_pointer_width = "32")))]
const _: () = panic!("Platforms with pointer width other than 64 or 32 bit are not supported");
},
))
)
}
}

6
tasks/ast_tools/src/generators/ast_builder.rs
View file

@ -8,9 +8,9 @@ use proc_macro2::TokenStream;
use quote::{format_ident, quote, ToTokens};
use syn::{parse_quote, Ident, Type};
use crate::codegen::generated_header;
use crate::{
codegen::LateCtx,
generators::generated_header,
output,
schema::{
EnumDef, FieldDef, GetIdent, InheritDef, StructDef, ToType, TypeDef, TypeName, VariantDef,
@ -36,7 +36,7 @@ impl Generator for AstBuilderGenerator {
let header = generated_header!();
GeneratorOutput::Stream((
GeneratorOutput(
output(crate::AST_CRATE, "ast_builder.rs"),
quote! {
#header
@ -66,7 +66,7 @@ impl Generator for AstBuilderGenerator {
#(#fns)*
}
},
))
)
}
}

8
tasks/ast_tools/src/generators/ast_kind.rs
View file

@ -3,14 +3,14 @@ use quote::quote;
use syn::{parse_quote, Arm, Ident, Type, Variant};
use crate::{
codegen::LateCtx,
codegen::{generated_header, LateCtx},
output,
schema::{GetIdent, ToType, TypeDef},
util::ToIdent,
Generator, GeneratorOutput,
};
use super::{define_generator, generated_header};
use super::define_generator;
define_generator! {
pub struct AstKindGenerator;
@ -156,7 +156,7 @@ impl Generator for AstKindGenerator {
let header = generated_header!();
GeneratorOutput::Stream((
GeneratorOutput(
output(crate::AST_CRATE, "ast_kind.rs"),
quote! {
#header
@ -190,6 +190,6 @@ impl Generator for AstKindGenerator {
}
}
},
))
)
}
}

92
tasks/ast_tools/src/generators/mod.rs
View file

@ -7,15 +7,11 @@ use crate::codegen::LateCtx;
mod assert_layouts;
mod ast_builder;
mod ast_kind;
mod derive_clone_in;
mod derive_get_span;
mod visit;
pub use assert_layouts::AssertLayouts;
pub use ast_builder::AstBuilderGenerator;
pub use ast_kind::AstKindGenerator;
pub use derive_clone_in::DeriveCloneIn;
pub use derive_get_span::{DeriveGetSpan, DeriveGetSpanMut};
pub use visit::{VisitGenerator, VisitMutGenerator};
/// Inserts a newline in the `TokenStream`.
@ -30,78 +26,8 @@ pub trait Generator {
fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput;
}
pub type GeneratedTokenStream = (/* output path */ PathBuf, TokenStream);
pub type GeneratedDataStream = (/* output path */ PathBuf, Vec<u8>);
// TODO: remove me
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub enum GeneratorOutput {
None,
Info(Vec<u8>),
Data(GeneratedDataStream),
Stream(GeneratedTokenStream),
}
// TODO: remove me
#[allow(dead_code)]
impl GeneratorOutput {
pub fn is_none(&self) -> bool {
matches!(self, Self::None)
}
pub fn expect_none(&self) {
assert!(self.is_none());
}
pub fn to_info(&self) -> &[u8] {
if let Self::Info(it) = self {
it
} else {
panic!();
}
}
pub fn to_data(&self) -> &GeneratedDataStream {
if let Self::Data(it) = self {
it
} else {
panic!();
}
}
pub fn to_stream(&self) -> &GeneratedTokenStream {
if let Self::Stream(it) = self {
it
} else {
panic!();
}
}
pub fn into_info(self) -> Vec<u8> {
if let Self::Info(it) = self {
it
} else {
panic!();
}
}
pub fn into_data(self) -> GeneratedDataStream {
if let Self::Data(it) = self {
it
} else {
panic!();
}
}
pub fn into_stream(self) -> GeneratedTokenStream {
if let Self::Stream(it) = self {
it
} else {
panic!();
}
}
}
pub struct GeneratorOutput(/* output path */ pub PathBuf, pub TokenStream);
macro_rules! define_generator {
($vis:vis struct $ident:ident $($lifetime:lifetime)? $($rest:tt)*) => {
@ -134,19 +60,3 @@ macro_rules! insert {
}
#[allow(unused_imports)]
pub(crate) use insert;
/// Creates a generated file warning + required information for a generated file.
macro_rules! generated_header {
() => {{
let file = file!().replace("\\", "/");
// TODO add generation date, AST source hash, etc here.
let edit_comment = format!("@ To edit this generated file you have to edit `{file}`");
quote::quote! {
//!@ Auto-generated code, DO NOT EDIT DIRECTLY!
#![doc = #edit_comment]
//!@@line_break
}
}};
}
pub(crate) use generated_header;

14
tasks/ast_tools/src/generators/visit.rs
View file

@ -7,7 +7,7 @@ use quote::{format_ident, quote, ToTokens};
use syn::{parse_quote, Ident};
use crate::{
codegen::LateCtx,
codegen::{generated_header, LateCtx},
generators::ast_kind::BLACK_LIST as KIND_BLACK_LIST,
markers::VisitArg,
output,
@ -16,7 +16,7 @@ use crate::{
Generator, GeneratorOutput,
};
use super::{define_generator, generated_header};
use super::define_generator;
define_generator! {
pub struct VisitGenerator;
@ -28,19 +28,13 @@ define_generator! {
impl Generator for VisitGenerator {
fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput {
GeneratorOutput::Stream((
output(crate::AST_CRATE, "visit.rs"),
generate_visit::<false>(ctx),
))
GeneratorOutput(output(crate::AST_CRATE, "visit.rs"), generate_visit::<false>(ctx))
}
}
impl Generator for VisitMutGenerator {
fn generate(&mut self, ctx: &LateCtx) -> GeneratorOutput {
GeneratorOutput::Stream((
output(crate::AST_CRATE, "visit_mut.rs"),
generate_visit::<true>(ctx),
))
GeneratorOutput(output(crate::AST_CRATE, "visit_mut.rs"), generate_visit::<true>(ctx))
}
}

75
tasks/ast_tools/src/main.rs
View file

@ -6,6 +6,7 @@ use itertools::Itertools;
use syn::parse_file;
mod codegen;
mod derives;
mod fmt;
mod generators;
mod layout;
@ -15,10 +16,10 @@ mod rust_ast;
mod schema;
mod util;
use fmt::{cargo_fmt, pretty_print};
use derives::{DeriveCloneIn, DeriveGetSpan, DeriveGetSpanMut};
use fmt::cargo_fmt;
use generators::{
AssertLayouts, AstBuilderGenerator, AstKindGenerator, DeriveCloneIn, DeriveGetSpan,
DeriveGetSpanMut, GeneratedDataStream, GeneratedTokenStream, Generator, GeneratorOutput,
AssertLayouts, AstBuilderGenerator, AstKindGenerator, Generator, GeneratorOutput,
VisitGenerator, VisitMutGenerator,
};
use passes::{CalcLayout, Linker};
@ -60,28 +61,25 @@ fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
.fold(AstCodegen::default(), AstCodegen::add_file)
.pass(Linker)
.pass(CalcLayout)
.gen(AssertLayouts)
.gen(AstKindGenerator)
.gen(AstBuilderGenerator)
.gen(DeriveCloneIn)
.gen(DeriveGetSpan)
.gen(DeriveGetSpanMut)
.gen(VisitGenerator)
.gen(VisitMutGenerator)
.generate()?;
let (streams, outputs): (Vec<_>, Vec<_>) =
outputs.into_iter().partition(|it| matches!(it.1, GeneratorOutput::Stream(_)));
let (binaries, _): (Vec<_>, Vec<_>) =
outputs.into_iter().partition(|it| matches!(it.1, GeneratorOutput::Data(_)));
.derive(DeriveCloneIn)
.derive(DeriveGetSpan)
.derive(DeriveGetSpanMut)
.generate(AssertLayouts)
.generate(AstKindGenerator)
.generate(AstBuilderGenerator)
.generate(VisitGenerator)
.generate(VisitMutGenerator)
.run()?;
if !cli_options.dry_run {
let side_effects =
write_generated_streams(streams.into_iter().map(|it| it.1.into_stream()))?
.into_iter()
.chain(write_data_streams(binaries.into_iter().map(|it| it.1.into_data()))?)
.collect();
let side_effects = outputs
.into_iter()
.filter_map(|it| {
let path = it.path();
it.apply().unwrap();
path
})
.collect();
write_ci_filter(SOURCE_PATHS, side_effects, ".github/.generated_ast_watch_list.yml")?;
}
@ -102,37 +100,6 @@ fn output(krate: &str, path: &str) -> PathBuf {
std::path::PathBuf::from_iter(vec![krate, "src", "generated", path])
}
/// Writes all streams and returns a vector pointing to side-effects written on the disk
fn write_generated_streams(
streams: impl IntoIterator<Item = GeneratedTokenStream>,
) -> std::io::Result<Vec<String>> {
streams
.into_iter()
.map(|(path, stream)| {
let path = path.into_os_string();
let path = path.to_str().unwrap();
let content = pretty_print(&stream);
write_all_to(content.as_bytes(), path)?;
Ok(path.to_string().replace('\\', "/"))
})
.collect()
}
/// Writes all streams and returns a vector pointing to side-effects written on the disk
fn write_data_streams(
streams: impl IntoIterator<Item = GeneratedDataStream>,
) -> std::io::Result<Vec<String>> {
streams
.into_iter()
.map(|(path, stream)| {
let path = path.into_os_string();
let path = path.to_str().unwrap();
write_all_to(&stream, path)?;
Ok(path.to_string().replace('\\', "/"))
})
.collect()
}
fn write_ci_filter(
inputs: &[&str],
side_effects: Vec<String>,

140
tasks/ast_tools/src/rust_ast.rs
View file

@ -30,13 +30,27 @@ impl From<Ident> for Inherit {
}
}
#[derive(Debug, Default, Clone)]
#[derive(Debug, Clone)]
pub struct EnumMeta {
pub inherits: Vec<Inherit>,
pub layout_32: Layout,
pub layout_64: Layout,
pub visitable: bool,
pub ast: bool,
pub module_path: String,
}
impl EnumMeta {
fn new(module_path: String) -> Self {
Self {
inherits: Vec::default(),
layout_32: Layout::default(),
layout_64: Layout::default(),
visitable: false,
ast: false,
module_path,
}
}
}
#[derive(Debug)]
@ -61,19 +75,26 @@ impl Enum {
}
}
impl From<ItemEnum> for Enum {
fn from(item: ItemEnum) -> Self {
Self { item, meta: EnumMeta::default() }
}
}
/// Placeholder for now!
#[derive(Debug, Default, Clone)]
#[derive(Debug, Clone)]
pub struct StructMeta {
pub layout_32: Layout,
pub layout_64: Layout,
pub visitable: bool,
pub ast: bool,
pub module_path: String,
}
impl StructMeta {
fn new(module_path: String) -> Self {
Self {
layout_32: Layout::default(),
layout_64: Layout::default(),
visitable: false,
ast: false,
module_path,
}
}
}
#[derive(Debug)]
@ -83,6 +104,10 @@ pub struct Struct {
}
impl Struct {
pub fn with_meta(item: ItemStruct, meta: StructMeta) -> Self {
Self { item, meta }
}
pub fn ident(&self) -> &Ident {
&self.item.ident
}
@ -94,9 +119,26 @@ impl Struct {
}
}
impl From<ItemStruct> for Struct {
fn from(item: ItemStruct) -> Self {
Self { item, meta: StructMeta::default() }
#[derive(Debug)]
pub struct Macro {
pub item: ItemMacro,
pub meta: MacroMeta,
}
impl Macro {
pub fn with_meta(item: ItemMacro, meta: MacroMeta) -> Self {
Self { item, meta }
}
}
#[derive(Debug)]
pub struct MacroMeta {
pub module_path: String,
}
impl MacroMeta {
fn new(module_path: String) -> Self {
Self { module_path }
}
}
@ -106,7 +148,7 @@ pub enum AstType {
Struct(Struct),
// we need this to expand `inherit` macro calls.
Macro(ItemMacro),
Macro(Macro),
}
impl ToTokens for AstType {
@ -114,18 +156,30 @@ impl ToTokens for AstType {
match self {
Self::Enum(it) => it.item.to_tokens(tokens),
Self::Struct(it) => it.item.to_tokens(tokens),
Self::Macro(it) => it.to_tokens(tokens),
Self::Macro(it) => it.item.to_tokens(tokens),
}
}
}
impl AstType {
fn new(item: Item, module_path: String) -> Result<Self> {
match item {
Item::Enum(it) => Ok(AstType::Enum(Enum::with_meta(it, EnumMeta::new(module_path)))),
Item::Struct(it) => {
Ok(AstType::Struct(Struct::with_meta(it, StructMeta::new(module_path))))
}
Item::Macro(it) => {
Ok(AstType::Macro(Macro::with_meta(it, MacroMeta::new(module_path))))
}
_ => Err(String::from("Unsupported Item!")),
}
}
pub fn ident(&self) -> Option<&Ident> {
match self {
AstType::Enum(ty) => Some(ty.ident()),
AstType::Struct(ty) => Some(ty.ident()),
AstType::Macro(tt) => tt.ident.as_ref(),
AstType::Macro(ty) => ty.item.ident.as_ref(),
}
}
@ -156,7 +210,7 @@ impl AstType {
match self {
AstType::Enum(it) => assign!(it),
AstType::Struct(it) => assign!(it),
AstType::Macro(it) => return Err(unexpanded_macro_err(it)),
AstType::Macro(it) => return Err(unexpanded_macro_err(&it.item)),
}
Ok(())
}
@ -165,7 +219,7 @@ impl AstType {
match self {
AstType::Enum(it) => it.meta.ast = value,
AstType::Struct(it) => it.meta.ast = value,
AstType::Macro(it) => return Err(unexpanded_macro_err(it)),
AstType::Macro(it) => return Err(unexpanded_macro_err(&it.item)),
}
Ok(())
}
@ -174,7 +228,7 @@ impl AstType {
match self {
AstType::Enum(it) => Ok(it.meta.layout_32.clone()),
AstType::Struct(it) => Ok(it.meta.layout_32.clone()),
AstType::Macro(it) => Err(unexpanded_macro_err(it)),
AstType::Macro(it) => Err(unexpanded_macro_err(&it.item)),
}
}
@ -182,7 +236,7 @@ impl AstType {
match self {
AstType::Enum(it) => Ok(it.meta.layout_64.clone()),
AstType::Struct(it) => Ok(it.meta.layout_64.clone()),
AstType::Macro(it) => Err(unexpanded_macro_err(it)),
AstType::Macro(it) => Err(unexpanded_macro_err(&it.item)),
}
}
@ -200,20 +254,16 @@ impl AstType {
match self {
AstType::Enum(it) => assign!(it),
AstType::Struct(it) => assign!(it),
AstType::Macro(it) => return Err(unexpanded_macro_err(it)),
AstType::Macro(it) => return Err(unexpanded_macro_err(&it.item)),
}
Ok(())
}
}
impl TryFrom<Item> for AstType {
type Error = String;
fn try_from(item: Item) -> Result<Self> {
match item {
Item::Enum(it) => Ok(AstType::Enum(it.into())),
Item::Struct(it) => Ok(AstType::Struct(it.into())),
Item::Macro(it) => Ok(AstType::Macro(it)),
_ => Err(String::from("Unsupported Item!")),
pub fn module_path(&self) -> String {
match self {
AstType::Enum(it) => it.meta.module_path.clone(),
AstType::Struct(it) => it.meta.module_path.clone(),
AstType::Macro(it) => it.meta.module_path.clone(),
}
}
}
@ -221,11 +271,8 @@ impl TryFrom<Item> for AstType {
const LOAD_ERROR: &str = "should be loaded by now!";
#[derive(Debug)]
pub struct Module {
pub path: PathBuf,
// TODO: remove me
#[allow(dead_code)]
#[allow(clippy::struct_field_names)]
pub module: String,
pub file: PathBuf,
pub path: String,
pub shebang: Option<String>,
pub attrs: Vec<Attribute>,
pub items: Vec<AstRef>,
@ -240,15 +287,25 @@ impl ToTokens for Module {
}
impl Module {
pub fn with_path(path: PathBuf) -> Self {
let module = path.file_stem().map(|it| it.to_string_lossy().to_string()).unwrap();
Self { path, module, shebang: None, attrs: Vec::new(), items: Vec::new(), loaded: false }
/// Expects a file path to a rust source file in the `crates` directory.
pub fn with_path(file: PathBuf) -> Self {
let path = {
let no_ext = file.with_extension("");
let string = no_ext.to_string_lossy();
let mut parts = string.split('/');
assert_eq!(parts.next(), Some("crates"));
let krate = parts.next().unwrap();
assert_eq!(parts.next(), Some("src"));
let mut parts = [krate].into_iter().chain(parts);
parts.join("::")
};
Self { file, path, shebang: None, attrs: Vec::new(), items: Vec::new(), loaded: false }
}
pub fn load(mut self) -> Result<Self> {
assert!(!self.loaded, "can't load twice!");
let mut file = std::fs::File::open(&self.path).normalize()?;
let mut file = std::fs::File::open(&self.file).normalize()?;
let mut content = String::new();
file.read_to_string(&mut content).normalize()?;
let file = parse_file(content.as_str()).normalize()?;
@ -263,7 +320,7 @@ impl Module {
Item::Macro(m) if m.mac.path.is_ident("inherit_variants") => true,
_ => false,
})
.map(TryInto::try_into)
.map(|it| AstType::new(it, self.path.clone()))
.map_ok(|it| Rc::new(RefCell::new(it)))
.collect::<Result<_>>()?;
self.loaded = true;
@ -294,8 +351,9 @@ impl Module {
pub fn expand(ast_ref: &AstRef) -> Result<()> {
let to_replace = match &*ast_ref.borrow() {
AstType::Macro(mac) => {
ast_ref @ AstType::Macro(mac) => {
let (enum_, inherits) = mac
.item
.mac
.parse_body_with(|input: &ParseBuffer| {
// Because of `@inherit`s we can't use the actual `ItemEnum` parse,
@ -349,7 +407,7 @@ pub fn expand(ast_ref: &AstRef) -> Result<()> {
enum_,
EnumMeta {
inherits: inherits.into_iter().map(Into::into).collect(),
..EnumMeta::default()
..EnumMeta::new(ast_ref.module_path())
},
)))
}

8
tasks/ast_tools/src/schema/defs.rs
View file

@ -36,6 +36,10 @@ impl TypeDef {
let generated_derives = self.generated_derives();
generated_derives.iter().any(|it| it == derive)
}
pub fn module_path(&self) -> &str {
with_either!(self, it => &it.module_path)
}
}
#[derive(Debug, Serialize)]
@ -58,6 +62,8 @@ pub struct StructDef {
pub generated_derives: Vec<String>,
#[serde(skip)]
pub markers: OuterMarkers,
#[serde(skip)]
pub module_path: String,
}
#[derive(Debug, Serialize)]
@ -77,6 +83,8 @@ pub struct EnumDef {
pub align_32: usize,
pub offsets_32: Option<Vec<usize>>,
pub generated_derives: Vec<String>,
#[serde(skip)]
pub module_path: String,
}
impl EnumDef {

9
tasks/ast_tools/src/schema/mod.rs
View file

@ -131,7 +131,7 @@ fn lower_ast_type(ty: &rust::AstType, ctx: &codegen::EarlyCtx) -> TypeDef {
match ty {
rust::AstType::Enum(it) => TypeDef::Enum(lower_ast_enum(it, ctx)),
rust::AstType::Struct(it) => TypeDef::Struct(lower_ast_struct(it, ctx)),
rust::AstType::Macro(it) => panic!("{}", unexpanded_macro_err(it)),
rust::AstType::Macro(it) => panic!("{}", unexpanded_macro_err(&it.item)),
}
}
@ -167,6 +167,8 @@ fn lower_ast_enum(it @ rust::Enum { item, meta }: &rust::Enum, ctx: &codegen::Ea
offsets_32,
generated_derives: parse_generate_derive(&item.attrs),
module_path: meta.module_path.clone(),
}
}
@ -190,15 +192,18 @@ fn lower_ast_struct(
visitable: meta.visitable,
fields: item.fields.iter().map(|fi| lower_field(fi, ctx)).collect(),
has_lifetime: item.generics.lifetimes().count() > 0,
size_64,
align_64,
offsets_64,
size_32,
align_32,
offsets_32,
markers: parse_outer_markers(&item.attrs).unwrap(),
markers: parse_outer_markers(&item.attrs).unwrap(),
generated_derives: parse_generate_derive(&item.attrs),
module_path: meta.module_path.clone(),
}
}