feat(oxc,napi/transform): napi/transform use oxc compiler pipeline (#6298)

part of #6156
This commit is contained in:
Boshen 2024-10-05 16:35:09 +00:00
parent 0cea6e9fa1
commit 8729755baa
9 changed files with 299 additions and 223 deletions

1
Cargo.lock generated
View file

@ -1992,7 +1992,6 @@ dependencies = [
"napi-build",
"napi-derive",
"oxc",
"rustc-hash",
]
[[package]]

View file

@ -2,14 +2,18 @@ use std::{mem, ops::ControlFlow, path::Path};
use oxc_allocator::Allocator;
use oxc_ast::{ast::Program, Trivias};
use oxc_codegen::{CodeGenerator, CodegenOptions, CommentOptions};
use oxc_codegen::{CodeGenerator, CodegenOptions, CodegenReturn, CommentOptions};
use oxc_diagnostics::OxcDiagnostic;
use oxc_isolated_declarations::{IsolatedDeclarations, IsolatedDeclarationsOptions};
use oxc_mangler::{MangleOptions, Mangler};
use oxc_minifier::{CompressOptions, Compressor};
use oxc_parser::{ParseOptions, Parser, ParserReturn};
use oxc_semantic::{ScopeTree, SemanticBuilder, SemanticBuilderReturn, SymbolTable};
use oxc_span::SourceType;
use oxc_transformer::{TransformOptions, Transformer, TransformerReturn};
use oxc_transformer::{
InjectGlobalVariables, InjectGlobalVariablesConfig, ReplaceGlobalDefines,
ReplaceGlobalDefinesConfig, TransformOptions, Transformer, TransformerReturn,
};
#[derive(Default)]
pub struct Compiler {
@ -22,8 +26,8 @@ impl CompilerInterface for Compiler {
self.errors.extend(errors);
}
fn after_codegen(&mut self, printed: String) {
self.printed = printed;
fn after_codegen(&mut self, ret: CodegenReturn) {
self.printed = ret.source_text;
}
}
@ -49,14 +53,30 @@ impl Compiler {
pub trait CompilerInterface {
fn handle_errors(&mut self, _errors: Vec<OxcDiagnostic>) {}
fn enable_sourcemap(&self) -> bool {
false
}
fn parse_options(&self) -> ParseOptions {
ParseOptions::default()
}
fn isolated_declaration_options(&self) -> Option<IsolatedDeclarationsOptions> {
None
}
fn transform_options(&self) -> Option<TransformOptions> {
Some(TransformOptions::default())
}
fn define_options(&self) -> Option<ReplaceGlobalDefinesConfig> {
None
}
fn inject_options(&self) -> Option<InjectGlobalVariablesConfig> {
None
}
fn compress_options(&self) -> Option<CompressOptions> {
None
}
@ -89,6 +109,8 @@ pub trait CompilerInterface {
ControlFlow::Continue(())
}
fn after_isolated_declarations(&mut self, _ret: CodegenReturn) {}
fn after_transform(
&mut self,
_program: &mut Program<'_>,
@ -97,7 +119,7 @@ pub trait CompilerInterface {
ControlFlow::Continue(())
}
fn after_codegen(&mut self, _printed: String) {}
fn after_codegen(&mut self, _ret: CodegenReturn) {}
fn compile(&mut self, source_text: &str, source_type: SourceType, source_path: &Path) {
let allocator = Allocator::default();
@ -112,11 +134,23 @@ pub trait CompilerInterface {
self.handle_errors(parser_return.errors);
}
/* Semantic */
let mut program = parser_return.program;
let trivias = parser_return.trivias;
/* Isolated Declarations */
if let Some(options) = self.isolated_declaration_options() {
self.isolated_declaration(
options,
&allocator,
&program,
source_text,
source_path,
&trivias,
);
}
/* Semantic */
let mut semantic_return = self.semantic(&program, source_text, source_path);
if !semantic_return.errors.is_empty() {
self.handle_errors(semantic_return.errors);
@ -126,7 +160,7 @@ pub trait CompilerInterface {
return;
}
let (symbols, scopes) = semantic_return.semantic.into_symbol_table_and_scope_tree();
let (mut symbols, mut scopes) = semantic_return.semantic.into_symbol_table_and_scope_tree();
/* Transform */
@ -150,6 +184,23 @@ pub trait CompilerInterface {
if self.after_transform(&mut program, &mut transformer_return).is_break() {
return;
}
symbols = transformer_return.symbols;
scopes = transformer_return.scopes;
}
if let Some(config) = self.define_options() {
let ret =
ReplaceGlobalDefines::new(&allocator, config).build(symbols, scopes, &mut program);
symbols = ret.symbols;
scopes = ret.scopes;
}
if let Some(config) = self.inject_options() {
let _ret =
InjectGlobalVariables::new(&allocator, config).build(symbols, scopes, &mut program);
// symbols = ret.symbols;
// scopes = ret.scopes;
}
/* Compress */
@ -165,8 +216,8 @@ pub trait CompilerInterface {
/* Codegen */
if let Some(options) = self.codegen_options() {
let printed = self.codegen(&program, source_text, &trivias, mangler, options);
self.after_codegen(printed);
let ret = self.codegen(&program, source_text, source_path, &trivias, mangler, options);
self.after_codegen(ret);
}
}
@ -199,6 +250,29 @@ pub trait CompilerInterface {
.build(program)
}
fn isolated_declaration<'a>(
&mut self,
options: IsolatedDeclarationsOptions,
allocator: &'a Allocator,
program: &Program<'a>,
source_text: &'a str,
source_path: &Path,
trivias: &Trivias,
) {
let ret =
IsolatedDeclarations::new(allocator, source_text, trivias, options).build(program);
self.handle_errors(ret.errors);
let ret = self.codegen(
&ret.program,
source_text,
source_path,
trivias,
None,
self.codegen_options().unwrap_or_default(),
);
self.after_isolated_declarations(ret);
}
#[allow(clippy::too_many_arguments)]
fn transform<'a>(
&self,
@ -232,16 +306,20 @@ pub trait CompilerInterface {
&self,
program: &Program<'a>,
source_text: &'a str,
source_path: &Path,
trivias: &Trivias,
mangler: Option<Mangler>,
options: CodegenOptions,
) -> String {
) -> CodegenReturn {
let comment_options = CommentOptions { preserve_annotate_comments: true };
CodeGenerator::new()
let mut codegen = CodeGenerator::new()
.with_options(options)
.with_mangler(mangler)
.enable_comment(source_text, trivias.clone(), comment_options)
.build(program)
.source_text
.enable_comment(source_text, trivias.clone(), comment_options);
if self.enable_sourcemap() {
codegen =
codegen.enable_source_map(source_path.to_string_lossy().as_ref(), source_text);
}
codegen.build(program)
}
}

View file

@ -1,3 +1,5 @@
// NOTE: Types must be aligned with [@types/babel__core](https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/babel__core/index.d.ts).
#![allow(rustdoc::bare_urls)]
use std::path::PathBuf;
@ -8,7 +10,43 @@ use rustc_hash::FxHashMap;
use oxc_transformer::{JsxRuntime, RewriteExtensionsMode};
use super::isolated_declarations::IsolatedDeclarationsOptions;
use super::{isolated_declarations::IsolatedDeclarationsOptions, source_map::SourceMap};
#[derive(Default)]
#[napi(object)]
pub struct TransformResult {
/// The transformed code.
///
/// If parsing failed, this will be an empty string.
pub code: String,
/// The source map for the transformed code.
///
/// This will be set if {@link TransformOptions#sourcemap} is `true`.
pub map: Option<SourceMap>,
/// The `.d.ts` declaration file for the transformed code. Declarations are
/// only generated if `declaration` is set to `true` and a TypeScript file
/// is provided.
///
/// If parsing failed and `declaration` is set, this will be an empty string.
///
/// @see {@link TypeScriptOptions#declaration}
/// @see [declaration tsconfig option](https://www.typescriptlang.org/tsconfig/#declaration)
pub declaration: Option<String>,
/// Declaration source map. Only generated if both
/// {@link TypeScriptOptions#declaration declaration} and
/// {@link TransformOptions#sourcemap sourcemap} are set to `true`.
pub declaration_map: Option<SourceMap>,
/// Parse and transformation errors.
///
/// Oxc's parser recovers from common syntax errors, meaning that
/// transformed code may still be available even if there are errors in this
/// list.
pub errors: Vec<String>,
}
/// Options for transforming a JavaScript or TypeScript file.
///

View file

@ -21,9 +21,7 @@ test = false
doctest = false
[dependencies]
oxc = { workspace = true, features = ["napi", "isolated_declarations", "transformer", "sourcemap", "codegen", "semantic"] }
rustc-hash = { workspace = true }
oxc = { workspace = true, features = ["full", "napi"] }
napi = { workspace = true }
napi-derive = { workspace = true }

View file

@ -1,6 +1,5 @@
use std::{
cell::{Ref, RefCell, RefMut},
path::Path,
cell::{Ref, RefCell},
sync::Arc,
};
@ -9,7 +8,6 @@ use oxc::{
ast::{ast::Program, Trivias},
codegen::Codegen,
diagnostics::{Error, NamedSource, OxcDiagnostic},
napi::{isolated_declarations::IsolatedDeclarationsOptions, transform::TransformOptions},
parser::{Parser, ParserReturn},
span::SourceType,
};
@ -23,9 +21,6 @@ pub(crate) struct TransformContext<'a> {
/// Generate source maps?
source_map: bool,
/// Generate `.d.ts` files?
declarations: Option<IsolatedDeclarationsOptions>,
/// Path to the file being transformed.
filename: &'a str,
@ -43,24 +38,17 @@ impl<'a> TransformContext<'a> {
filename: &'a str,
source_text: &'a str,
source_type: SourceType,
options: Option<&TransformOptions>,
source_map: Option<bool>,
) -> Self {
let ParserReturn { errors, program, trivias, .. } =
Parser::new(allocator, source_text, source_type).parse();
// Options that are added by this napi crates and don't exist in
// oxc_transformer.
let source_map = options.as_ref().and_then(|o| o.sourcemap).unwrap_or_default();
let declarations =
options.as_ref().and_then(|o| o.typescript.as_ref()).and_then(|t| t.declaration);
let source_map = source_map.unwrap_or_default();
Self {
allocator,
program: RefCell::new(program),
trivias,
source_map,
declarations,
filename,
source_text,
@ -74,31 +62,16 @@ impl<'a> TransformContext<'a> {
self.filename
}
#[inline]
pub fn file_path(&self) -> &'a Path {
Path::new(self.filename)
}
#[inline]
pub fn source_text(&self) -> &'a str {
self.source_text
}
#[inline]
pub(crate) fn declarations(&self) -> Option<&IsolatedDeclarationsOptions> {
self.declarations.as_ref()
}
#[inline]
pub fn program(&self) -> Ref<'_, Program<'a>> {
self.program.borrow()
}
#[inline]
pub fn program_mut(&self) -> RefMut<'_, Program<'a>> {
self.program.borrow_mut()
}
pub fn codegen(&self) -> Codegen<'a> {
let codegen = Codegen::new();
if self.source_map {

View file

@ -3,10 +3,7 @@ use oxc::{
allocator::Allocator,
codegen::{CodegenReturn, CommentOptions},
isolated_declarations::IsolatedDeclarations,
napi::{
isolated_declarations::{IsolatedDeclarationsOptions, IsolatedDeclarationsResult},
transform::TransformOptions,
},
napi::isolated_declarations::{IsolatedDeclarationsOptions, IsolatedDeclarationsResult},
span::SourceType,
};
@ -23,13 +20,8 @@ pub fn isolated_declaration(
let source_type = SourceType::from_path(&filename).unwrap_or_default().with_typescript(true);
let allocator = Allocator::default();
let options = options.unwrap_or_default();
let ctx = TransformContext::new(
&allocator,
&filename,
&source_text,
source_type,
Some(&TransformOptions { sourcemap: options.sourcemap, ..Default::default() }),
);
let ctx =
TransformContext::new(&allocator, &filename, &source_text, source_type, options.sourcemap);
let transformed_ret = build_declarations(&ctx, options);
IsolatedDeclarationsResult {

View file

@ -1,57 +1,123 @@
use std::{path::Path, sync::Arc};
use napi::Either;
use napi_derive::napi;
use rustc_hash::FxHashMap;
use oxc::{
allocator::Allocator,
codegen::CodegenReturn,
napi::{source_map::SourceMap, transform::TransformOptions},
semantic::{ScopeTree, SemanticBuilder, SymbolTable},
span::SourceType,
transformer::{
InjectGlobalVariables, InjectGlobalVariablesConfig, InjectImport, ReplaceGlobalDefines,
ReplaceGlobalDefinesConfig, Transformer,
diagnostics::{Error, NamedSource, OxcDiagnostic},
napi::{
source_map::SourceMap,
transform::{TransformOptions, TransformResult},
},
span::SourceType,
transformer::{InjectGlobalVariablesConfig, InjectImport, ReplaceGlobalDefinesConfig},
CompilerInterface,
};
use crate::{context::TransformContext, isolated_declaration};
#[derive(Default)]
struct Compiler {
transform_options: oxc::transformer::TransformOptions,
sourcemap: bool,
// NOTE: Use JSDoc syntax for all doc comments, not rustdoc.
// NOTE: Types must be aligned with [@types/babel__core](https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/babel__core/index.d.ts).
printed: String,
printed_sourcemap: Option<SourceMap>,
#[napi(object)]
pub struct TransformResult {
/// The transformed code.
///
/// If parsing failed, this will be an empty string.
pub code: String,
declaration: Option<String>,
declaration_map: Option<SourceMap>,
/// The source map for the transformed code.
///
/// This will be set if {@link TransformOptions#sourcemap} is `true`.
pub map: Option<SourceMap>,
define: Option<ReplaceGlobalDefinesConfig>,
inject: Option<InjectGlobalVariablesConfig>,
/// The `.d.ts` declaration file for the transformed code. Declarations are
/// only generated if `declaration` is set to `true` and a TypeScript file
/// is provided.
///
/// If parsing failed and `declaration` is set, this will be an empty string.
///
/// @see {@link TypeScriptOptions#declaration}
/// @see [declaration tsconfig option](https://www.typescriptlang.org/tsconfig/#declaration)
pub declaration: Option<String>,
errors: Vec<OxcDiagnostic>,
}
/// Declaration source map. Only generated if both
/// {@link TypeScriptOptions#declaration declaration} and
/// {@link TransformOptions#sourcemap sourcemap} are set to `true`.
pub declaration_map: Option<SourceMap>,
impl Compiler {
fn new(options: Option<TransformOptions>) -> Result<Self, Vec<OxcDiagnostic>> {
let mut options = options;
let sourcemap = options.as_ref().and_then(|o| o.sourcemap).unwrap_or_default();
/// Parse and transformation errors.
///
/// Oxc's parser recovers from common syntax errors, meaning that
/// transformed code may still be available even if there are errors in this
/// list.
pub errors: Vec<String>,
let define = options
.as_mut()
.and_then(|options| options.define.take())
.map(|map| {
let define = map.into_iter().collect::<Vec<_>>();
ReplaceGlobalDefinesConfig::new(&define)
})
.transpose()?;
let inject = options
.as_mut()
.and_then(|options| options.inject.take())
.map(|map| {
map.into_iter()
.map(|(local, value)| match value {
Either::A(source) => Ok(InjectImport::default_specifier(&source, &local)),
Either::B(v) => {
if v.len() != 2 {
return Err(vec![OxcDiagnostic::error(
"Inject plugin did not receive a tuple [string, string].",
)]);
}
let source = v[0].to_string();
Ok(if v[1] == "*" {
InjectImport::namespace_specifier(&source, &local)
} else {
InjectImport::named_specifier(&source, Some(&v[1]), &local)
})
}
})
.collect::<Result<Vec<_>, _>>()
})
.transpose()?
.map(InjectGlobalVariablesConfig::new);
let transform_options =
options.map(oxc::transformer::TransformOptions::from).unwrap_or_default();
Ok(Self {
transform_options,
sourcemap,
printed: String::default(),
printed_sourcemap: None,
declaration: None,
declaration_map: None,
define,
inject,
errors: vec![],
})
}
}
impl CompilerInterface for Compiler {
fn handle_errors(&mut self, errors: Vec<OxcDiagnostic>) {
self.errors.extend(errors);
}
fn enable_sourcemap(&self) -> bool {
self.sourcemap
}
fn transform_options(&self) -> Option<oxc::transformer::TransformOptions> {
Some(self.transform_options.clone())
}
fn define_options(&self) -> Option<ReplaceGlobalDefinesConfig> {
self.define.clone()
}
fn inject_options(&self) -> Option<InjectGlobalVariablesConfig> {
self.inject.clone()
}
fn after_codegen(&mut self, ret: CodegenReturn) {
self.printed = ret.source_text;
self.printed_sourcemap = ret.source_map.map(SourceMap::from);
}
fn after_isolated_declarations(&mut self, ret: CodegenReturn) {
self.declaration.replace(ret.source_text);
self.declaration_map = ret.source_map.map(SourceMap::from);
}
}
/// Transpile a JavaScript or TypeScript into a target ECMAScript version.
@ -71,8 +137,9 @@ pub fn transform(
source_text: String,
options: Option<TransformOptions>,
) -> TransformResult {
let source_path = Path::new(&filename);
let source_type = {
let mut source_type = SourceType::from_path(&filename).unwrap_or_default();
let mut source_type = SourceType::from_path(source_path).unwrap_or_default();
// Force `script` or `module`
match options.as_ref().and_then(|options| options.source_type.as_deref()) {
Some("script") => source_type = source_type.with_script(true),
@ -82,125 +149,56 @@ pub fn transform(
source_type
};
let allocator = Allocator::default();
let ctx =
TransformContext::new(&allocator, &filename, &source_text, source_type, options.as_ref());
let declarations_result = source_type
.is_typescript()
.then(|| ctx.declarations())
.flatten()
.map(|options| isolated_declaration::build_declarations(&ctx, *options));
let transpile_result = transpile(&ctx, options);
let (declaration, declaration_map) = declarations_result
.map_or((None, None), |d| (Some(d.source_text), d.source_map.map(Into::into)));
let mut compiler = match Compiler::new(options) {
Ok(compiler) => compiler,
Err(errors) => {
return TransformResult {
errors: wrap_diagnostics(&filename, source_type, &source_text, errors),
..Default::default()
}
}
};
compiler.compile(&source_text, source_type, source_path);
TransformResult {
code: transpile_result.source_text,
map: transpile_result.source_map.map(Into::into),
declaration,
declaration_map,
errors: ctx.take_and_render_reports(),
code: compiler.printed,
map: compiler.printed_sourcemap,
declaration: compiler.declaration,
declaration_map: compiler.declaration_map,
errors: wrap_diagnostics(&filename, source_type, &source_text, compiler.errors),
}
}
fn transpile(ctx: &TransformContext<'_>, options: Option<TransformOptions>) -> CodegenReturn {
let semantic_ret = SemanticBuilder::new(ctx.source_text())
// Estimate transformer will triple scopes, symbols, references
.with_excess_capacity(2.0)
.with_check_syntax_error(true)
.build(&ctx.program());
ctx.add_diagnostics(semantic_ret.errors);
let mut options = options;
let define = options.as_mut().and_then(|options| options.define.take());
let inject = options.as_mut().and_then(|options| options.inject.take());
let options = options.map(oxc::transformer::TransformOptions::from).unwrap_or_default();
let (mut symbols, mut scopes) = semantic_ret.semantic.into_symbol_table_and_scope_tree();
let ret = Transformer::new(
ctx.allocator,
ctx.file_path(),
ctx.source_text(),
ctx.trivias.clone(),
options,
)
.build_with_symbols_and_scopes(symbols, scopes, &mut ctx.program_mut());
ctx.add_diagnostics(ret.errors);
symbols = ret.symbols;
scopes = ret.scopes;
if let Some(define) = define {
(symbols, scopes) = define_plugin(ctx, define, symbols, scopes);
fn wrap_diagnostics(
filename: &str,
source_type: SourceType,
source_text: &str,
errors: Vec<OxcDiagnostic>,
) -> Vec<String> {
if errors.is_empty() {
return vec![];
}
if let Some(inject) = inject {
_ = inject_plugin(ctx, inject, symbols, scopes);
}
ctx.codegen().build(&ctx.program())
}
fn define_plugin(
ctx: &TransformContext<'_>,
define: FxHashMap<String, String>,
symbols: SymbolTable,
scopes: ScopeTree,
) -> (SymbolTable, ScopeTree) {
let define = define.into_iter().collect::<Vec<_>>();
match ReplaceGlobalDefinesConfig::new(&define) {
Ok(config) => {
let ret = ReplaceGlobalDefines::new(ctx.allocator, config).build(
symbols,
scopes,
&mut ctx.program_mut(),
);
(ret.symbols, ret.scopes)
}
Err(errors) => {
ctx.add_diagnostics(errors);
(symbols, scopes)
}
}
}
fn inject_plugin(
ctx: &TransformContext<'_>,
inject: FxHashMap<String, Either<String, Vec<String>>>,
symbols: SymbolTable,
scopes: ScopeTree,
) -> (SymbolTable, ScopeTree) {
let Ok(injects) = inject
.into_iter()
.map(|(local, value)| match value {
Either::A(source) => Ok(InjectImport::default_specifier(&source, &local)),
Either::B(v) => {
if v.len() != 2 {
return Err(());
}
let source = v[0].to_string();
Ok(if v[1] == "*" {
InjectImport::namespace_specifier(&source, &local)
let source = {
let lang = match (source_type.is_javascript(), source_type.is_jsx()) {
(true, false) => "JavaScript",
(true, true) => "JSX",
(false, true) => "TypeScript React",
(false, false) => {
if source_type.is_typescript_definition() {
"TypeScript Declaration"
} else {
InjectImport::named_specifier(&source, Some(&v[1]), &local)
})
"TypeScript"
}
}
})
.collect::<Result<Vec<_>, ()>>()
else {
return (symbols, scopes);
};
let ns = NamedSource::new(filename, source_text.to_string()).with_language(lang);
Arc::new(ns)
};
let config = InjectGlobalVariablesConfig::new(injects);
let ret = InjectGlobalVariables::new(ctx.allocator, config).build(
symbols,
scopes,
&mut ctx.program_mut(),
);
(ret.symbols, ret.scopes)
errors
.into_iter()
.map(move |diagnostic| Error::from(diagnostic).with_source_code(Arc::clone(&source)))
.map(|error| format!("{error:?}"))
.collect()
}

View file

@ -5,7 +5,7 @@ use rustc_hash::FxHashSet;
use oxc::{
allocator::Allocator,
ast::{ast::Program, Trivias},
codegen::CodegenOptions,
codegen::{CodegenOptions, CodegenReturn},
diagnostics::OxcDiagnostic,
minifier::CompressOptions,
parser::{ParseOptions, ParserReturn},
@ -110,8 +110,8 @@ impl CompilerInterface for Driver {
ControlFlow::Continue(())
}
fn after_codegen(&mut self, printed: String) {
self.printed = printed;
fn after_codegen(&mut self, ret: CodegenReturn) {
self.printed = ret.source_text;
}
}

View file

@ -1,9 +1,8 @@
use std::{mem, ops::ControlFlow, path::Path};
use oxc::{
ast::ast::Program,
ast::Trivias,
codegen::{CodeGenerator, CodegenOptions},
ast::{ast::Program, Trivias},
codegen::{CodeGenerator, CodegenOptions, CodegenReturn},
diagnostics::OxcDiagnostic,
mangler::Mangler,
span::SourceType,
@ -36,8 +35,8 @@ impl CompilerInterface for Driver {
self.errors.extend(errors);
}
fn after_codegen(&mut self, printed: String) {
self.printed = printed;
fn after_codegen(&mut self, ret: CodegenReturn) {
self.printed = ret.source_text;
}
fn after_transform(
@ -62,11 +61,12 @@ impl CompilerInterface for Driver {
&self,
program: &Program<'a>,
_source_text: &'a str,
_source_path: &Path,
_trivias: &Trivias,
mangler: Option<Mangler>,
options: CodegenOptions,
) -> String {
CodeGenerator::new().with_options(options).with_mangler(mangler).build(program).source_text
) -> CodegenReturn {
CodeGenerator::new().with_options(options).with_mangler(mangler).build(program)
}
}