refactor(codegen): clean up API around building sourcemaps (#2602)

closes #2564
This commit is contained in:
Boshen 2024-03-04 16:03:33 +08:00 committed by GitHub
parent 20c7bf7835
commit ef932a3c27
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 138 additions and 117 deletions

View file

@ -28,13 +28,14 @@ fn main() -> std::io::Result<()> {
println!("Original:");
println!("{source_text}");
let codegen_options = CodegenOptions::default();
let printed = Codegen::<false>::new(source_text.len(), codegen_options).build(&ret.program);
let options = CodegenOptions::default();
let printed =
Codegen::<false>::new(&source_text, options.clone()).build(&ret.program).source_text;
println!("Printed:");
println!("{printed}");
let ret = Parser::new(&allocator, &printed, source_type).parse();
let minified = Codegen::<true>::new(source_text.len(), codegen_options).build(&ret.program);
let minified = Codegen::<true>::new(&source_text, options).build(&ret.program).source_text;
println!("Minified:");
println!("{minified}");

View file

@ -2,7 +2,7 @@ use std::{env, path::Path};
use base64::{prelude::BASE64_STANDARD, Engine};
use oxc_allocator::Allocator;
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_codegen::{Codegen, CodegenOptions, CodegenReturn};
use oxc_parser::Parser;
use oxc_span::SourceType;
@ -26,21 +26,27 @@ fn main() -> std::io::Result<()> {
return Ok(());
}
let codegen_options = CodegenOptions::default();
let mut codegen = Codegen::<false>::new(source_text.len(), codegen_options);
let content = codegen.with_sourcemap(&source_text, "").build(&ret.program);
let map = codegen.into_sourcemap();
let codegen_options = CodegenOptions {
enable_source_map: Some(path.to_string_lossy().to_string()),
enable_typescript: true,
};
let CodegenReturn { source_text, source_map } =
Codegen::<false>::new(&source_text, codegen_options).build(&ret.program);
if let Some(source_map) = source_map {
let mut buff = vec![];
source_map.to_writer(&mut buff).unwrap();
let result = String::from_utf8(buff).unwrap();
let hash = BASE64_STANDARD.encode(format!(
"{}\0{}{}\0{}",
source_text.len(),
source_text,
result.len(),
result
));
println!("https://evanw.github.io/source-map-visualization/#{hash}");
}
let mut buff = vec![];
map.to_writer(&mut buff).unwrap();
let result = String::from_utf8(buff).unwrap();
let hash = BASE64_STANDARD.encode(format!(
"{}\0{}{}\0{}",
content.len(),
content,
result.len(),
result
));
println!("https://evanw.github.io/source-map-visualization/#{hash}");
Ok(())
}

View file

@ -34,11 +34,20 @@ pub use crate::{
};
// use crate::mangler::Mangler;
#[derive(Debug, Default, Clone, Copy)]
#[derive(Debug, Default, Clone)]
pub struct CodegenOptions {
/// Pass in the filename to enable source map support.
pub enable_source_map: Option<String>,
/// Enable TypeScript code generation.
pub enable_typescript: bool,
}
pub struct CodegenReturn {
pub source_text: String,
pub source_map: Option<sourcemap::SourceMap>,
}
pub struct Codegen<const MINIFY: bool> {
#[allow(unused)]
options: CodegenOptions,
@ -64,7 +73,6 @@ pub struct Codegen<const MINIFY: bool> {
/// Track the current indentation level
indentation: u8,
// sourcemap
sourcemap_builder: SourcemapBuilder,
}
@ -76,10 +84,16 @@ pub enum Separator {
}
impl<const MINIFY: bool> Codegen<MINIFY> {
pub fn new(source_len: usize, options: CodegenOptions) -> Self {
pub fn new(source_text: &str, options: CodegenOptions) -> Self {
// Initialize the output code buffer to reduce memory reallocation.
// Minification will reduce by at least half of the original size.
let source_len = source_text.len();
let capacity = if MINIFY { source_len / 2 } else { source_len };
let mut sourcemap_builder = SourcemapBuilder::default();
if let Some(source_name) = &options.enable_source_map {
sourcemap_builder.with_name_and_source(source_name, source_text);
}
Self {
options,
// mangler: None,
@ -93,7 +107,7 @@ impl<const MINIFY: bool> Codegen<MINIFY> {
start_of_arrow_expr: 0,
start_of_default_export: 0,
indentation: 0,
sourcemap_builder: SourcemapBuilder::default(),
sourcemap_builder,
}
}
@ -101,27 +115,18 @@ impl<const MINIFY: bool> Codegen<MINIFY> {
// self.mangler = Some(mangler);
// }
pub fn with_sourcemap(&mut self, source: &str, source_name: &str) -> &mut Self {
self.sourcemap_builder
.with_enable_sourcemap(true)
.with_source_and_name(source, source_name);
self
pub fn build(mut self, program: &Program<'_>) -> CodegenReturn {
program.gen(&mut self, Context::default());
let source_text = self.into_source_text();
let source_map = self.sourcemap_builder.into_sourcemap();
CodegenReturn { source_text, source_map }
}
pub fn build(&mut self, program: &Program<'_>) -> String {
program.gen(self, Context::default());
self.into_code()
}
pub fn into_code(&mut self) -> String {
// SAFETY: criteria of `from_utf8_unchecked`.are met.
pub fn into_source_text(&mut self) -> String {
// SAFETY: criteria of `from_utf8_unchecked` are met.
unsafe { String::from_utf8_unchecked(std::mem::take(&mut self.code)) }
}
pub fn into_sourcemap(self) -> sourcemap::SourceMap {
self.sourcemap_builder.into_sourcemap()
}
fn code(&self) -> &Vec<u8> {
&self.code
}
@ -174,7 +179,7 @@ impl<const MINIFY: bool> Codegen<MINIFY> {
}
fn peek_nth(&self, n: usize) -> Option<char> {
// SAFETY: criteria of `from_utf8_unchecked`.are met.
// SAFETY: criteria of `from_utf8_unchecked` are met.
unsafe { from_utf8_unchecked(self.code()) }.chars().nth_back(n)
}

View file

@ -46,20 +46,15 @@ impl Default for SourcemapBuilder {
}
impl SourcemapBuilder {
pub fn with_enable_sourcemap(&mut self, enable_sourcemap: bool) -> &mut Self {
self.enable_sourcemap = enable_sourcemap;
self
}
pub fn with_source_and_name(&mut self, source: &str, name: &str) -> &mut Self {
pub fn with_name_and_source(&mut self, name: &str, source: &str) {
self.enable_sourcemap = true;
self.line_offset_tables = Self::generate_line_offset_tables(source);
self.source_id = self.sourcemap_builder.add_source(name);
self.sourcemap_builder.set_source_contents(self.source_id, Some(source));
self
}
pub fn into_sourcemap(self) -> sourcemap::SourceMap {
self.sourcemap_builder.into_sourcemap()
pub fn into_sourcemap(self) -> Option<sourcemap::SourceMap> {
self.enable_sourcemap.then(|| self.sourcemap_builder.into_sourcemap())
}
pub fn add_source_mapping(&mut self, output: &Vec<u8>, position: u32, name: Option<&str>) {
@ -314,7 +309,7 @@ mod test {
fn assert_mapping(source: &str, mappings: &[(u32, u32, u32)]) {
let mut builder = SourcemapBuilder::default();
builder.with_source_and_name(source, "x.js");
builder.with_name_and_source("x.js", source);
for (position, expected_line, expected_col) in mappings.iter().copied() {
let (line, col) = builder.search_original_line_and_column(position);
assert_eq!(
@ -329,7 +324,7 @@ mod test {
fn add_source_mapping() {
fn create_mappings(source: &str, line: u32, column: u32) {
let mut builder = SourcemapBuilder::default();
builder.with_enable_sourcemap(true).with_source_and_name(source, "x.js");
builder.with_name_and_source("x.js", source);
let output: Vec<u8> = source.as_bytes().into();
for (i, _ch) in source.char_indices() {
#[allow(clippy::cast_possible_truncation)]

View file

@ -8,7 +8,8 @@ fn test(source_text: &str, expected: &str) {
let source_type = SourceType::default().with_module(true);
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
let result = Codegen::<false>::new(source_text.len(), CodegenOptions::default()).build(program);
let result =
Codegen::<false>::new(source_text, CodegenOptions::default()).build(program).source_text;
assert_eq!(expected, result, "for source {source_text}, expect {expected}, got {result}");
}
@ -20,9 +21,8 @@ fn test_ts(source_text: &str, expected: &str, is_typescript_definition: bool) {
.with_module(true);
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
let result =
Codegen::<false>::new(source_text.len(), CodegenOptions { enable_typescript: true })
.build(program);
let codegen_options = CodegenOptions { enable_typescript: true, ..CodegenOptions::default() };
let result = Codegen::<false>::new(source_text, codegen_options).build(program).source_text;
assert_eq!(expected, result, "for source {source_text}, expect {expected}, got {result}");
}

View file

@ -151,7 +151,7 @@ impl<'a> LintContext<'a> {
#[allow(clippy::unused_self)]
pub fn codegen(&self) -> Codegen<false> {
Codegen::<false>::new(0, CodegenOptions::default())
Codegen::<false>::new("", CodegenOptions::default())
}
/* JSDoc */

View file

@ -97,7 +97,7 @@ impl NoUnsafeNegation {
expr.operator.gen(&mut codegen, Context::default());
codegen.print_expression(&expr.right);
codegen.print(b')');
codegen.into_code()
codegen.into_source_text()
};
Fix::new(modified_code, expr.span)
};

View file

@ -187,7 +187,7 @@ fn build_code(expr: &CallExpression, ctx: &LintContext) -> (String, Span) {
formatter.print(b')');
}
(formatter.into_code(), expr.span)
(formatter.into_source_text(), expr.span)
}
#[test]

View file

@ -75,7 +75,7 @@ impl Rule for RequireNumberToFixedDigitsArgument {
formatter.print_str(span_source_code.as_bytes());
formatter.print_str(b"0)");
formatter.into_code()
formatter.into_source_text()
};
Fix::new(modified_code, parenthesis_span)

View file

@ -94,7 +94,7 @@ impl Rule for SwitchCaseBraces {
.for_each(|x| x.gen(&mut formatter, Context::default()));
formatter.print(b'}');
formatter.into_code()
formatter.into_source_text()
};
Fix::new(modified_code, case.span)

View file

@ -43,8 +43,9 @@ fn minify(source_text: &str, source_type: SourceType, mangle: bool, whitespace:
let options = MinifierOptions { mangle, ..MinifierOptions::default() };
Minifier::new(options).build(&allocator, program);
if whitespace {
Codegen::<true>::new(source_text.len(), CodegenOptions::default()).build(program)
Codegen::<true>::new(source_text, CodegenOptions::default()).build(program)
} else {
Codegen::<false>::new(source_text.len(), CodegenOptions::default()).build(program)
Codegen::<false>::new(source_text, CodegenOptions::default()).build(program)
}
.source_text
}

View file

@ -19,7 +19,7 @@ pub(crate) fn minify(
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
Minifier::new(options).build(&allocator, program);
Codegen::<true>::new(source_text.len(), CodegenOptions::default()).build(program)
Codegen::<true>::new(source_text, CodegenOptions::default()).build(program).source_text
}
pub(crate) fn test(source_text: &str, expected: &str) {

View file

@ -56,7 +56,7 @@ fn main() {
Transformer::new(&allocator, source_type, semantic, transform_options).build(program).unwrap();
let printed =
Codegen::<false>::new(source_text.len(), CodegenOptions::default()).build(program);
Codegen::<false>::new(&source_text, CodegenOptions::default()).build(program).source_text;
println!("Transformed:\n");
println!("{printed}");
}

View file

@ -31,7 +31,7 @@ impl Tester {
}
}
fn transform(&self, source_text: &str) -> Result<std::string::String, std::vec::Vec<Error>> {
fn transform(&self, source_text: &str) -> Result<String, std::vec::Vec<Error>> {
let program = Parser::new(&self.allocator, source_text, self.source_type).parse().program;
let semantic = SemanticBuilder::new(source_text, self.source_type)
.build_module_record(PathBuf::new(), &program)
@ -41,14 +41,13 @@ impl Tester {
let program = self.allocator.alloc(program);
Transformer::new(&self.allocator, self.source_type, semantic, self.options.clone())
.build(program)
.map(move |()| {
Codegen::<false>::new(source_text.len(), CodegenOptions::default()).build(program)
})
.build(program)?;
Ok(Codegen::<false>::new(source_text, CodegenOptions::default()).build(program).source_text)
}
fn codegen(&self, source_text: &str) -> String {
let program = Parser::new(&self.allocator, source_text, self.source_type).parse().program;
Codegen::<false>::new(source_text.len(), CodegenOptions::default()).build(&program)
Codegen::<false>::new(source_text, CodegenOptions::default()).build(&program).source_text
}
}

View file

@ -264,18 +264,14 @@ impl Oxc {
Minifier::new(options).build(&allocator, program);
}
let codegen_options = CodegenOptions {
enable_typescript: codegen_options.enable_typescript,
..CodegenOptions::default()
};
self.codegen_text = if minifier_options.whitespace() {
Codegen::<true>::new(
source_text.len(),
CodegenOptions { enable_typescript: codegen_options.enable_typescript },
)
.build(program)
Codegen::<true>::new(source_text, codegen_options).build(program).source_text
} else {
Codegen::<false>::new(
source_text.len(),
CodegenOptions { enable_typescript: codegen_options.enable_typescript },
)
.build(program)
Codegen::<false>::new(source_text, codegen_options).build(program).source_text
};
Ok(())

View file

@ -14,11 +14,14 @@ fn bench_codegen_sourcemap(criterion: &mut Criterion) {
group.bench_with_input(id, &file.source_text, |b, source_text| {
let allocator = Allocator::default();
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let codegen_options = CodegenOptions::default();
let codegen_options = CodegenOptions {
enable_source_map: Some(file.file_name.clone()),
..CodegenOptions::default()
};
b.iter_with_large_drop(|| {
let mut codegen = Codegen::<false>::new(source_text.len(), codegen_options);
codegen.with_sourcemap(source_text, "").build(&program);
codegen.into_sourcemap();
Codegen::<false>::new(source_text, codegen_options.clone())
.build(&program)
.source_map
});
});
}

View file

@ -72,11 +72,12 @@ fn get_normal_result(
let options = CodegenOptions::default();
let allocator = Allocator::default();
let parse_result1 = Parser::new(&allocator, source_text, source_type).parse();
let source_text1 =
Codegen::<false>::new(source_text.len(), options).build(&parse_result1.program);
let source_text1 = Codegen::<false>::new(source_text, options.clone())
.build(&parse_result1.program)
.source_text;
let parse_result2 = Parser::new(&allocator, &source_text1, source_type).parse();
let source_text2 =
Codegen::<false>::new(source_text1.len(), options).build(&parse_result2.program);
Codegen::<false>::new(&source_text1, options).build(&parse_result2.program).source_text;
let result = source_text1 == source_text2;
if !result {
@ -113,11 +114,12 @@ fn get_minify_result(
let options = CodegenOptions::default();
let allocator = Allocator::default();
let parse_result1 = Parser::new(&allocator, source_text, source_type).parse();
let source_text1 =
Codegen::<true>::new(source_text.len(), options).build(&parse_result1.program);
let source_text1 = Codegen::<true>::new(source_text, options.clone())
.build(&parse_result1.program)
.source_text;
let parse_result2 = Parser::new(&allocator, source_text1.as_str(), source_type).parse();
let source_text2 =
Codegen::<true>::new(source_text1.len(), options).build(&parse_result2.program);
Codegen::<true>::new(&source_text1, options).build(&parse_result2.program).source_text;
let result = source_text1 == source_text2;
if !result {
@ -151,14 +153,15 @@ fn get_typescript_result(
source_text: &str,
source_type: SourceType,
) -> bool {
let options = CodegenOptions { enable_typescript: true };
let options = CodegenOptions { enable_source_map: None, enable_typescript: true };
let allocator = Allocator::default();
let parse_result1 = Parser::new(&allocator, source_text, source_type).parse();
let source_text1 =
Codegen::<false>::new(source_text.len(), options).build(&parse_result1.program);
let source_text1 = Codegen::<false>::new(source_text, options.clone())
.build(&parse_result1.program)
.source_text;
let parse_result2 = Parser::new(&allocator, &source_text1, source_type).parse();
let source_text2 =
Codegen::<false>::new(source_text1.len(), options).build(&parse_result2.program);
Codegen::<false>::new(&source_text1, options).build(&parse_result2.program).source_text;
let result = source_text1 == source_text2;
if !result {

View file

@ -100,5 +100,5 @@ fn minify(source_text: &str, source_type: SourceType, options: MinifierOptions)
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
Minifier::new(options).build(&allocator, program);
Codegen::<true>::new(source_text.len(), CodegenOptions::default()).build(program)
Codegen::<true>::new(source_text, CodegenOptions::default()).build(program).source_text
}

View file

@ -138,8 +138,9 @@ impl Case for CodegenRuntimeTest262Case {
let source_type = SourceType::default().with_module(is_module);
let allocator = Allocator::default();
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let mut text = Codegen::<false>::new(source_text.len(), CodegenOptions::default())
.build(&program);
let mut text = Codegen::<false>::new(source_text, CodegenOptions::default())
.build(&program)
.source_text;
if is_only_strict {
text = format!("\"use strict\";\n{text}");
}

View file

@ -1,12 +1,17 @@
use crate::suite::{Case, Suite, TestResult};
use oxc_span::SourceType;
use oxc_tasks_common::{project_root, TestFiles};
use std::io::Write;
use std::{
fs::File,
io::Write,
path::{Path, PathBuf},
};
use oxc_allocator::Allocator;
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_parser::Parser;
use oxc_span::SourceType;
use oxc_tasks_common::{project_root, TestFiles};
use crate::suite::{Case, Suite, TestResult};
static FIXTURES_PATH: &str =
"tasks/coverage/babel/packages/babel-generator/test/fixtures/sourcemaps";
@ -188,8 +193,8 @@ impl Case for SourcemapCase {
fn execute(&mut self, source_type: SourceType) -> TestResult {
let source_text = self.code();
let allocator = oxc_allocator::Allocator::default();
let ret = oxc_parser::Parser::new(&allocator, source_text, source_type).parse();
let allocator = Allocator::default();
let ret = Parser::new(&allocator, source_text, source_type).parse();
if !ret.errors.is_empty() {
if let Some(error) = ret.errors.into_iter().next() {
@ -198,10 +203,13 @@ impl Case for SourcemapCase {
}
}
let codegen_options = oxc_codegen::CodegenOptions::default();
let mut codegen = oxc_codegen::Codegen::<false>::new(source_text.len(), codegen_options);
let content = codegen.with_sourcemap(source_text, "").build(&ret.program);
let map = codegen.into_sourcemap();
let codegen_options = CodegenOptions {
enable_source_map: Some(self.path.to_string_lossy().to_string()),
..CodegenOptions::default()
};
let codegen_ret = Codegen::<false>::new(source_text, codegen_options).build(&ret.program);
let content = codegen_ret.source_text;
let map = codegen_ret.source_map.unwrap();
let tokens = map
.tokens()
.map(|token| {

View file

@ -73,7 +73,7 @@ fn minify(source_text: &str, source_type: SourceType, options: MinifierOptions)
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
Minifier::new(options).build(&allocator, program);
Codegen::<true>::new(source_text.len(), CodegenOptions::default()).build(program)
Codegen::<true>::new(source_text, CodegenOptions::default()).build(program).source_text
}
fn gzip_size(s: &str) -> usize {

View file

@ -179,8 +179,9 @@ pub trait TestCase {
.build(transformed_program);
result.map(|()| {
Codegen::<false>::new(source_text.len(), CodegenOptions::default())
Codegen::<false>::new(&source_text, CodegenOptions::default())
.build(transformed_program)
.source_text
})
}
}
@ -247,15 +248,15 @@ impl TestCase for ConformanceTestCase {
let transformer =
Transformer::new(&allocator, source_type, semantic, transform_options.clone());
let codegen_options = CodegenOptions::default();
let mut transformed_code = String::new();
let mut actual_errors = String::new();
let result = transformer.build(program);
if result.is_ok() {
transformed_code =
Codegen::<false>::new(input.len(), CodegenOptions::default()).build(program);
Codegen::<false>::new(&input, codegen_options.clone()).build(program).source_text;
} else {
actual_errors =
result.err().unwrap().iter().map(std::string::ToString::to_string).collect();
actual_errors = result.err().unwrap().iter().map(ToString::to_string).collect();
}
let babel_options = self.options();
@ -268,12 +269,12 @@ impl TestCase for ConformanceTestCase {
}
// The transformation should be equal to input.js If output.js does not exist.
let program = Parser::new(&allocator, &input, source_type).parse().program;
Codegen::<false>::new(input.len(), CodegenOptions::default()).build(&program)
Codegen::<false>::new(&input, codegen_options.clone()).build(&program).source_text
},
|output| {
// Get expected code by parsing the source text, so we can get the same code generated result.
let program = Parser::new(&allocator, &output, source_type).parse().program;
Codegen::<false>::new(output.len(), CodegenOptions::default()).build(&program)
Codegen::<false>::new(&output, codegen_options.clone()).build(&program).source_text
},
);
@ -333,8 +334,9 @@ impl ExecTestCase {
let source_type = SourceType::from_path(&target_path).unwrap();
let transformed_program =
Parser::new(&allocator, &source_text, source_type).parse().program;
let result = Codegen::<false>::new(source_text.len(), CodegenOptions::default())
.build(&transformed_program);
let result = Codegen::<false>::new(&source_text, CodegenOptions::default())
.build(&transformed_program)
.source_text;
fs::write(&target_path, result).unwrap();

View file

@ -127,8 +127,9 @@ impl TypeScriptFixtures {
result
.map(|()| {
Codegen::<false>::new(source_text.len(), CodegenOptions::default())
Codegen::<false>::new(&source_text, CodegenOptions::default())
.build(transformed_program)
.source_text
})
.map_err(|e| e.iter().map(ToString::to_string).collect())
}