refactor(minifier): make the minifier api only accept an ast (#990)

This commit is contained in:
Boshen 2023-10-14 00:51:29 +08:00 committed by GitHub
parent f32bf27106
commit 801d78a3c6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 116 additions and 66 deletions

3
Cargo.lock generated
View file

@ -1669,7 +1669,10 @@ dependencies = [
"humansize", "humansize",
"jemallocator", "jemallocator",
"mimalloc", "mimalloc",
"oxc_allocator",
"oxc_codegen",
"oxc_minifier", "oxc_minifier",
"oxc_parser",
"oxc_span", "oxc_span",
"oxc_tasks_common", "oxc_tasks_common",
] ]

View file

@ -18,18 +18,19 @@ doctest = false
[dependencies] [dependencies]
oxc_allocator = { workspace = true } oxc_allocator = { workspace = true }
oxc_span = { workspace = true } oxc_span = { workspace = true }
oxc_parser = { workspace = true }
oxc_ast = { workspace = true } oxc_ast = { workspace = true }
oxc_semantic = { workspace = true } oxc_semantic = { workspace = true }
oxc_syntax = { workspace = true } oxc_syntax = { workspace = true }
oxc_index = { workspace = true } oxc_index = { workspace = true }
oxc_codegen = { workspace = true }
num-bigint = { workspace = true } num-bigint = { workspace = true }
itertools = { workspace = true } itertools = { workspace = true }
num-traits = { workspace = true } num-traits = { workspace = true }
[dev-dependencies] [dev-dependencies]
oxc_parser = { workspace = true }
oxc_codegen = { workspace = true }
insta = { workspace = true } insta = { workspace = true }
walkdir = { workspace = true } walkdir = { workspace = true }
pico-args = { workspace = true } pico-args = { workspace = true }

View file

@ -1,7 +1,11 @@
use std::path::Path; use std::path::Path;
use oxc_allocator::Allocator;
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_minifier::{Minifier, MinifierOptions}; use oxc_minifier::{Minifier, MinifierOptions};
use oxc_parser::Parser;
use oxc_span::SourceType; use oxc_span::SourceType;
use pico_args::Arguments; use pico_args::Arguments;
#[cfg(not(target_env = "msvc"))] #[cfg(not(target_env = "msvc"))]
@ -29,12 +33,20 @@ fn main() {
let source_type = SourceType::from_path(path).unwrap(); let source_type = SourceType::from_path(path).unwrap();
let options = MinifierOptions { mangle, ..MinifierOptions::default() }; let options = MinifierOptions { mangle, ..MinifierOptions::default() };
let printed = Minifier::new(&source_text, source_type, options).build(); let printed = minify(&source_text, source_type, options);
println!("{printed}"); println!("{printed}");
if twice { if twice {
let options = MinifierOptions { mangle, ..MinifierOptions::default() }; let options = MinifierOptions { mangle, ..MinifierOptions::default() };
let printed = Minifier::new(&printed, source_type, options).build(); let printed = minify(&printed, source_type, options);
println!("{printed}"); println!("{printed}");
} }
} }
fn minify(source_text: &str, source_type: SourceType, options: MinifierOptions) -> String {
let allocator = Allocator::default();
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
Minifier::new(options).build(&allocator, program);
Codegen::<true>::new(source_text.len(), CodegenOptions).build(program)
}

View file

@ -4,52 +4,39 @@ mod compressor;
mod mangler; mod mangler;
use oxc_allocator::Allocator; use oxc_allocator::Allocator;
use oxc_codegen::Codegen; use oxc_ast::ast::Program;
use oxc_parser::Parser;
use oxc_span::SourceType;
pub use crate::{ pub use crate::{
compressor::{CompressOptions, Compressor}, compressor::{CompressOptions, Compressor},
mangler::ManglerBuilder, mangler::ManglerBuilder,
}; };
pub use oxc_codegen::CodegenOptions;
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
pub struct MinifierOptions { pub struct MinifierOptions {
pub mangle: bool, pub mangle: bool,
pub compress: CompressOptions, pub compress: CompressOptions,
pub codegen: CodegenOptions,
} }
impl Default for MinifierOptions { impl Default for MinifierOptions {
fn default() -> Self { fn default() -> Self {
Self { mangle: true, compress: CompressOptions::default(), codegen: CodegenOptions } Self { mangle: true, compress: CompressOptions::default() }
} }
} }
pub struct Minifier<'a> { pub struct Minifier {
source_text: &'a str,
source_type: SourceType,
options: MinifierOptions, options: MinifierOptions,
} }
impl<'a> Minifier<'a> { impl Minifier {
pub fn new(source_text: &'a str, source_type: SourceType, options: MinifierOptions) -> Self { pub fn new(options: MinifierOptions) -> Self {
Self { source_text, source_type, options } Self { options }
} }
pub fn build(self) -> String { pub fn build<'a>(self, allocator: &'a Allocator, program: &mut Program<'a>) {
let allocator = Allocator::default(); Compressor::new(allocator, self.options.compress).build(program);
let ret = Parser::new(&allocator, self.source_text, self.source_type).parse();
let program = allocator.alloc(ret.program);
Compressor::new(&allocator, self.options.compress).build(program);
let codegen = Codegen::<true>::new(self.source_text.len(), self.options.codegen);
// if self.options.mangle { // if self.options.mangle {
// let mangler = ManglerBuilder.build(program); // let mangler = ManglerBuilder.build(program);
// printer.with_mangler(mangler); // printer.with_mangler(mangler);
// } // }
codegen.build(program)
} }
} }

View file

@ -6,9 +6,24 @@ mod oxc;
mod tdewolff; mod tdewolff;
mod terser; mod terser;
use oxc_minifier::{CodegenOptions, CompressOptions, Minifier, MinifierOptions}; use oxc_allocator::Allocator;
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_minifier::{CompressOptions, Minifier, MinifierOptions};
use oxc_parser::Parser;
use oxc_span::SourceType; use oxc_span::SourceType;
pub(crate) fn minify(
source_text: &str,
source_type: SourceType,
options: MinifierOptions,
) -> String {
let allocator = Allocator::default();
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
Minifier::new(options).build(&allocator, program);
Codegen::<true>::new(source_text.len(), CodegenOptions).build(program)
}
pub(crate) fn test(source_text: &str, expected: &str) { pub(crate) fn test(source_text: &str, expected: &str) {
let options = MinifierOptions { mangle: false, ..MinifierOptions::default() }; let options = MinifierOptions { mangle: false, ..MinifierOptions::default() };
test_with_options(source_text, expected, options); test_with_options(source_text, expected, options);
@ -16,7 +31,7 @@ pub(crate) fn test(source_text: &str, expected: &str) {
pub(crate) fn test_with_options(source_text: &str, expected: &str, options: MinifierOptions) { pub(crate) fn test_with_options(source_text: &str, expected: &str, options: MinifierOptions) {
let source_type = SourceType::default(); let source_type = SourceType::default();
let minified = Minifier::new(source_text, source_type, options).build(); let minified = minify(source_text, source_type, options);
assert_eq!(expected, minified, "for source {source_text}"); assert_eq!(expected, minified, "for source {source_text}");
} }
@ -27,17 +42,16 @@ pub(crate) fn test_same(source_text: &str) {
pub(crate) fn test_reparse(source_text: &str) { pub(crate) fn test_reparse(source_text: &str) {
let source_type = SourceType::default(); let source_type = SourceType::default();
let options = MinifierOptions { mangle: false, ..MinifierOptions::default() }; let options = MinifierOptions { mangle: false, ..MinifierOptions::default() };
let minified = Minifier::new(source_text, source_type, options).build(); let minified = minify(source_text, source_type, options);
let minified2 = Minifier::new(&minified, source_type, options).build(); let minified2 = minify(&minified, source_type, options);
assert_eq!(minified, minified2, "for source {source_text}"); assert_eq!(minified, minified2, "for source {source_text}");
} }
pub(crate) fn test_without_compress_booleans(source_text: &str, expected: &str) { pub(crate) fn test_without_compress_booleans(source_text: &str, expected: &str) {
let source_type = SourceType::default(); let source_type = SourceType::default();
let compress_options = CompressOptions { booleans: false, ..CompressOptions::default() }; let compress_options = CompressOptions { booleans: false, ..CompressOptions::default() };
let options = let options = MinifierOptions { mangle: false, compress: compress_options };
MinifierOptions { mangle: false, compress: compress_options, codegen: CodegenOptions }; let minified = minify(source_text, source_type, options);
let minified = Minifier::new(source_text, source_type, options).build();
assert_eq!(expected, minified, "for source {source_text}"); assert_eq!(expected, minified, "for source {source_text}");
} }
@ -50,7 +64,7 @@ where
let snapshot: String = sources let snapshot: String = sources
.into_iter() .into_iter()
.map(|source| { .map(|source| {
let minified = Minifier::new(source, source_type, options).build(); let minified = minify(source, source_type, options);
format!( format!(
"==================================== SOURCE ==================================== "==================================== SOURCE ====================================
{source} {source}

View file

@ -23,7 +23,6 @@ fn console_removal() {
let options = MinifierOptions { let options = MinifierOptions {
mangle: false, mangle: false,
compress: CompressOptions { drop_console: true, ..CompressOptions::default() }, compress: CompressOptions { drop_console: true, ..CompressOptions::default() },
..MinifierOptions::default()
}; };
test_with_options("console.log('hi')", "", options); test_with_options("console.log('hi')", "", options);
test_with_options("let x = console.error('oops')", "let x", options); test_with_options("let x = console.error('oops')", "let x", options);

View file

@ -1,11 +1,13 @@
use oxc_allocator::Allocator; use oxc_allocator::Allocator;
#[allow(clippy::wildcard_imports)] #[allow(clippy::wildcard_imports)]
use oxc_ast::ast::*; use oxc_ast::ast::*;
use oxc_minifier::{CodegenOptions, CompressOptions, Minifier, MinifierOptions}; use oxc_minifier::{CompressOptions, MinifierOptions};
use oxc_parser::Parser; use oxc_parser::Parser;
use oxc_span::{SourceType, Span}; use oxc_span::{SourceType, Span};
use walkdir::WalkDir; use walkdir::WalkDir;
use super::minify;
#[test] #[test]
fn test() { fn test() {
let files = WalkDir::new("tests/terser/fixtures") let files = WalkDir::new("tests/terser/fixtures")
@ -65,12 +67,8 @@ impl TestCase {
} }
let source_type = SourceType::default(); let source_type = SourceType::default();
let options = MinifierOptions { let options = MinifierOptions { mangle: false, compress: self.compress_options };
mangle: false, let minified_source_text = minify(self.input.as_ref(), source_type, options);
compress: self.compress_options,
codegen: CodegenOptions,
};
let minified_source_text = Minifier::new(self.input.as_ref(), source_type, options).build();
assert_eq!( assert_eq!(
remove_whitespace(minified_source_text.as_str()), remove_whitespace(minified_source_text.as_str()),
remove_whitespace(self.expect.as_ref()), remove_whitespace(self.expect.as_ref()),

View file

@ -48,6 +48,7 @@ oxc_semantic = { workspace = true }
oxc_resolver = { workspace = true } oxc_resolver = { workspace = true }
oxc_linter = { workspace = true } oxc_linter = { workspace = true }
oxc_transformer = { workspace = true } oxc_transformer = { workspace = true }
rayon = { workspace = true } rayon = { workspace = true }
criterion = { workspace = true } criterion = { workspace = true }
codspeed-criterion-compat = { workspace = true, optional = true } codspeed-criterion-compat = { workspace = true, optional = true }

View file

@ -6,9 +6,12 @@ static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
#[global_allocator] #[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
use oxc_benchmark::{criterion_group, criterion_main, BenchmarkId, Criterion}; use oxc_allocator::Allocator;
use oxc_minifier::{Minifier, MinifierOptions}; use oxc_minifier::{Minifier, MinifierOptions};
use oxc_parser::Parser;
use oxc_span::SourceType; use oxc_span::SourceType;
use oxc_benchmark::{criterion_group, criterion_main, BenchmarkId, Criterion};
use oxc_tasks_common::TestFiles; use oxc_tasks_common::TestFiles;
fn bench_minifier(criterion: &mut Criterion) { fn bench_minifier(criterion: &mut Criterion) {
@ -20,7 +23,13 @@ fn bench_minifier(criterion: &mut Criterion) {
|b, source_text| { |b, source_text| {
let source_type = SourceType::from_path(&file.file_name).unwrap(); let source_type = SourceType::from_path(&file.file_name).unwrap();
let options = MinifierOptions::default(); let options = MinifierOptions::default();
b.iter_with_large_drop(|| Minifier::new(source_text, source_type, options).build()); b.iter_with_large_drop(|| {
let allocator = Allocator::default();
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
Minifier::new(options).build(&allocator, program);
allocator
});
}, },
); );
} }

View file

@ -1,6 +1,9 @@
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use oxc_allocator::Allocator;
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_minifier::{CompressOptions, Minifier, MinifierOptions}; use oxc_minifier::{CompressOptions, Minifier, MinifierOptions};
use oxc_parser::Parser;
use oxc_span::SourceType; use oxc_span::SourceType;
use crate::{ use crate::{
@ -83,11 +86,19 @@ fn get_result(source_text: &str, source_type: SourceType) -> TestResult {
compress: CompressOptions { evaluate: false, ..CompressOptions::default() }, compress: CompressOptions { evaluate: false, ..CompressOptions::default() },
..MinifierOptions::default() ..MinifierOptions::default()
}; };
let source_text1 = Minifier::new(source_text, source_type, options).build(); let source_text1 = minify(source_text, source_type, options);
let source_text2 = Minifier::new(&source_text1, source_type, options).build(); let source_text2 = minify(&source_text1, source_type, options);
if source_text1 == source_text2 { if source_text1 == source_text2 {
TestResult::Passed TestResult::Passed
} else { } else {
TestResult::ParseError(String::new(), false) TestResult::ParseError(String::new(), false)
} }
} }
fn minify(source_text: &str, source_type: SourceType, options: MinifierOptions) -> String {
let allocator = Allocator::default();
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
Minifier::new(options).build(&allocator, program);
Codegen::<true>::new(source_text.len(), CodegenOptions).build(program)
}

View file

@ -10,6 +10,9 @@ doctest = false
[dependencies] [dependencies]
oxc_span = { workspace = true } oxc_span = { workspace = true }
oxc_allocator = { workspace = true }
oxc_parser = { workspace = true }
oxc_codegen = { workspace = true }
oxc_minifier = { workspace = true } oxc_minifier = { workspace = true }
oxc_tasks_common = { workspace = true } oxc_tasks_common = { workspace = true }

View file

@ -14,7 +14,11 @@ use std::{
use brotlic::{BlockSize, BrotliEncoderOptions, CompressorWriter, Quality, WindowSize}; use brotlic::{BlockSize, BrotliEncoderOptions, CompressorWriter, Quality, WindowSize};
use flate2::{write::GzEncoder, Compression}; use flate2::{write::GzEncoder, Compression};
use humansize::{format_size, DECIMAL}; use humansize::{format_size, DECIMAL};
use oxc_allocator::Allocator;
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_minifier::{CompressOptions, Minifier, MinifierOptions}; use oxc_minifier::{CompressOptions, Minifier, MinifierOptions};
use oxc_parser::Parser;
use oxc_span::SourceType; use oxc_span::SourceType;
use oxc_tasks_common::{project_root, TestFile, TestFiles}; use oxc_tasks_common::{project_root, TestFile, TestFiles};
@ -41,7 +45,7 @@ pub fn run() -> Result<(), io::Error> {
)); ));
out.push_str(&format!(" {:width$}\n", "Brotli", width = 10)); out.push_str(&format!(" {:width$}\n", "Brotli", width = 10));
for file in files.files() { for file in files.files() {
let minified = minify(file); let minified = minify_twice(file);
let s = format!( let s = format!(
"{:width$} -> {:width$} -> {:width$} {:width$} {}\n\n", "{:width$} -> {:width$} -> {:width$} {:width$} {}\n\n",
format_size(file.source_text.len(), DECIMAL), format_size(file.source_text.len(), DECIMAL),
@ -60,18 +64,26 @@ pub fn run() -> Result<(), io::Error> {
Ok(()) Ok(())
} }
fn minify(file: &TestFile) -> String { fn minify_twice(file: &TestFile) -> String {
let source_type = SourceType::from_path(&file.file_name).unwrap(); let source_type = SourceType::from_path(&file.file_name).unwrap();
let options = MinifierOptions { let options = MinifierOptions {
compress: CompressOptions { evaluate: false, ..CompressOptions::default() }, compress: CompressOptions { evaluate: false, ..CompressOptions::default() },
..MinifierOptions::default() ..MinifierOptions::default()
}; };
let source_text1 = Minifier::new(&file.source_text, source_type, options).build(); let source_text1 = minify(&file.source_text, source_type, options);
let source_text2 = Minifier::new(&source_text1, source_type, options).build(); let source_text2 = minify(&source_text1, source_type, options);
assert!(source_text1 == source_text2, "Minification failed for {}", &file.file_name); assert!(source_text1 == source_text2, "Minification failed for {}", &file.file_name);
source_text2 source_text2
} }
fn minify(source_text: &str, source_type: SourceType, options: MinifierOptions) -> String {
let allocator = Allocator::default();
let program = Parser::new(&allocator, source_text, source_type).parse().program;
let program = allocator.alloc(program);
Minifier::new(options).build(&allocator, program);
Codegen::<true>::new(source_text.len(), CodegenOptions).build(program)
}
fn gzip_size(s: &str) -> usize { fn gzip_size(s: &str) -> usize {
let mut e = GzEncoder::new(Vec::new(), Compression::best()); let mut e = GzEncoder::new(Vec::new(), Compression::best());
e.write_all(s.as_bytes()).unwrap(); e.write_all(s.as_bytes()).unwrap();