mirror of
https://github.com/danbulant/oxc
synced 2026-05-19 12:19:15 +00:00
feat: add oxc sourcemap crate (#2825)
The sourcemap implement port from [rust-sourcemap](https://github.com/getsentry/rust-sourcemap), but has some different with it. - Encode sourcemap at parallel, including quote `sourceContent` and encode token to `vlq` mappings. - Avoid `Sourcemap` some methods overhead, like `SourceMap::tokens()` caused extra overhead at common cases. Here using `SourceViewToken` to instead of it.
This commit is contained in:
underfin
GitHub
parent
2c6bfa3dde
commit
b199cb89a2
24 changed files with 1038 additions and 155 deletions
2
.github/workflows/benchmark.yml
vendored
2
.github/workflows/benchmark.yml
vendored
|
|
@ -35,7 +35,7 @@ jobs:
|
|||
matrix:
|
||||
# Run each benchmark in own job.
|
||||
# Linter benchmark is by far the slowest, so split each fixture into own job.
|
||||
component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap]
|
||||
component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, sourcemap]
|
||||
include:
|
||||
- component: linter
|
||||
fixture: 0
|
||||
|
|
|
|||
131
Cargo.lock
generated
131
Cargo.lock
generated
|
|
@ -129,18 +129,6 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitvec"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
|
||||
dependencies = [
|
||||
"funty",
|
||||
"radium",
|
||||
"tap",
|
||||
"wyz",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
|
|
@ -465,22 +453,6 @@ dependencies = [
|
|||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "data-encoding"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
|
||||
|
||||
[[package]]
|
||||
name = "debugid"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
|
|
@ -607,12 +579,6 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "funty"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.30"
|
||||
|
|
@ -834,12 +800,6 @@ dependencies = [
|
|||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "if_chain"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed"
|
||||
|
||||
[[package]]
|
||||
name = "ignore"
|
||||
version = "0.4.22"
|
||||
|
|
@ -1173,7 +1133,7 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"semver 1.0.22",
|
||||
"semver",
|
||||
"syn 2.0.55",
|
||||
]
|
||||
|
||||
|
|
@ -1194,7 +1154,7 @@ checksum = "ad444c78696e8b4de2de8a45468d0cc76630e2ed235c84e102c197b89d295a0b"
|
|||
dependencies = [
|
||||
"indexmap",
|
||||
"rustc-hash",
|
||||
"semver 1.0.22",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
|
|
@ -1346,6 +1306,7 @@ dependencies = [
|
|||
"oxc_minifier",
|
||||
"oxc_parser",
|
||||
"oxc_semantic",
|
||||
"oxc_sourcemap",
|
||||
"oxc_span",
|
||||
"oxc_syntax",
|
||||
"oxc_transformer",
|
||||
|
|
@ -1389,6 +1350,7 @@ dependencies = [
|
|||
"oxc_parser",
|
||||
"oxc_prettier",
|
||||
"oxc_semantic",
|
||||
"oxc_sourcemap",
|
||||
"oxc_span",
|
||||
"oxc_tasks_common",
|
||||
"oxc_transformer",
|
||||
|
|
@ -1426,10 +1388,9 @@ dependencies = [
|
|||
"oxc_allocator",
|
||||
"oxc_ast",
|
||||
"oxc_parser",
|
||||
"oxc_sourcemap",
|
||||
"oxc_span",
|
||||
"oxc_syntax",
|
||||
"rustc-hash",
|
||||
"sourcemap",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1449,6 +1410,7 @@ dependencies = [
|
|||
"oxc_parser",
|
||||
"oxc_prettier",
|
||||
"oxc_semantic",
|
||||
"oxc_sourcemap",
|
||||
"oxc_span",
|
||||
"oxc_tasks_common",
|
||||
"phf",
|
||||
|
|
@ -1723,6 +1685,17 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oxc_sourcemap"
|
||||
version = "0.10.0"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"rayon",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oxc_span"
|
||||
version = "0.10.0"
|
||||
|
|
@ -2026,12 +1999,6 @@ dependencies = [
|
|||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "radium"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
|
|
@ -2183,15 +2150,6 @@ version = "1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
|
||||
dependencies = [
|
||||
"semver 0.9.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.32"
|
||||
|
|
@ -2269,15 +2227,6 @@ version = "1.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
|
||||
dependencies = [
|
||||
"semver-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.22"
|
||||
|
|
@ -2287,12 +2236,6 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver-parser"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||
|
||||
[[package]]
|
||||
name = "seq-macro"
|
||||
version = "0.3.5"
|
||||
|
|
@ -2458,25 +2401,6 @@ dependencies = [
|
|||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sourcemap"
|
||||
version = "8.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf0b8c0d9d32f81aa0ab2b68ab634d9bbce287423606656fddb456ac8601aec3"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bitvec",
|
||||
"data-encoding",
|
||||
"debugid",
|
||||
"if_chain",
|
||||
"rustc-hash",
|
||||
"rustc_version",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"unicode-id-start",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.8"
|
||||
|
|
@ -2532,12 +2456,6 @@ dependencies = [
|
|||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tap"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.10.1"
|
||||
|
|
@ -2944,12 +2862,6 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.0"
|
||||
|
|
@ -3223,15 +3135,6 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wyz"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed"
|
||||
dependencies = [
|
||||
"tap",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yansi"
|
||||
version = "1.0.1"
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ oxc_semantic = { version = "0.10.0", path = "crates/oxc_semantic" }
|
|||
oxc_span = { version = "0.10.0", path = "crates/oxc_span" }
|
||||
oxc_syntax = { version = "0.10.0", path = "crates/oxc_syntax" }
|
||||
oxc_transformer = { version = "0.10.0", path = "crates/oxc_transformer" }
|
||||
oxc_sourcemap = { version = "0.10.0", path = "crates/oxc_sourcemap" }
|
||||
|
||||
# publish = false
|
||||
oxc_macros = { path = "crates/oxc_macros" }
|
||||
|
|
@ -160,7 +161,6 @@ petgraph = "0.6.4"
|
|||
rust-lapper = "1.1.0"
|
||||
serde_yaml = "0.9.34"
|
||||
similar = "2.4.0"
|
||||
sourcemap = "8.0.0"
|
||||
textwrap = "0.16.0"
|
||||
unicode-width = "0.1.11"
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ oxc_semantic = { workspace = true, optional = true }
|
|||
oxc_transformer = { workspace = true, optional = true }
|
||||
oxc_minifier = { workspace = true, optional = true }
|
||||
oxc_codegen = { workspace = true, optional = true }
|
||||
oxc_sourcemap = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
serialize = ["oxc_ast/serialize", "oxc_span/serialize", "oxc_syntax/serialize", "oxc_semantic?/serialize"]
|
||||
|
|
@ -38,3 +39,4 @@ semantic = ["oxc_semantic"]
|
|||
transformer = ["oxc_transformer"]
|
||||
minifier = ["oxc_minifier"]
|
||||
codegen = ["oxc_codegen"]
|
||||
sourcemap = ["oxc_sourcemap"]
|
||||
|
|
|
|||
|
|
@ -60,3 +60,9 @@ pub mod codegen {
|
|||
#[doc(inline)]
|
||||
pub use oxc_codegen::*;
|
||||
}
|
||||
|
||||
#[cfg(feature = "sourcemap")]
|
||||
pub mod sourcemap {
|
||||
#[doc(inline)]
|
||||
pub use oxc_sourcemap::*;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,9 +23,8 @@ oxc_ast = { workspace = true }
|
|||
oxc_span = { workspace = true }
|
||||
oxc_allocator = { workspace = true }
|
||||
oxc_syntax = { workspace = true }
|
||||
sourcemap = { workspace = true }
|
||||
oxc_sourcemap = { workspace = true }
|
||||
bitflags = { workspace = true }
|
||||
rustc-hash = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
oxc_parser = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -33,9 +33,7 @@ fn main() -> std::io::Result<()> {
|
|||
.build(&ret.program);
|
||||
|
||||
if let Some(source_map) = source_map {
|
||||
let mut buff = vec![];
|
||||
source_map.to_writer(&mut buff).unwrap();
|
||||
let result = String::from_utf8(buff).unwrap();
|
||||
let result = source_map.to_json_string();
|
||||
let hash = BASE64_STANDARD.encode(format!(
|
||||
"{}\0{}{}\0{}",
|
||||
source_text.len(),
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ mod gen;
|
|||
mod gen_ts;
|
||||
mod operator;
|
||||
mod sourcemap_builder;
|
||||
mod sourcemap_visualizer;
|
||||
use std::str::from_utf8_unchecked;
|
||||
|
||||
#[allow(clippy::wildcard_imports)]
|
||||
|
|
@ -31,7 +30,6 @@ pub use crate::{
|
|||
context::Context,
|
||||
gen::{Gen, GenExpr},
|
||||
operator::Operator,
|
||||
sourcemap_visualizer::SourcemapVisualizer,
|
||||
};
|
||||
// use crate::mangler::Mangler;
|
||||
|
||||
|
|
@ -46,7 +44,7 @@ pub struct CodegenOptions {
|
|||
|
||||
pub struct CodegenReturn {
|
||||
pub source_text: String,
|
||||
pub source_map: Option<sourcemap::SourceMap>,
|
||||
pub source_map: Option<oxc_sourcemap::SourceMap>,
|
||||
}
|
||||
|
||||
pub struct Codegen<const MINIFY: bool> {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ pub struct LineOffsetTable {
|
|||
byte_offset_to_start_of_line: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[allow(clippy::struct_field_names)]
|
||||
pub struct SourcemapBuilder {
|
||||
enable_sourcemap: bool,
|
||||
|
|
@ -26,36 +27,19 @@ pub struct SourcemapBuilder {
|
|||
last_position: Option<u32>,
|
||||
last_search_line: usize,
|
||||
line_offset_tables: Vec<LineOffsetTable>,
|
||||
sourcemap_builder: sourcemap::SourceMapBuilder,
|
||||
sourcemap_builder: oxc_sourcemap::SourceMapBuilder,
|
||||
generated_line: u32,
|
||||
generated_column: u32,
|
||||
}
|
||||
|
||||
impl Default for SourcemapBuilder {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enable_sourcemap: false,
|
||||
source_id: 0,
|
||||
last_generated_update: 0,
|
||||
last_position: None,
|
||||
last_search_line: 0,
|
||||
line_offset_tables: vec![],
|
||||
sourcemap_builder: sourcemap::SourceMapBuilder::new(None),
|
||||
generated_line: 0,
|
||||
generated_column: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SourcemapBuilder {
|
||||
pub fn with_name_and_source(&mut self, name: &str, source: &str) {
|
||||
self.enable_sourcemap = true;
|
||||
self.line_offset_tables = Self::generate_line_offset_tables(source);
|
||||
self.source_id = self.sourcemap_builder.add_source(name);
|
||||
self.sourcemap_builder.set_source_contents(self.source_id, Some(source));
|
||||
self.source_id = self.sourcemap_builder.set_source_and_content(name, source);
|
||||
}
|
||||
|
||||
pub fn into_sourcemap(self) -> Option<sourcemap::SourceMap> {
|
||||
pub fn into_sourcemap(self) -> Option<oxc_sourcemap::SourceMap> {
|
||||
self.enable_sourcemap.then(|| self.sourcemap_builder.into_sourcemap())
|
||||
}
|
||||
|
||||
|
|
@ -67,14 +51,13 @@ impl SourcemapBuilder {
|
|||
let (original_line, original_column) = self.search_original_line_and_column(position);
|
||||
self.update_generated_line_and_column(output);
|
||||
let name_id = name.map(|s| self.sourcemap_builder.add_name(s));
|
||||
self.sourcemap_builder.add_raw(
|
||||
self.sourcemap_builder.add_token(
|
||||
self.generated_line,
|
||||
self.generated_column,
|
||||
original_line,
|
||||
original_column,
|
||||
Some(self.source_id),
|
||||
name_id,
|
||||
false,
|
||||
);
|
||||
self.last_position = Some(position);
|
||||
}
|
||||
|
|
|
|||
26
crates/oxc_sourcemap/Cargo.toml
Normal file
26
crates/oxc_sourcemap/Cargo.toml
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
[package]
|
||||
name = "oxc_sourcemap"
|
||||
version = "0.10.0"
|
||||
authors.workspace = true
|
||||
description.workspace = true
|
||||
edition.workspace = true
|
||||
homepage.workspace = true
|
||||
keywords.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
categories.workspace = true
|
||||
include = ["/src"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
[dependencies]
|
||||
rustc-hash = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
base64-simd = "0.7"
|
||||
4
crates/oxc_sourcemap/README.md
Normal file
4
crates/oxc_sourcemap/README.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
The sourcemap implement port from [rust-sourcemap](https://github.com/getsentry/rust-sourcemap), but has some different with it.
|
||||
|
||||
- Encode sourcemap at parallel, including quote `sourceContent` and encode token to `vlq` mappings.
|
||||
- Avoid `Sourcemap` some methods overhead, like `SourceMap::tokens()`.
|
||||
134
crates/oxc_sourcemap/src/concat_sourcemap_builder.rs
Normal file
134
crates/oxc_sourcemap/src/concat_sourcemap_builder.rs
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
use crate::{token::TokenChunk, SourceMap, Token};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The `ConcatSourceMapBuilder` is a helper to concat sourcemaps.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ConcatSourceMapBuilder {
|
||||
pub(crate) names: Vec<Arc<str>>,
|
||||
pub(crate) sources: Vec<Arc<str>>,
|
||||
pub(crate) source_contents: Vec<Arc<str>>,
|
||||
pub(crate) tokens: Vec<Token>,
|
||||
/// The `token_chunks` is used for encode tokens to vlq mappings at parallel.
|
||||
pub(crate) token_chunks: Vec<TokenChunk>,
|
||||
pub(crate) token_chunk_prev_name_id: u32,
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
impl ConcatSourceMapBuilder {
|
||||
pub fn add_sourcemap(&mut self, sourcemap: &SourceMap, line_offset: u32) {
|
||||
let source_offset = self.sources.len() as u32;
|
||||
let name_offset = self.names.len() as u32;
|
||||
|
||||
// Add `token_chunks`, See `TokenChunk`.
|
||||
if let Some(last_token) = self.tokens.last() {
|
||||
self.token_chunks.push(TokenChunk::new(
|
||||
self.tokens.len() as u32,
|
||||
self.tokens.len() as u32 + sourcemap.tokens.len() as u32,
|
||||
last_token.get_dst_line(),
|
||||
last_token.get_dst_col(),
|
||||
last_token.get_src_line(),
|
||||
last_token.get_src_col(),
|
||||
self.token_chunk_prev_name_id,
|
||||
source_offset - 1,
|
||||
));
|
||||
} else {
|
||||
self.token_chunks.push(TokenChunk::new(
|
||||
0,
|
||||
sourcemap.tokens.len() as u32,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
));
|
||||
}
|
||||
|
||||
// Extend `sources` and `source_contents`.
|
||||
self.sources.reserve(sourcemap.sources.len());
|
||||
for (index, source) in sourcemap.get_sources().enumerate() {
|
||||
let source_content = sourcemap.get_source_content(index as u32).unwrap_or_default();
|
||||
self.sources.push(source.into());
|
||||
self.source_contents.push(source_content.into());
|
||||
}
|
||||
|
||||
// Extend `names`.
|
||||
self.names.reserve(sourcemap.names.len());
|
||||
self.names.extend(sourcemap.get_names().map(Into::into));
|
||||
|
||||
// Extend `tokens`.
|
||||
self.tokens.reserve(sourcemap.tokens.len());
|
||||
let tokens = sourcemap.get_tokens().map(|token| {
|
||||
Token::new(
|
||||
token.get_dst_line() + line_offset,
|
||||
token.get_dst_col(),
|
||||
token.get_src_line(),
|
||||
token.get_src_col(),
|
||||
token.get_source_id().map(|x| x + source_offset),
|
||||
token.get_name_id().map(|x| {
|
||||
self.token_chunk_prev_name_id = x + name_offset;
|
||||
self.token_chunk_prev_name_id
|
||||
}),
|
||||
)
|
||||
});
|
||||
self.tokens.extend(tokens);
|
||||
}
|
||||
|
||||
pub fn into_sourcemap(self) -> SourceMap {
|
||||
SourceMap::new(
|
||||
None,
|
||||
self.names,
|
||||
self.sources,
|
||||
Some(self.source_contents),
|
||||
self.tokens,
|
||||
Some(self.token_chunks),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concat_sourcemap_builder() {
|
||||
let sm1 = SourceMap::new(
|
||||
None,
|
||||
vec!["foo".into(), "foo2".into()],
|
||||
vec!["foo.js".into()],
|
||||
None,
|
||||
vec![Token::new(1, 1, 1, 1, Some(0), Some(0))],
|
||||
None,
|
||||
);
|
||||
let sm2 = SourceMap::new(
|
||||
None,
|
||||
vec!["bar".into()],
|
||||
vec!["bar.js".into()],
|
||||
None,
|
||||
vec![Token::new(1, 1, 1, 1, Some(0), Some(0))],
|
||||
None,
|
||||
);
|
||||
|
||||
let mut builder = ConcatSourceMapBuilder::default();
|
||||
builder.add_sourcemap(&sm1, 0);
|
||||
builder.add_sourcemap(&sm2, 2);
|
||||
|
||||
let sm = SourceMap::new(
|
||||
None,
|
||||
vec!["foo".into(), "foo2".into(), "bar".into()],
|
||||
vec!["foo.js".into(), "bar.js".into()],
|
||||
None,
|
||||
vec![Token::new(1, 1, 1, 1, Some(0), Some(0)), Token::new(3, 1, 1, 1, Some(1), Some(2))],
|
||||
None,
|
||||
);
|
||||
let concat_sm = builder.into_sourcemap();
|
||||
|
||||
assert_eq!(concat_sm.tokens, sm.tokens);
|
||||
assert_eq!(concat_sm.sources, sm.sources);
|
||||
assert_eq!(concat_sm.names, sm.names);
|
||||
assert_eq!(
|
||||
concat_sm.token_chunks,
|
||||
Some(vec![
|
||||
TokenChunk::new(0, 1, 0, 0, 0, 0, 0, 0,),
|
||||
TokenChunk::new(1, 2, 1, 1, 1, 1, 0, 0,)
|
||||
])
|
||||
);
|
||||
|
||||
assert_eq!(sm.to_json_string(), sm.to_json_string());
|
||||
}
|
||||
143
crates/oxc_sourcemap/src/decode.rs
Normal file
143
crates/oxc_sourcemap/src/decode.rs
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
/// Port from https://github.com/getsentry/rust-sourcemap/blob/master/src/decoder.rs
|
||||
/// It is a helper for decode vlq soucemap string to `SourceMap`.
|
||||
use crate::error::{Error, Result};
|
||||
use crate::{SourceMap, Token};
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct JSONSourceMap {
|
||||
file: Option<String>,
|
||||
mappings: String,
|
||||
sources: Vec<String>,
|
||||
sources_content: Option<Vec<String>>,
|
||||
names: Vec<String>,
|
||||
}
|
||||
|
||||
pub fn decode(value: &str) -> Result<SourceMap> {
|
||||
let json: JSONSourceMap = serde_json::from_str(value)?;
|
||||
let file = json.file.map(Into::into);
|
||||
let names = json.names.into_iter().map(Into::into).collect::<Vec<_>>();
|
||||
let sources = json.sources.into_iter().map(Into::into).collect::<Vec<_>>();
|
||||
let source_contents =
|
||||
json.sources_content.map(|v| v.into_iter().map(Into::into).collect::<Vec<_>>());
|
||||
let tokens = decode_mapping(&json.mappings, names.len(), sources.len())?;
|
||||
Ok(SourceMap::new(file, names, sources, source_contents, tokens, None))
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
fn decode_mapping(mapping: &str, names_len: usize, sources_len: usize) -> Result<Vec<Token>> {
|
||||
let mut tokens = vec![];
|
||||
|
||||
let mut dst_col;
|
||||
let mut src_id = 0;
|
||||
let mut src_line = 0;
|
||||
let mut src_col = 0;
|
||||
let mut name_id = 0;
|
||||
let mut nums = Vec::with_capacity(6);
|
||||
|
||||
for (dst_line, line) in mapping.split(';').enumerate() {
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
dst_col = 0;
|
||||
|
||||
for segment in line.split(',') {
|
||||
if segment.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
nums.clear();
|
||||
parse_vlq_segment_into(segment, &mut nums)?;
|
||||
dst_col = (i64::from(dst_col) + nums[0]) as u32;
|
||||
|
||||
let mut src = !0;
|
||||
let mut name = !0;
|
||||
|
||||
if nums.len() > 1 {
|
||||
if nums.len() != 4 && nums.len() != 5 {
|
||||
return Err(Error::BadSegmentSize(nums.len() as u32));
|
||||
}
|
||||
src_id = (i64::from(src_id) + nums[1]) as u32;
|
||||
if src_id >= sources_len as u32 {
|
||||
return Err(Error::BadSourceReference(src_id));
|
||||
}
|
||||
|
||||
src = src_id;
|
||||
src_line = (i64::from(src_line) + nums[2]) as u32;
|
||||
src_col = (i64::from(src_col) + nums[3]) as u32;
|
||||
|
||||
if nums.len() > 4 {
|
||||
name_id = (i64::from(name_id) + nums[4]) as u32;
|
||||
if name_id >= names_len as u32 {
|
||||
return Err(Error::BadNameReference(name_id));
|
||||
}
|
||||
name = name_id;
|
||||
}
|
||||
}
|
||||
|
||||
tokens.push(Token::new(
|
||||
dst_line as u32,
|
||||
dst_col,
|
||||
src_line,
|
||||
src_col,
|
||||
if src == !0 { None } else { Some(src) },
|
||||
if name == !0 { None } else { Some(name) },
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(tokens)
|
||||
}
|
||||
|
||||
#[rustfmt::skip]
|
||||
const B64: [i8; 256] = [ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1 - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, ];
|
||||
|
||||
fn parse_vlq_segment_into(segment: &str, rv: &mut Vec<i64>) -> Result<()> {
|
||||
let mut cur = 0;
|
||||
let mut shift = 0;
|
||||
|
||||
for c in segment.bytes() {
|
||||
let enc = i64::from(B64[c as usize]);
|
||||
let val = enc & 0b11111;
|
||||
let cont = enc >> 5;
|
||||
cur += val.checked_shl(shift).ok_or(Error::VlqOverflow)?;
|
||||
shift += 5;
|
||||
|
||||
if cont == 0 {
|
||||
let sign = cur & 1;
|
||||
cur >>= 1;
|
||||
if sign != 0 {
|
||||
cur = -cur;
|
||||
}
|
||||
rv.push(cur);
|
||||
cur = 0;
|
||||
shift = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if cur != 0 || shift != 0 {
|
||||
Err(Error::VlqLeftover)
|
||||
} else if rv.is_empty() {
|
||||
Err(Error::VlqNoValues)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_sourcemap() {
|
||||
let input = r#"{
|
||||
"version": 3,
|
||||
"sources": ["coolstuff.js"],
|
||||
"sourceRoot": "x",
|
||||
"names": ["x","alert"],
|
||||
"mappings": "AAAA,GAAIA,GAAI,EACR,IAAIA,GAAK,EAAG,CACVC,MAAM"
|
||||
}"#;
|
||||
let sm = SourceMap::from_json_string(input).unwrap();
|
||||
let mut iter = sm.get_source_view_tokens().filter(|token| token.get_name_id().is_some());
|
||||
assert_eq!(iter.next().unwrap().to_tuple(), (Some("coolstuff.js"), 0, 4, Some("x")));
|
||||
assert_eq!(iter.next().unwrap().to_tuple(), (Some("coolstuff.js"), 1, 4, Some("x")));
|
||||
assert_eq!(iter.next().unwrap().to_tuple(), (Some("coolstuff.js"), 2, 2, Some("alert")));
|
||||
assert!(iter.next().is_none());
|
||||
}
|
||||
138
crates/oxc_sourcemap/src/encode.rs
Normal file
138
crates/oxc_sourcemap/src/encode.rs
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
/// Port from https://github.com/getsentry/rust-sourcemap/blob/master/src/encoder.rs
|
||||
/// It is a helper for encode `SourceMap` to vlq sourcemap string, but here some different.
|
||||
/// - Quote `source_content` at parallel.
|
||||
/// - If you using `ConcatSourceMapBuilder`, serialize `tokens` to vlq `mappings` at parallel.
|
||||
use crate::{token::TokenChunk, SourceMap, Token};
|
||||
use rayon::prelude::*;
|
||||
|
||||
pub fn encode(sourcemap: &SourceMap) -> String {
|
||||
let mut buf = String::new();
|
||||
buf.push_str("{\"version\":3,");
|
||||
if let Some(file) = sourcemap.get_file() {
|
||||
buf.push_str("\"file\":\"");
|
||||
buf.push_str(file);
|
||||
buf.push_str("\",");
|
||||
}
|
||||
buf.push_str("\"names\":[");
|
||||
buf.push_str(&sourcemap.names.iter().map(|x| format!("{x:?}")).collect::<Vec<_>>().join(","));
|
||||
buf.push_str("],\"sources\":[");
|
||||
buf.push_str(&sourcemap.sources.iter().map(|x| format!("{x:?}")).collect::<Vec<_>>().join(","));
|
||||
// Quote `source_content` at parallel.
|
||||
if let Some(source_contents) = &sourcemap.source_contents {
|
||||
buf.push_str("],\"sourcesContent\":[");
|
||||
buf.push_str(
|
||||
&source_contents.par_iter().map(|x| format!("{x:?}")).collect::<Vec<_>>().join(","),
|
||||
);
|
||||
}
|
||||
buf.push_str("],\"mappings\":\"");
|
||||
buf.push_str(&serialize_sourcemap_mappings(sourcemap));
|
||||
buf.push_str("\"}");
|
||||
buf
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
fn serialize_sourcemap_mappings(sm: &SourceMap) -> String {
|
||||
sm.token_chunks.as_ref().map_or_else(
|
||||
|| {
|
||||
serialize_mappings(
|
||||
&sm.tokens,
|
||||
&TokenChunk::new(0, sm.tokens.len() as u32, 0, 0, 0, 0, 0, 0),
|
||||
)
|
||||
},
|
||||
|token_chunks| {
|
||||
// Serialize `tokens` to vlq `mappings` at parallel.
|
||||
token_chunks
|
||||
.par_iter()
|
||||
.map(|token_chunk| serialize_mappings(&sm.tokens, token_chunk))
|
||||
.collect::<String>()
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn serialize_mappings(tokens: &[Token], token_chunk: &TokenChunk) -> String {
|
||||
let mut rv = String::new();
|
||||
|
||||
let TokenChunk {
|
||||
start,
|
||||
end,
|
||||
mut prev_dst_line,
|
||||
mut prev_dst_col,
|
||||
mut prev_src_line,
|
||||
mut prev_src_col,
|
||||
mut prev_name_id,
|
||||
mut prev_source_id,
|
||||
} = *token_chunk;
|
||||
|
||||
for (idx, token) in tokens[start as usize..end as usize].iter().enumerate() {
|
||||
if token.get_dst_line() != prev_dst_line {
|
||||
prev_dst_col = 0;
|
||||
while token.get_dst_line() != prev_dst_line {
|
||||
rv.push(';');
|
||||
prev_dst_line += 1;
|
||||
}
|
||||
} else if idx > 0 {
|
||||
if Some(token) == tokens.get(idx - 1) {
|
||||
continue;
|
||||
}
|
||||
rv.push(',');
|
||||
}
|
||||
|
||||
encode_vlq_diff(&mut rv, token.get_dst_col(), prev_dst_col);
|
||||
prev_dst_col = token.get_dst_col();
|
||||
|
||||
if let Some(source_id) = token.get_source_id() {
|
||||
encode_vlq_diff(&mut rv, source_id, prev_source_id);
|
||||
prev_source_id = source_id;
|
||||
encode_vlq_diff(&mut rv, token.get_src_line(), prev_src_line);
|
||||
prev_src_line = token.get_src_line();
|
||||
encode_vlq_diff(&mut rv, token.get_src_col(), prev_src_col);
|
||||
prev_src_col = token.get_src_col();
|
||||
if let Some(name_id) = token.get_name_id() {
|
||||
encode_vlq_diff(&mut rv, name_id, prev_name_id);
|
||||
prev_name_id = name_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rv
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn encode_vlq_diff(out: &mut String, a: u32, b: u32) {
|
||||
encode_vlq(out, i64::from(a) - i64::from(b));
|
||||
}
|
||||
|
||||
const B64_CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
|
||||
fn encode_vlq(out: &mut String, num: i64) {
|
||||
let mut num = if num < 0 { ((-num) << 1) + 1 } else { num << 1 };
|
||||
|
||||
loop {
|
||||
let mut digit = num & 0b11111;
|
||||
num >>= 5;
|
||||
if num > 0 {
|
||||
digit |= 1 << 5;
|
||||
}
|
||||
out.push(B64_CHARS[digit as usize] as char);
|
||||
if num == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode() {
|
||||
let input = r#"{
|
||||
"version": 3,
|
||||
"sources": ["coolstuff.js"],
|
||||
"names": ["x","alert"],
|
||||
"mappings": "AAAA,GAAIA,GAAI,EACR,IAAIA,GAAK,EAAG,CACVC,MAAM"
|
||||
}"#;
|
||||
let sm = SourceMap::from_json_string(input).unwrap();
|
||||
let sm2 = SourceMap::from_json_string(&sm.to_json_string()).unwrap();
|
||||
|
||||
for (tok1, tok2) in sm.get_tokens().zip(sm2.get_tokens()) {
|
||||
assert_eq!(tok1, tok2);
|
||||
}
|
||||
}
|
||||
26
crates/oxc_sourcemap/src/error.rs
Normal file
26
crates/oxc_sourcemap/src/error.rs
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
/// a VLQ string was malformed and data was left over
|
||||
VlqLeftover,
|
||||
/// a VLQ string was empty and no values could be decoded.
|
||||
VlqNoValues,
|
||||
/// The input encoded a number that didn't fit into i64.
|
||||
VlqOverflow,
|
||||
/// `serde_json` parsing failure
|
||||
BadJson(serde_json::Error),
|
||||
/// a mapping segment had an unsupported size
|
||||
BadSegmentSize(u32),
|
||||
/// a reference to a non existing source was encountered
|
||||
BadSourceReference(u32),
|
||||
/// a reference to a non existing name was encountered
|
||||
BadNameReference(u32),
|
||||
}
|
||||
|
||||
/// The result of decoding.
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
impl From<serde_json::Error> for Error {
|
||||
fn from(err: serde_json::Error) -> Error {
|
||||
Error::BadJson(err)
|
||||
}
|
||||
}
|
||||
16
crates/oxc_sourcemap/src/lib.rs
Normal file
16
crates/oxc_sourcemap/src/lib.rs
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
mod concat_sourcemap_builder;
|
||||
#[allow(clippy::cast_sign_loss)]
|
||||
mod decode;
|
||||
mod encode;
|
||||
mod error;
|
||||
mod sourcemap;
|
||||
mod sourcemap_builder;
|
||||
mod sourcemap_visualizer;
|
||||
mod token;
|
||||
|
||||
pub use concat_sourcemap_builder::ConcatSourceMapBuilder;
|
||||
pub use error::Error;
|
||||
pub use sourcemap::SourceMap;
|
||||
pub use sourcemap_builder::SourceMapBuilder;
|
||||
pub use sourcemap_visualizer::SourcemapVisualizer;
|
||||
pub use token::{SourceViewToken, Token};
|
||||
215
crates/oxc_sourcemap/src/sourcemap.rs
Normal file
215
crates/oxc_sourcemap/src/sourcemap.rs
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
use crate::{
|
||||
decode::decode,
|
||||
encode::encode,
|
||||
error::Result,
|
||||
token::{Token, TokenChunk},
|
||||
SourceViewToken,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SourceMap {
|
||||
pub(crate) file: Option<Arc<str>>,
|
||||
pub(crate) names: Vec<Arc<str>>,
|
||||
pub(crate) sources: Vec<Arc<str>>,
|
||||
pub(crate) source_contents: Option<Vec<Arc<str>>>,
|
||||
pub(crate) tokens: Vec<Token>,
|
||||
pub(crate) token_chunks: Option<Vec<TokenChunk>>,
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
impl SourceMap {
|
||||
pub fn new(
|
||||
file: Option<Arc<str>>,
|
||||
names: Vec<Arc<str>>,
|
||||
sources: Vec<Arc<str>>,
|
||||
source_contents: Option<Vec<Arc<str>>>,
|
||||
tokens: Vec<Token>,
|
||||
token_chunks: Option<Vec<TokenChunk>>,
|
||||
) -> Self {
|
||||
Self { file, names, sources, source_contents, tokens, token_chunks }
|
||||
}
|
||||
|
||||
/// Convert `SourceMap` to vlq sourcemap string.
|
||||
#[allow(clippy::missing_errors_doc)]
|
||||
pub fn from_json_string(value: &str) -> Result<Self> {
|
||||
decode(value)
|
||||
}
|
||||
|
||||
/// Convert the vlq sourcemap string to `SourceMap`.
|
||||
pub fn to_json_string(&self) -> String {
|
||||
encode(self)
|
||||
}
|
||||
|
||||
/// Convert `SourceMap` to vlq sourcemap data url.
|
||||
pub fn to_data_url(&self) -> String {
|
||||
let base_64_str =
|
||||
base64_simd::Base64::STANDARD.encode_to_boxed_str(self.to_json_string().as_bytes());
|
||||
format!("data:application/json;charset=utf-8;base64,{base_64_str}")
|
||||
}
|
||||
|
||||
pub fn get_file(&self) -> Option<&str> {
|
||||
self.file.as_deref()
|
||||
}
|
||||
|
||||
pub fn set_file(&mut self, file: &str) {
|
||||
self.file = Some(file.into());
|
||||
}
|
||||
|
||||
pub fn get_names(&self) -> impl Iterator<Item = &str> {
|
||||
self.names.iter().map(std::convert::AsRef::as_ref)
|
||||
}
|
||||
|
||||
pub fn get_sources(&self) -> impl Iterator<Item = &str> {
|
||||
self.sources.iter().map(std::convert::AsRef::as_ref)
|
||||
}
|
||||
|
||||
pub fn get_source_contents(&self) -> Option<impl Iterator<Item = &str>> {
|
||||
self.source_contents.as_ref().map(|v| v.iter().map(std::convert::AsRef::as_ref))
|
||||
}
|
||||
|
||||
pub fn get_token(&self, index: u32) -> Option<&Token> {
|
||||
self.tokens.get(index as usize)
|
||||
}
|
||||
|
||||
/// Get raw tokens.
|
||||
pub fn get_tokens(&self) -> impl Iterator<Item = &Token> {
|
||||
self.tokens.iter()
|
||||
}
|
||||
|
||||
/// Get source view tokens. See [`SourceViewToken`] for more information.
|
||||
pub fn get_source_view_tokens(&self) -> impl Iterator<Item = SourceViewToken<'_>> {
|
||||
self.tokens.iter().map(|token| SourceViewToken::new(token, self))
|
||||
}
|
||||
|
||||
pub fn get_name(&self, id: u32) -> Option<&str> {
|
||||
self.names.get(id as usize).map(std::convert::AsRef::as_ref)
|
||||
}
|
||||
|
||||
pub fn get_source(&self, id: u32) -> Option<&str> {
|
||||
self.sources.get(id as usize).map(std::convert::AsRef::as_ref)
|
||||
}
|
||||
|
||||
pub fn get_source_content(&self, id: u32) -> Option<&str> {
|
||||
self.source_contents
|
||||
.as_ref()
|
||||
.and_then(|x| x.get(id as usize).map(std::convert::AsRef::as_ref))
|
||||
}
|
||||
|
||||
pub fn get_source_and_content(&self, id: u32) -> Option<(&str, &str)> {
|
||||
let source = self.get_source(id)?;
|
||||
let content = self.get_source_content(id)?;
|
||||
Some((source, content))
|
||||
}
|
||||
|
||||
/// Generate a lookup table, it will be used at `lookup_token` or `lookup_source_view_token`.
|
||||
pub fn generate_lookup_table(&self) -> Vec<(u32, u32, u32)> {
|
||||
let mut table = self
|
||||
.tokens
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, token)| (token.dst_line, token.dst_col, idx as u32))
|
||||
.collect::<Vec<_>>();
|
||||
table.sort_unstable();
|
||||
table
|
||||
}
|
||||
|
||||
/// Lookup a token by line and column, it will used at remapping.
|
||||
pub fn lookup_token(
|
||||
&self,
|
||||
lookup_table: &[(u32, u32, u32)],
|
||||
line: u32,
|
||||
col: u32,
|
||||
) -> Option<&Token> {
|
||||
let table = greatest_lower_bound(lookup_table, &(line, col), |table| (table.0, table.1))?;
|
||||
self.get_token(table.2)
|
||||
}
|
||||
|
||||
/// Lookup a token by line and column, it will used at remapping. See `SourceViewToken`.
|
||||
pub fn lookup_source_view_token(
|
||||
&self,
|
||||
lookup_table: &[(u32, u32, u32)],
|
||||
line: u32,
|
||||
col: u32,
|
||||
) -> Option<SourceViewToken<'_>> {
|
||||
self.lookup_token(lookup_table, line, col).map(|token| SourceViewToken::new(token, self))
|
||||
}
|
||||
}
|
||||
|
||||
fn greatest_lower_bound<'a, T, K: Ord, F: Fn(&'a T) -> K>(
|
||||
slice: &'a [T],
|
||||
key: &K,
|
||||
map: F,
|
||||
) -> Option<&'a T> {
|
||||
let mut idx = match slice.binary_search_by_key(key, &map) {
|
||||
Ok(index) => index,
|
||||
Err(index) => {
|
||||
// If there is no match, then we know for certain that the index is where we should
|
||||
// insert a new token, and that the token directly before is the greatest lower bound.
|
||||
return slice.get(index.checked_sub(1)?);
|
||||
}
|
||||
};
|
||||
|
||||
// If we get an exact match, then we need to continue looking at previous tokens to see if
|
||||
// they also match. We use a linear search because the number of exact matches is generally
|
||||
// very small, and almost certainly smaller than the number of tokens before the index.
|
||||
for i in (0..idx).rev() {
|
||||
if map(&slice[i]) == *key {
|
||||
idx = i;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
slice.get(idx)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sourcemap_lookup_token() {
|
||||
let input = r#"{
|
||||
"version": 3,
|
||||
"sources": ["coolstuff.js"],
|
||||
"sourceRoot": "x",
|
||||
"names": ["x","alert"],
|
||||
"mappings": "AAAA,GAAIA,GAAI,EACR,IAAIA,GAAK,EAAG,CACVC,MAAM"
|
||||
}"#;
|
||||
let sm = SourceMap::from_json_string(input).unwrap();
|
||||
let lookup_table = sm.generate_lookup_table();
|
||||
assert_eq!(
|
||||
sm.lookup_source_view_token(&lookup_table, 0, 0).unwrap().to_tuple(),
|
||||
(Some("coolstuff.js"), 0, 0, None)
|
||||
);
|
||||
assert_eq!(
|
||||
sm.lookup_source_view_token(&lookup_table, 0, 3).unwrap().to_tuple(),
|
||||
(Some("coolstuff.js"), 0, 4, Some("x"))
|
||||
);
|
||||
assert_eq!(
|
||||
sm.lookup_source_view_token(&lookup_table, 0, 24).unwrap().to_tuple(),
|
||||
(Some("coolstuff.js"), 2, 8, None)
|
||||
);
|
||||
|
||||
// Lines continue out to infinity
|
||||
assert_eq!(
|
||||
sm.lookup_source_view_token(&lookup_table, 0, 1000).unwrap().to_tuple(),
|
||||
(Some("coolstuff.js"), 2, 8, None)
|
||||
);
|
||||
|
||||
// Token can return prior lines.
|
||||
assert_eq!(
|
||||
sm.lookup_source_view_token(&lookup_table, 1000, 0).unwrap().to_tuple(),
|
||||
(Some("coolstuff.js"), 2, 8, None)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sourcemap_source_view_token() {
|
||||
let sm = SourceMap::new(
|
||||
None,
|
||||
vec!["foo".into()],
|
||||
vec!["foo.js".into()],
|
||||
None,
|
||||
vec![Token::new(1, 1, 1, 1, Some(0), Some(0))],
|
||||
None,
|
||||
);
|
||||
let mut source_view_tokens = sm.get_source_view_tokens();
|
||||
assert_eq!(source_view_tokens.next().unwrap().to_tuple(), (Some("foo.js"), 1, 1, Some("foo")));
|
||||
}
|
||||
89
crates/oxc_sourcemap/src/sourcemap_builder.rs
Normal file
89
crates/oxc_sourcemap/src/sourcemap_builder.rs
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
use crate::{token::Token, SourceMap};
|
||||
|
||||
/// The `SourceMapBuilder` is a helper to generate sourcemap.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SourceMapBuilder {
|
||||
pub(crate) names_map: FxHashMap<Arc<str>, u32>,
|
||||
pub(crate) names: Vec<Arc<str>>,
|
||||
pub(crate) sources: Vec<Arc<str>>,
|
||||
pub(crate) sources_map: FxHashMap<Arc<str>, u32>,
|
||||
pub(crate) source_contents: Vec<Arc<str>>,
|
||||
pub(crate) tokens: Vec<Token>,
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
impl SourceMapBuilder {
|
||||
/// Add item to `SourceMap::name`.
|
||||
pub fn add_name(&mut self, name: &str) -> u32 {
|
||||
let count = self.names.len() as u32;
|
||||
let id = *self.names_map.entry(name.into()).or_insert(count);
|
||||
if id == count {
|
||||
self.names.push(name.into());
|
||||
}
|
||||
id
|
||||
}
|
||||
|
||||
/// Add item to `SourceMap::sources` and `SourceMap::source_contents`.
|
||||
/// If `source` maybe duplicate, please use it.
|
||||
pub fn add_source_and_content(&mut self, source: &str, source_content: &str) -> u32 {
|
||||
let count = self.sources.len() as u32;
|
||||
let id = *self.sources_map.entry(source.into()).or_insert(count);
|
||||
if id == count {
|
||||
self.sources.push(source.into());
|
||||
self.source_contents.push(source_content.into());
|
||||
}
|
||||
id
|
||||
}
|
||||
|
||||
/// Add item to `SourceMap::sources` and `SourceMap::source_contents`.
|
||||
/// If `source` hasn't duplicate，it will avoid extra hash calculation.
|
||||
pub fn set_source_and_content(&mut self, source: &str, source_content: &str) -> u32 {
|
||||
let count = self.sources.len() as u32;
|
||||
self.sources.push(source.into());
|
||||
self.source_contents.push(source_content.into());
|
||||
count
|
||||
}
|
||||
|
||||
/// Add item to `SourceMap::tokens`.
|
||||
pub fn add_token(
|
||||
&mut self,
|
||||
dst_line: u32,
|
||||
dst_col: u32,
|
||||
src_line: u32,
|
||||
src_col: u32,
|
||||
src_id: Option<u32>,
|
||||
name_id: Option<u32>,
|
||||
) {
|
||||
self.tokens.push(Token::new(dst_line, dst_col, src_line, src_col, src_id, name_id));
|
||||
}
|
||||
|
||||
pub fn into_sourcemap(self) -> SourceMap {
|
||||
SourceMap::new(
|
||||
None,
|
||||
self.names,
|
||||
self.sources,
|
||||
Some(self.source_contents),
|
||||
self.tokens,
|
||||
None,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sourcemap_builder() {
|
||||
let mut builder = SourceMapBuilder::default();
|
||||
builder.set_source_and_content("baz.js", "");
|
||||
builder.add_name("x");
|
||||
|
||||
let sm = builder.into_sourcemap();
|
||||
assert_eq!(sm.get_source(0), Some("baz.js"));
|
||||
assert_eq!(sm.get_name(0), Some("x"));
|
||||
|
||||
let expected =
|
||||
r#"{"version":3,"names":["x"],"sources":["baz.js"],"sourcesContent":[""],"mappings":""}"#;
|
||||
assert_eq!(expected, sm.to_json_string());
|
||||
}
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
use crate::SourceMap;
|
||||
use rustc_hash::FxHashMap;
|
||||
use sourcemap::SourceMap;
|
||||
|
||||
/// The `SourcemapVisualizer` is a helper for sourcemap testing.
|
||||
/// It print the mapping of original content and final content tokens.
|
||||
pub struct SourcemapVisualizer<'a> {
|
||||
output: &'a str,
|
||||
sourcemap: &'a SourceMap,
|
||||
|
|
@ -16,22 +18,26 @@ impl<'a> SourcemapVisualizer<'a> {
|
|||
let mut source_log_map = FxHashMap::default();
|
||||
let source_contents_lines_map: FxHashMap<String, Option<Vec<Vec<u16>>>> = self
|
||||
.sourcemap
|
||||
.sources()
|
||||
.sources
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(source_id, source)| {
|
||||
(
|
||||
source.to_string(),
|
||||
self.sourcemap
|
||||
.get_source_view(source_id as u32)
|
||||
.map(|source_view| Self::generate_line_utf16_tables(source_view.source())),
|
||||
.get_source_content(source_id as u32)
|
||||
.map(Self::generate_line_utf16_tables),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let output_lines = Self::generate_line_utf16_tables(self.output);
|
||||
|
||||
let mut s = String::new();
|
||||
|
||||
self.sourcemap.tokens().reduce(|pre_token, token| {
|
||||
if let Some(source) = pre_token.get_source() {
|
||||
self.sourcemap.tokens.iter().reduce(|pre_token, token| {
|
||||
if let Some(source) =
|
||||
pre_token.get_source_id().and_then(|id| self.sourcemap.get_source(id))
|
||||
{
|
||||
if let Some(Some(source_contents_lines)) = source_contents_lines_map.get(source) {
|
||||
// Print source
|
||||
source_log_map.entry(source).or_insert_with(|| {
|
||||
|
|
@ -129,13 +135,13 @@ mod test {
|
|||
|
||||
#[test]
|
||||
fn should_work() {
|
||||
let sourcemap = SourceMap::from_slice(r#"{
|
||||
let sourcemap = SourceMap::from_json_string(r#"{
|
||||
"version":3,
|
||||
"sources":["shared.js","index.js"],
|
||||
"sourcesContent":["const a = 'shared.js'\n\nexport { a }","import { a as a2 } from './shared'\nconst a = 'index.js'\nconsole.log(a, a2)\n"],
|
||||
"names":["a","a$1"],
|
||||
"mappings":";;AAAA,MAAMA,IAAI;;;ACCV,MAAMC,MAAI;AACV,QAAQ,IAAIA,KAAGD,EAAG"
|
||||
}"#.as_bytes()).unwrap();
|
||||
}"#).unwrap();
|
||||
let output = "\n// shared.js\nconst a = 'shared.js';\n\n// index.js\nconst a$1 = 'index.js';\nconsole.log(a$1, a);\n";
|
||||
let visualizer = SourcemapVisualizer::new(output, &sourcemap);
|
||||
let visualizer_text = visualizer.into_visualizer_text();
|
||||
145
crates/oxc_sourcemap/src/token.rs
Normal file
145
crates/oxc_sourcemap/src/token.rs
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
use crate::SourceMap;
|
||||
|
||||
/// The `Token` is used to generate vlq `mappings`.
|
||||
#[derive(Debug, PartialEq, Eq, Clone)]
|
||||
pub struct Token {
|
||||
pub(crate) dst_line: u32,
|
||||
pub(crate) dst_col: u32,
|
||||
pub(crate) src_line: u32,
|
||||
pub(crate) src_col: u32,
|
||||
pub(crate) source_id: Option<u32>,
|
||||
pub(crate) name_id: Option<u32>,
|
||||
}
|
||||
|
||||
impl Token {
|
||||
pub fn new(
|
||||
dst_line: u32,
|
||||
dst_col: u32,
|
||||
src_line: u32,
|
||||
src_col: u32,
|
||||
source_id: Option<u32>,
|
||||
name_id: Option<u32>,
|
||||
) -> Self {
|
||||
Self { dst_line, dst_col, src_line, src_col, source_id, name_id }
|
||||
}
|
||||
|
||||
pub fn get_dst_line(&self) -> u32 {
|
||||
self.dst_line
|
||||
}
|
||||
|
||||
pub fn get_dst_col(&self) -> u32 {
|
||||
self.dst_col
|
||||
}
|
||||
|
||||
pub fn get_src_line(&self) -> u32 {
|
||||
self.src_line
|
||||
}
|
||||
|
||||
pub fn get_src_col(&self) -> u32 {
|
||||
self.src_col
|
||||
}
|
||||
|
||||
pub fn get_name_id(&self) -> Option<u32> {
|
||||
self.name_id
|
||||
}
|
||||
|
||||
pub fn get_source_id(&self) -> Option<u32> {
|
||||
self.source_id
|
||||
}
|
||||
}
|
||||
|
||||
/// The `TokenChunk` used by encode tokens to vlq mappings at parallel.
|
||||
/// It is a slice of `SourceMap::tokens`, it is a unit of parallel.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct TokenChunk {
|
||||
pub start: u32,
|
||||
pub end: u32,
|
||||
pub prev_dst_line: u32,
|
||||
pub prev_dst_col: u32,
|
||||
pub prev_src_line: u32,
|
||||
pub prev_src_col: u32,
|
||||
pub prev_name_id: u32,
|
||||
pub prev_source_id: u32,
|
||||
}
|
||||
|
||||
impl TokenChunk {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
start: u32,
|
||||
end: u32,
|
||||
prev_dst_line: u32,
|
||||
prev_dst_col: u32,
|
||||
prev_src_line: u32,
|
||||
prev_src_col: u32,
|
||||
prev_name_id: u32,
|
||||
prev_source_id: u32,
|
||||
) -> Self {
|
||||
Self {
|
||||
start,
|
||||
end,
|
||||
prev_dst_line,
|
||||
prev_dst_col,
|
||||
prev_src_line,
|
||||
prev_src_col,
|
||||
prev_name_id,
|
||||
prev_source_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The `SourceViewToken` provider extra `source` and `source_content` value.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct SourceViewToken<'a> {
|
||||
pub(crate) token: &'a Token,
|
||||
pub(crate) sourcemap: &'a SourceMap,
|
||||
}
|
||||
|
||||
impl<'a> SourceViewToken<'a> {
|
||||
pub fn new(token: &'a Token, sourcemap: &'a SourceMap) -> Self {
|
||||
Self { token, sourcemap }
|
||||
}
|
||||
|
||||
pub fn get_dst_line(&self) -> u32 {
|
||||
self.token.dst_line
|
||||
}
|
||||
|
||||
pub fn get_dst_col(&self) -> u32 {
|
||||
self.token.dst_col
|
||||
}
|
||||
|
||||
pub fn get_src_line(&self) -> u32 {
|
||||
self.token.src_line
|
||||
}
|
||||
|
||||
pub fn get_src_col(&self) -> u32 {
|
||||
self.token.src_col
|
||||
}
|
||||
|
||||
pub fn get_name_id(&self) -> Option<u32> {
|
||||
self.token.name_id
|
||||
}
|
||||
|
||||
pub fn get_source_id(&self) -> Option<u32> {
|
||||
self.token.source_id
|
||||
}
|
||||
|
||||
pub fn get_name(&self) -> Option<&str> {
|
||||
self.token.name_id.and_then(|id| self.sourcemap.get_name(id))
|
||||
}
|
||||
|
||||
pub fn get_source(&self) -> Option<&str> {
|
||||
self.token.source_id.and_then(|id| self.sourcemap.get_source(id))
|
||||
}
|
||||
|
||||
pub fn get_source_content(&self) -> Option<&str> {
|
||||
self.token.source_id.and_then(|id| self.sourcemap.get_source_content(id))
|
||||
}
|
||||
|
||||
pub fn get_source_and_content(&self) -> Option<(&str, &str)> {
|
||||
self.token.source_id.and_then(|id| self.sourcemap.get_source_and_content(id))
|
||||
}
|
||||
|
||||
pub fn to_tuple(&self) -> (Option<&str>, u32, u32, Option<&str>) {
|
||||
(self.get_source(), self.get_src_line(), self.get_src_col(), self.get_name())
|
||||
}
|
||||
}
|
||||
|
|
@ -42,6 +42,10 @@ harness = false
|
|||
name = "codegen_sourcemap"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "sourcemap"
|
||||
harness = false
|
||||
|
||||
# Broken
|
||||
# [[bench]]
|
||||
# name = "prettier"
|
||||
|
|
@ -71,6 +75,7 @@ oxc_span = { workspace = true, optional = true }
|
|||
oxc_tasks_common = { workspace = true, optional = true }
|
||||
oxc_transformer = { workspace = true, optional = true }
|
||||
oxc_codegen = { workspace = true, optional = true }
|
||||
oxc_sourcemap = { workspace = true, optional = true }
|
||||
|
||||
criterion = { package = "criterion2", version = "0.6.0", default-features = false }
|
||||
serde = { workspace = true, optional = true }
|
||||
|
|
@ -88,6 +93,7 @@ default = [
|
|||
"dep:oxc_tasks_common",
|
||||
"dep:oxc_transformer",
|
||||
"dep:oxc_codegen",
|
||||
"dep:oxc_sourcemap",
|
||||
]
|
||||
codspeed = ["criterion/codspeed"]
|
||||
codspeed_napi = ["criterion/codspeed", "dep:serde", "dep:serde_json"]
|
||||
|
|
|
|||
44
tasks/benchmark/benches/sourcemap.rs
Normal file
44
tasks/benchmark/benches/sourcemap.rs
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
use oxc_allocator::Allocator;
|
||||
use oxc_benchmark::{criterion_group, criterion_main, BenchmarkId, Criterion};
|
||||
use oxc_codegen::{Codegen, CodegenOptions, CodegenReturn};
|
||||
use oxc_parser::Parser;
|
||||
use oxc_sourcemap::ConcatSourceMapBuilder;
|
||||
use oxc_span::SourceType;
|
||||
use oxc_tasks_common::TestFiles;
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
fn bench_sourcemap(criterion: &mut Criterion) {
|
||||
let mut group = criterion.benchmark_group("sourcemap");
|
||||
|
||||
for file in TestFiles::minimal().files() {
|
||||
let id = BenchmarkId::from_parameter(&file.file_name);
|
||||
let source_type = SourceType::from_path(&file.file_name).unwrap();
|
||||
group.bench_with_input(id, &file.source_text, |b, source_text| {
|
||||
let allocator = Allocator::default();
|
||||
let program = Parser::new(&allocator, source_text, source_type).parse().program;
|
||||
let codegen_options =
|
||||
CodegenOptions { enable_source_map: true, ..CodegenOptions::default() };
|
||||
b.iter_with_large_drop(|| {
|
||||
let CodegenReturn { source_map, source_text } = Codegen::<false>::new(
|
||||
file.file_name.as_str(),
|
||||
source_text,
|
||||
codegen_options.clone(),
|
||||
)
|
||||
.build(&program);
|
||||
let line = source_text.matches('\n').count() as u32;
|
||||
if let Some(sourcemap) = source_map {
|
||||
let mut concat_sourcemap_builder = ConcatSourceMapBuilder::default();
|
||||
for i in 0..1 {
|
||||
concat_sourcemap_builder.add_sourcemap(&sourcemap, line * i);
|
||||
}
|
||||
concat_sourcemap_builder.into_sourcemap().to_json_string();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(sourcemap, bench_sourcemap);
|
||||
criterion_main!(sourcemap);
|
||||
|
|
@ -31,6 +31,7 @@ oxc_minifier = { workspace = true }
|
|||
oxc_prettier = { workspace = true }
|
||||
oxc_span = { workspace = true }
|
||||
oxc_tasks_common = { workspace = true }
|
||||
oxc_sourcemap = { workspace = true }
|
||||
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -5,8 +5,9 @@ use std::{
|
|||
};
|
||||
|
||||
use oxc_allocator::Allocator;
|
||||
use oxc_codegen::{Codegen, CodegenOptions, SourcemapVisualizer};
|
||||
use oxc_codegen::{Codegen, CodegenOptions};
|
||||
use oxc_parser::Parser;
|
||||
use oxc_sourcemap::SourcemapVisualizer;
|
||||
use oxc_span::SourceType;
|
||||
use oxc_tasks_common::{project_root, TestFiles};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue