From a2cfc867cb48e775966de952e6fa2ed975be73da Mon Sep 17 00:00:00 2001 From: underfin Date: Thu, 21 Mar 2024 11:19:09 +0800 Subject: [PATCH] feat: SourcemapVisualizer (#2773) Export `SourcemapVisualizer` from codegen, it will be used oxc and rolldown sourcemap test, so it support multiply source print, it will using sourcemap `sourcesContent` as original source. --- Cargo.lock | 1 + crates/oxc_codegen/Cargo.toml | 1 + crates/oxc_codegen/src/lib.rs | 3 +- .../oxc_codegen/src/sourcemap_visualizer.rs | 159 ++++++++++++++++++ tasks/coverage/src/sourcemap.rs | 104 +----------- 5 files changed, 170 insertions(+), 98 deletions(-) create mode 100644 crates/oxc_codegen/src/sourcemap_visualizer.rs diff --git a/Cargo.lock b/Cargo.lock index e3df7fce4..78ec04241 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1392,6 +1392,7 @@ dependencies = [ "oxc_parser", "oxc_span", "oxc_syntax", + "rustc-hash", "sourcemap", ] diff --git a/crates/oxc_codegen/Cargo.toml b/crates/oxc_codegen/Cargo.toml index 99823b07b..cc5f10e72 100644 --- a/crates/oxc_codegen/Cargo.toml +++ b/crates/oxc_codegen/Cargo.toml @@ -25,6 +25,7 @@ oxc_allocator = { workspace = true } oxc_syntax = { workspace = true } sourcemap = { version = "7.1.1" } bitflags = { workspace = true } +rustc-hash = { workspace = true } [dev-dependencies] oxc_parser = { workspace = true } diff --git a/crates/oxc_codegen/src/lib.rs b/crates/oxc_codegen/src/lib.rs index 7d5572751..5a9322bcc 100644 --- a/crates/oxc_codegen/src/lib.rs +++ b/crates/oxc_codegen/src/lib.rs @@ -13,7 +13,7 @@ mod gen; mod gen_ts; mod operator; mod sourcemap_builder; - +mod sourcemap_visualizer; use std::str::from_utf8_unchecked; #[allow(clippy::wildcard_imports)] @@ -31,6 +31,7 @@ pub use crate::{ context::Context, gen::{Gen, GenExpr}, operator::Operator, + sourcemap_visualizer::SourcemapVisualizer, }; // use crate::mangler::Mangler; diff --git a/crates/oxc_codegen/src/sourcemap_visualizer.rs b/crates/oxc_codegen/src/sourcemap_visualizer.rs new file mode 100644 index 000000000..cd0109e08 --- /dev/null +++ b/crates/oxc_codegen/src/sourcemap_visualizer.rs @@ -0,0 +1,159 @@ +use rustc_hash::FxHashMap; +use sourcemap::SourceMap; + +pub struct SourcemapVisualizer<'a> { + output: &'a str, + sourcemap: &'a SourceMap, +} + +impl<'a> SourcemapVisualizer<'a> { + pub fn new(output: &'a str, sourcemap: &'a SourceMap) -> Self { + Self { output, sourcemap } + } + + #[allow(clippy::cast_possible_truncation)] + pub fn into_visualizer_text(self) -> String { + let mut source_log_map = FxHashMap::default(); + let source_contents_lines_map: FxHashMap>>> = self + .sourcemap + .sources() + .enumerate() + .map(|(source_id, source)| { + ( + source.to_string(), + self.sourcemap + .get_source_view(source_id as u32) + .map(|source_view| Self::generate_line_utf16_tables(source_view.source())), + ) + }) + .collect(); + let output_lines = Self::generate_line_utf16_tables(self.output); + let mut s = String::new(); + + self.sourcemap.tokens().reduce(|pre_token, token| { + if let Some(source) = pre_token.get_source() { + if let Some(Some(source_contents_lines)) = source_contents_lines_map.get(source) { + // Print source + source_log_map.entry(source).or_insert_with(|| { + s.push('-'); + s.push(' '); + s.push_str(source); + s.push('\n'); + true + }); + + // Print token + s.push_str(&format!( + "({}:{}-{}:{}) {:?}", + pre_token.get_src_line(), + pre_token.get_src_col(), + token.get_src_line(), + token.get_src_col(), + Self::str_slice_by_token( + source_contents_lines, + (pre_token.get_src_line(), pre_token.get_src_col()), + (token.get_src_line(), token.get_src_col()) + ) + )); + + s.push_str(" --> "); + + s.push_str(&format!( + "({}:{}-{}:{}) {:?}", + pre_token.get_dst_line(), + pre_token.get_dst_col(), + token.get_dst_line(), + token.get_dst_col(), + Self::str_slice_by_token( + &output_lines, + (pre_token.get_dst_line(), pre_token.get_dst_col(),), + (token.get_dst_line(), token.get_dst_col(),) + ) + )); + s.push('\n'); + } + } + + token + }); + + s + } + + fn generate_line_utf16_tables(content: &str) -> Vec> { + let mut tables = vec![]; + let mut line_byte_offset = 0; + for (i, ch) in content.char_indices() { + match ch { + '\r' | '\n' | '\u{2028}' | '\u{2029}' => { + // Handle Windows-specific "\r\n" newlines + if ch == '\r' && content.chars().nth(i + 1) == Some('\n') { + continue; + } + tables.push(content[line_byte_offset..i].encode_utf16().collect::>()); + line_byte_offset = i; + } + _ => {} + } + } + tables.push(content[line_byte_offset..].encode_utf16().collect::>()); + tables + } + + fn str_slice_by_token(buff: &[Vec], start: (u32, u32), end: (u32, u32)) -> String { + if start.0 == end.0 { + return String::from_utf16(&buff[start.0 as usize][start.1 as usize..end.1 as usize]) + .unwrap(); + } + + let mut s = String::new(); + + for i in start.0..end.0 { + let slice = &buff[i as usize]; + if i == start.0 { + s.push_str(&String::from_utf16(&slice[start.1 as usize..]).unwrap()); + } else if i == end.0 { + s.push_str(&String::from_utf16(&slice[..end.1 as usize]).unwrap()); + } else { + s.push_str(&String::from_utf16(slice).unwrap()); + } + } + + s + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn should_work() { + let sourcemap = SourceMap::from_slice(r#"{ + "version":3, + "sources":["shared.js","index.js"], + "sourcesContent":["const a = 'shared.js'\n\nexport { a }","import { a as a2 } from './shared'\nconst a = 'index.js'\nconsole.log(a, a2)\n"], + "names":["a","a$1"], + "mappings":";;AAAA,MAAMA,IAAI;;;ACCV,MAAMC,MAAI;AACV,QAAQ,IAAIA,KAAGD,EAAG" + }"#.as_bytes()).unwrap(); + let output = "\n// shared.js\nconst a = 'shared.js';\n\n// index.js\nconst a$1 = 'index.js';\nconsole.log(a$1, a);\n"; + let visualizer = SourcemapVisualizer::new(output, &sourcemap); + let visualizer_text = visualizer.into_visualizer_text(); + assert_eq!( + visualizer_text, + r#"- shared.js +(0:0-0:6) "const " --> (2:0-2:6) "\nconst" +(0:6-0:10) "a = " --> (2:6-2:10) " a =" +(0:10-1:0) "'shared.js'" --> (2:10-5:0) " 'shared.js';\n\n// index.js" +- index.js +(1:0-1:6) "\nconst" --> (5:0-5:6) "\nconst" +(1:6-1:10) " a =" --> (5:6-5:12) " a$1 =" +(1:10-2:0) " 'index.js'" --> (5:12-6:0) " 'index.js';" +(2:0-2:8) "\nconsole" --> (6:0-6:8) "\nconsole" +(2:8-2:12) ".log" --> (6:8-6:12) ".log" +(2:12-2:15) "(a," --> (6:12-6:17) "(a$1," +(2:15-2:18) " a2" --> (6:17-6:19) " a" +"# + ); + } +} diff --git a/tasks/coverage/src/sourcemap.rs b/tasks/coverage/src/sourcemap.rs index 6a46fac61..ff2f10c16 100644 --- a/tasks/coverage/src/sourcemap.rs +++ b/tasks/coverage/src/sourcemap.rs @@ -5,7 +5,7 @@ use std::{ }; use oxc_allocator::Allocator; -use oxc_codegen::{Codegen, CodegenOptions}; +use oxc_codegen::{Codegen, CodegenOptions, SourcemapVisualizer}; use oxc_parser::Parser; use oxc_span::SourceType; use oxc_tasks_common::{project_root, TestFiles}; @@ -67,13 +67,12 @@ impl Suite for SourcemapSuite { let result = case.test_result(); let path = case.path().to_string_lossy(); let result = match result { - TestResult::Snapshot(snapshot) => snapshot, - TestResult::ParseError(error, _) => error, + TestResult::Snapshot(snapshot) => snapshot.to_string(), + TestResult::ParseError(error, _) => format!("- {path}\n{error}"), _ => { unreachable!() } }; - writeln!(file, "- {path}").unwrap(); writeln!(file, "{result}\n\n").unwrap(); } } @@ -90,82 +89,6 @@ impl SourcemapCase { pub fn source_type(&self) -> SourceType { self.source_type } - - fn generate_line_utf16_tables(content: &str) -> Vec> { - let mut tables = vec![]; - let mut line_byte_offset = 0; - for (i, ch) in content.char_indices() { - match ch { - '\r' | '\n' | '\u{2028}' | '\u{2029}' => { - // Handle Windows-specific "\r\n" newlines - if ch == '\r' && content.chars().nth(i + 1) == Some('\n') { - continue; - } - tables.push(content[line_byte_offset..i].encode_utf16().collect::>()); - line_byte_offset = i; - } - _ => {} - } - } - tables.push(content[line_byte_offset..].encode_utf16().collect::>()); - tables - } - - fn create_visualizer_text( - source: &str, - output: &str, - tokens: &[(u32, u32, u32, u32)], - ) -> String { - let source_lines = Self::generate_line_utf16_tables(source); - let output_lines = Self::generate_line_utf16_tables(output); - let mut s = String::new(); - - tokens.iter().reduce(|pre, cur| { - s.push_str(&format!( - "({}:{}-{}:{}) {:?}", - pre.0, - pre.1, - cur.0, - cur.1, - Self::str_slice_by_token(&source_lines, (pre.0, pre.1), (cur.0, cur.1)) - )); - s.push_str(" --> "); - s.push_str(&format!( - "({}:{}-{}:{}) {:?}", - pre.2, - pre.3, - cur.2, - cur.3, - Self::str_slice_by_token(&output_lines, (pre.2, pre.3), (cur.2, cur.3)) - )); - s.push('\n'); - cur - }); - - s - } - - fn str_slice_by_token(buff: &[Vec], start: (u32, u32), end: (u32, u32)) -> String { - if start.0 == end.0 { - return String::from_utf16(&buff[start.0 as usize][start.1 as usize..end.1 as usize]) - .unwrap(); - } - - let mut s = String::new(); - - for i in start.0..end.0 { - let slice = &buff[i as usize]; - if i == start.0 { - s.push_str(&String::from_utf16(&slice[start.1 as usize..]).unwrap()); - } else if i == end.0 { - s.push_str(&String::from_utf16(&slice[..end.1 as usize]).unwrap()); - } else { - s.push_str(&String::from_utf16(slice).unwrap()); - } - } - - s - } } impl Case for SourcemapCase { @@ -208,23 +131,10 @@ impl Case for SourcemapCase { ..CodegenOptions::default() }; let codegen_ret = Codegen::::new(source_text, codegen_options).build(&ret.program); - let content = codegen_ret.source_text; - let map = codegen_ret.source_map.unwrap(); - let tokens = map - .tokens() - .map(|token| { - ( - token.get_src_line(), - token.get_src_col(), - token.get_dst_line(), - token.get_dst_col(), - ) - }) - .collect::>(); - let mut result = String::new(); - result.push_str(Self::create_visualizer_text(source_text, &content, &tokens).as_str()); - - TestResult::Snapshot(result) + TestResult::Snapshot( + SourcemapVisualizer::new(&codegen_ret.source_text, &codegen_ret.source_map.unwrap()) + .into_visualizer_text(), + ) } }