dan/oxc
1
0
Fork 0
mirror of https://github.com/danbulant/oxc synced 2026-05-19 12:19:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(lexer): correct the span for irregular whitespaces (#2245)

Browse source 
closes #2236
This commit is contained in:
Boshen 2024-02-01 14:18:47 +08:00 committed by GitHub
parent 589fd0cdd1
commit 2beacd3f4d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 324 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
crates/oxc_linter/src/rules/eslint/no_irregular_whitespace.rs
View file

@ -42,6 +42,7 @@ impl Rule for NoIrregularWhitespace {
}
}
#[allow(clippy::unicode_not_nfc, clippy::invisible_characters)]
#[test]
fn test() {
use crate::tester::Tester;
@ -76,8 +77,8 @@ fn test() {
(r"' ';", None),
(r"'᠎';", None),
(r"'';", None),
// ("' ';", None), lint error
// (r"' ';", None), lint error
("' ';", None),
(r"' ';", None),
(r"' ';", None),
(r"' ';", None),
(r"' ';", None),
@ -87,7 +88,7 @@ fn test() {
(r"' ';", None),
(r"' ';", None),
(r"' ';", None),
// (r"'​';", None), lint error
(r"'​';", None),
(r"'\
';", None),
(r"'\
';", None),
(r"' ';", None),
@ -227,12 +228,12 @@ fn test() {
];
let fail = vec![
// (r"var any = 'thing';", None),
// (r"var any = 'thing';", None),
(r"var any = 'thing';", None),
(r"var any = 'thing';", None),
(r"var any   = 'thing';", None),
(r"var any  = 'thing';", None),
// (r"var any   = 'thing';", None),
// (r"var any   = 'thing';", None),
(r"var any   = 'thing';", None),
(r"var any   = 'thing';", None),
(r"var any   = 'thing';", None),
(r"var any   = 'thing';", None),
(r"var any   = 'thing';", None),
@ -242,16 +243,16 @@ fn test() {
(r"var any   = 'thing';", None),
(r"var any   = 'thing';", None),
(r"var any   = 'thing';", None),
// (r"var any 
 = 'thing';", None),
// (r"var any 
 = 'thing';", None),
(r"var any 
 = 'thing';", None),
(r"var any 
 = 'thing';", None),
(r"var any   = 'thing';", None),
(r"var any   = 'thing';", None),
(r"var any 　 = 'thing';", None),
// (
// r"var a = 'b',
c = 'd',
// e = 'f'
",
// None,
// ),
(
r"var a = 'b',
c = 'd',
e = 'f'
",
None,
),
(
r"var any 　 = 'thing', other 　 = 'thing';
var third 　 = 'thing';",
@ -346,42 +347,42 @@ fn test() {
　",
Some(serde_json::json!([{ "skipTemplates": true }])),
),
// (r"var foo = bar;", None),
// (r"var foo = bar;", None),
// (r"var foo = bar;", None),
// (r"var foo = bar;", None),
// (r"var foo = bar;", None),
// (r"var foo = bar ;", None),
// (r" ", None),
(r"var foo = bar;", None),
(r"var foo = bar;", None),
(r"var foo = bar;", None),
(r"var foo = bar;", None),
(r"var foo = bar;", None),
(r"var foo = bar ;", None),
(r" ", None),
("   ", None),
// (
// r"var foo =
// bar;",
// None,
// ),
// (
// r"var foo =
// bar;",
// None,
// ),
// (
// r"var foo =
// bar
// ;
// ",
// None,
// ),
// (r"var foo = 
bar;", None),
// (r"var foo =
 bar;", None),
// (r"var foo = bar;
", None),
// (r"
", None),
// (r"foo

", None),
// (r"foo

", None),
// (
// r"foo
// 
",
// None,
// r"var foo =
// bar;",
// None,
// ),
(
r"var foo =
bar;",
None,
),
(
r"var foo =
bar
;
",
None,
),
(r"var foo = 
bar;", None),
(r"var foo =
 bar;", None),
(r"var foo = bar;
", None),
(r"
", None),
(r"foo

", None),
(r"foo

", None),
(
r"foo

",
None,
),
// (r"foo 
 ", None),
// (r"<div> </div>;", None),
// (r"<div> </div>;", None),

294
crates/oxc_linter/src/snapshots/no_irregular_whitespace.snap
View file

@ -3,108 +3,166 @@ source: crates/oxc_linter/src/tester.rs
expression: no_irregular_whitespace
---
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any = 'thing';
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any = 'thing';
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any  = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any 
 = 'thing';
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any 
 = 'thing';
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any   = 'thing';
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any 　 = 'thing';
· ▲
· ──
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:13]
1 │ var a = 'b',
c = 'd',
· ─
2 │ e = 'f'

╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:2:18]
1 │ var a = 'b',
c = 'd',
2 │ e = 'f'

· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:9]
1 │ var any 　 = 'thing', other 　 = 'thing';
· ▲
· ──
2 │ var third 　 = 'thing';
╰────
help: Try to remove the irregular whitespace
@ -112,7 +170,7 @@ expression: no_irregular_whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:29]
1 │ var any 　 = 'thing', other 　 = 'thing';
· ▲
· ──
2 │ var third 　 = 'thing';
╰────
help: Try to remove the irregular whitespace
@ -121,28 +179,28 @@ expression: no_irregular_whitespace
╭─[no_irregular_whitespace.tsx:2:23]
1 │ var any 　 = 'thing', other 　 = 'thing';
2 │ var third 　 = 'thing';
· ▲
· ──
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:14]
1 │ `something ${　 10} another thing`
· ▲
· ──
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:16]
1 │ `something ${10　} another thing`
· ▲
· ──
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:1]
1 │ 　
· ▲
· ──
2 │ `　template`
╰────
help: Try to remove the irregular whitespace
@ -150,7 +208,7 @@ expression: no_irregular_whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:1]
1 │ 　
· ▲
· ──
2 │ `　multiline
╰────
help: Try to remove the irregular whitespace
@ -158,14 +216,14 @@ expression: no_irregular_whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:1]
1 │ 　`　template`
· ▲
· ──
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:1]
1 │ 　`　multiline
· ▲
· ──
2 │ template`
╰────
help: Try to remove the irregular whitespace
@ -173,7 +231,7 @@ expression: no_irregular_whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:13]
1 │ `　template`　
· ▲
· ──
╰────
help: Try to remove the irregular whitespace
@ -181,7 +239,7 @@ expression: no_irregular_whitespace
╭─[no_irregular_whitespace.tsx:2:22]
1 │ `　multiline
2 │ template`　
· ▲
· ──
╰────
help: Try to remove the irregular whitespace
@ -189,7 +247,7 @@ expression: no_irregular_whitespace
╭─[no_irregular_whitespace.tsx:2:13]
1 │ `　template`
2 │ 　
· ▲
· ──
╰────
help: Try to remove the irregular whitespace
@ -197,28 +255,204 @@ expression: no_irregular_whitespace
╭─[no_irregular_whitespace.tsx:3:13]
2 │ template`
3 │ 　
· ▲
· ──
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:11]
1 │ var foo = bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:10]
1 │ var foo = bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:11]
1 │ var foo = bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:11]
1 │ var foo = bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:11]
1 │ var foo = bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:11]
1 │ var foo = bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:11]
1 │ var foo = bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:12]
1 │ var foo = bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:11]
1 │ var foo = bar ;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:14]
1 │ var foo = bar ;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:1]
1 │
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:1]
1 │    
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:2]
1 │    
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:3]
1 │    
· ▲
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:2:9]
1 │ var foo =
2 │ bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:2:9]
1 │ var foo =
2 │ bar
· ─
3 │ ;
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:2:9]
1 │ var foo =
2 │ bar
· ─
3 │ ;
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:2:9]
1 │ var foo =
2 │ bar
· ─
3 │ ;
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:11]
1 │ var foo = 
bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:10]
1 │ var foo =
 bar;
· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:15]
1 │ var foo = bar;

· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:1]
1 │ 

· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:4]
1 │ foo


· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:5]
1 │ foo


· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:4]
1 │ foo


· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:1:5]
1 │ foo


· ─
╰────
help: Try to remove the irregular whitespace
⚠ eslint(no-irregular-whitespace): Unexpected irregular whitespace
╭─[no_irregular_whitespace.tsx:2:9]
1 │ foo
2 │ 

· ─
╰────
help: Try to remove the irregular whitespace

11
crates/oxc_parser/src/lexer/byte_handlers.rs
View file

@ -20,7 +20,7 @@ type ByteHandler = unsafe fn(&mut Lexer<'_>) -> Kind;
#[rustfmt::skip]
static BYTE_HANDLERS: [ByteHandler; 256] = [
// 0 1 2 3 4 5 6 7 8 9 A B C D E F //
ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, SPS, LIN, SPS, SPS, LIN, ERR, ERR, // 0
ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, SPS, LIN, ISP, ISP, LIN, ERR, ERR, // 0
ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, // 1
SPS, EXL, QOT, HAS, IDT, PRC, AMP, QOT, PNO, PNC, ATR, PLS, COM, MIN, PRD, SLH, // 2
ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, GTR, QST, // 3
@ -98,12 +98,19 @@ ascii_byte_handler!(ERR(lexer) {
Kind::Undetermined
});
// <SPACE> <TAB> <VT> <FF>
// <SPACE> <TAB> Normal Whitespace
ascii_byte_handler!(SPS(lexer) {
lexer.consume_char();
Kind::Skip
});
// <VT> <FF> Irregular Whitespace
ascii_byte_handler!(ISP(lexer) {
lexer.consume_char();
lexer.trivia_builder.add_irregular_whitespace(lexer.current.token.start, lexer.offset());
Kind::Skip
});
// '\r' '\n'
ascii_byte_handler!(LIN(lexer) {
lexer.consume_char();

4
crates/oxc_parser/src/lexer/unicode.rs
View file

@ -28,14 +28,16 @@ impl<'a> Lexer<'a> {
Kind::Ident
}
c if is_irregular_whitespace(c) => {
self.consume_char();
self.trivia_builder
.add_irregular_whitespace(self.current.token.start, self.offset());
self.consume_char();
Kind::Skip
}
c if is_irregular_line_terminator(c) => {
self.consume_char();
self.current.token.is_on_new_line = true;
self.trivia_builder
.add_irregular_whitespace(self.current.token.start, self.offset());
Kind::Skip
}
_ => {