mirror of
https://github.com/danbulant/oxc
synced 2026-05-19 04:08:41 +00:00
parent
fabf116f4b
commit
4d2888d6fc
12 changed files with 40 additions and 29 deletions
|
|
@ -1,6 +1,7 @@
|
|||
use super::{Kind, Lexer};
|
||||
use crate::diagnostics;
|
||||
|
||||
use super::{Kind, Lexer};
|
||||
|
||||
/// Handle next byte of source.
|
||||
///
|
||||
/// # SAFETY
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
use memchr::memmem::Finder;
|
||||
|
||||
use oxc_syntax::identifier::is_line_terminator;
|
||||
|
||||
use crate::diagnostics;
|
||||
|
||||
use super::{
|
||||
cold_branch,
|
||||
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
|
||||
source::SourcePosition,
|
||||
Kind, Lexer,
|
||||
};
|
||||
use crate::diagnostics;
|
||||
|
||||
// Irregular line breaks - '\u{2028}' (LS) and '\u{2029}' (PS)
|
||||
const LS_OR_PS_FIRST: u8 = 0xE2;
|
||||
|
|
|
|||
|
|
@ -6,12 +6,13 @@ use oxc_syntax::identifier::{
|
|||
is_identifier_part, is_identifier_part_unicode, is_identifier_start_unicode,
|
||||
};
|
||||
|
||||
use crate::diagnostics;
|
||||
|
||||
use super::{
|
||||
cold_branch,
|
||||
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
|
||||
Kind, Lexer, SourcePosition,
|
||||
};
|
||||
use crate::diagnostics;
|
||||
|
||||
const MIN_ESCAPED_STR_LEN: usize = 16;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
use memchr::{memchr, memchr2};
|
||||
|
||||
use oxc_syntax::identifier::is_identifier_part;
|
||||
|
||||
use crate::diagnostics;
|
||||
|
||||
use super::{
|
||||
cold_branch,
|
||||
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
|
||||
Kind, Lexer, Token,
|
||||
};
|
||||
use crate::diagnostics;
|
||||
|
||||
static NOT_ASCII_JSX_ID_CONTINUE_TABLE: SafeByteMatchTable =
|
||||
safe_byte_match_table!(|b| !(b.is_ascii_alphanumeric() || matches!(b, b'_' | b'$' | b'-')));
|
||||
|
|
|
|||
|
|
@ -197,7 +197,7 @@ pub enum Kind {
|
|||
}
|
||||
|
||||
#[allow(clippy::enum_glob_use)]
|
||||
use self::Kind::*;
|
||||
use Kind::*;
|
||||
|
||||
impl Kind {
|
||||
#[inline]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,17 @@
|
|||
//! * [rustc](https://github.com/rust-lang/rust/blob/1.82.0/compiler/rustc_lexer/src)
|
||||
//! * [v8](https://v8.dev/blog/scanner)
|
||||
|
||||
use std::collections::VecDeque;
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
use oxc_allocator::Allocator;
|
||||
use oxc_ast::ast::RegExpFlags;
|
||||
use oxc_diagnostics::OxcDiagnostic;
|
||||
use oxc_span::{SourceType, Span};
|
||||
|
||||
use crate::{diagnostics, UniquePromise};
|
||||
|
||||
mod byte_handlers;
|
||||
mod comment;
|
||||
mod identifier;
|
||||
|
|
@ -24,25 +35,13 @@ mod typescript;
|
|||
mod unicode;
|
||||
mod whitespace;
|
||||
|
||||
use std::collections::VecDeque;
|
||||
pub use kind::Kind;
|
||||
pub use number::{parse_big_int, parse_float, parse_int};
|
||||
pub use token::Token;
|
||||
|
||||
use oxc_allocator::Allocator;
|
||||
use oxc_ast::ast::RegExpFlags;
|
||||
use oxc_diagnostics::OxcDiagnostic;
|
||||
use oxc_span::{SourceType, Span};
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
use self::{
|
||||
byte_handlers::handle_byte,
|
||||
source::{Source, SourcePosition},
|
||||
trivia_builder::TriviaBuilder,
|
||||
};
|
||||
pub use self::{
|
||||
kind::Kind,
|
||||
number::{parse_big_int, parse_float, parse_int},
|
||||
token::Token,
|
||||
};
|
||||
use crate::{diagnostics, UniquePromise};
|
||||
use byte_handlers::handle_byte;
|
||||
use source::{Source, SourcePosition};
|
||||
use trivia_builder::TriviaBuilder;
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct LexerCheckpoint<'a> {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
use oxc_syntax::identifier::{is_identifier_part_ascii, is_identifier_start};
|
||||
|
||||
use super::{Kind, Lexer, Span};
|
||||
use crate::diagnostics;
|
||||
|
||||
use super::{Kind, Lexer, Span};
|
||||
|
||||
impl Lexer<'_> {
|
||||
/// 12.9.3 Numeric Literals with `0` prefix
|
||||
pub(super) fn read_zero(&mut self) -> Kind {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
use oxc_diagnostics::Result;
|
||||
use oxc_syntax::identifier::is_line_terminator;
|
||||
|
||||
use super::{Kind, Lexer, RegExpFlags, Token};
|
||||
use crate::diagnostics;
|
||||
|
||||
use super::{Kind, Lexer, RegExpFlags, Token};
|
||||
|
||||
impl Lexer<'_> {
|
||||
/// Re-tokenize the current `/` or `/=` and return `RegExp`
|
||||
/// See Section 12:
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
|
||||
use std::{marker::PhantomData, slice, str};
|
||||
|
||||
use super::search::SEARCH_BATCH_SIZE;
|
||||
use crate::{UniquePromise, MAX_LEN};
|
||||
|
||||
use super::search::SEARCH_BATCH_SIZE;
|
||||
|
||||
/// `Source` holds the source text for the lexer, and provides APIs to read it.
|
||||
///
|
||||
/// It provides a cursor which allows consuming source text either as `char`s, or as bytes.
|
||||
|
|
|
|||
|
|
@ -2,12 +2,13 @@ use std::cmp::max;
|
|||
|
||||
use oxc_allocator::String;
|
||||
|
||||
use crate::diagnostics;
|
||||
|
||||
use super::{
|
||||
cold_branch,
|
||||
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
|
||||
Kind, Lexer, LexerContext, Span, Token,
|
||||
};
|
||||
use crate::diagnostics;
|
||||
|
||||
const MIN_ESCAPED_STR_LEN: usize = 16;
|
||||
|
||||
|
|
|
|||
|
|
@ -2,12 +2,13 @@ use std::cmp::max;
|
|||
|
||||
use oxc_allocator::String;
|
||||
|
||||
use crate::diagnostics;
|
||||
|
||||
use super::{
|
||||
cold_branch,
|
||||
search::{byte_search, safe_byte_match_table, SafeByteMatchTable},
|
||||
Kind, Lexer, SourcePosition, Token,
|
||||
};
|
||||
use crate::diagnostics;
|
||||
|
||||
const MIN_ESCAPED_TEMPLATE_LIT_LEN: usize = 16;
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,10 @@ use oxc_syntax::identifier::{
|
|||
is_irregular_line_terminator, is_irregular_whitespace, CR, FF, LF, LS, PS, TAB, VT,
|
||||
};
|
||||
|
||||
use super::{Kind, Lexer, Span};
|
||||
use crate::diagnostics;
|
||||
|
||||
use super::{Kind, Lexer, Span};
|
||||
|
||||
enum SurrogatePair {
|
||||
// valid \u Hex4Digits \u Hex4Digits
|
||||
Astral(u32),
|
||||
|
|
|
|||
Loading…
Reference in a new issue