dan/oxc
1
0
Fork 0
mirror of https://github.com/danbulant/oxc synced 2026-05-25 04:42:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(parser): move source length check into lexer (#2206)

Browse source 
This change makes little difference in itself, but moving the check into
the lexer will allow some optimizations in lexer using unsafe code which
depend on this invariant.
This commit is contained in:
overlookmotel 2024-01-29 14:29:02 +00:00 committed by GitHub
parent e123be0a00
commit 71898ffdd5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 8 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
crates/oxc_parser/src/lexer/mod.rs
View file

@ -76,11 +76,12 @@ pub struct Lexer<'a> {
#[allow(clippy::unused_self)]
impl<'a> Lexer<'a> {
pub fn new(allocator: &'a Allocator, source: &'a str, source_type: SourceType) -> Self {
// Token's start and end are u32s, so limit for length of source is u32::MAX bytes.
// Only a debug assertion is required, as parser checks length of source before calling
// this method.
debug_assert!(source.len() <= MAX_LEN, "Source length exceeds MAX_LEN");
pub fn new(allocator: &'a Allocator, mut source: &'a str, source_type: SourceType) -> Self {
// If source exceeds size limit, substitute a short source which will fail to parse.
// `Parser::parse` will convert error to `diagnostics::OverlongSource`.
if source.len() > MAX_LEN {
source = "\0";
}
let token = Token {
// the first token is at the start of file, so is allows on a new line

8
crates/oxc_parser/src/lib.rs
View file

@ -155,12 +155,8 @@ pub struct Parser<'a> {
impl<'a> Parser<'a> {
/// Create a new parser
pub fn new(allocator: &'a Allocator, source_text: &'a str, source_type: SourceType) -> Self {
// If source exceeds size limit, substitute a short source which will fail to parse.
// `parse()` will convert error to `diagnostics::OverlongSource`.
let source_text_for_lexer = if source_text.len() > MAX_LEN { "\0" } else { source_text };
Self {
lexer: Lexer::new(allocator, source_text_for_lexer, source_type),
lexer: Lexer::new(allocator, source_text, source_type),
source_type,
source_text,
errors: vec![],
@ -254,7 +250,7 @@ impl<'a> Parser<'a> {
}
/// Check if source length exceeds MAX_LEN, if the file cannot be parsed.
/// Original parsing error is not real - `Parser::new` substituted "\0" as the source text.
/// Original parsing error is not real - `Lexer::new` substituted "\0" as the source text.
fn overlong_error(&self) -> Option<Error> {
if self.source_text.len() > MAX_LEN {
return Some(diagnostics::OverlongSource.into());