refactor(parser): move source length check into lexer (#2206)

This change makes little difference in itself, but moving the check into
the lexer will allow some optimizations in lexer using unsafe code which
depend on this invariant.

crates/oxc_parser/src/lexer/mod.rs

@@ -76,11 +76,12 @@ pub struct Lexer<'a> {
 
 #[allow(clippy::unused_self)]
 impl<'a> Lexer<'a> {
-    pub fn new(allocator: &'a Allocator, source: &'a str, source_type: SourceType) -> Self {
+    pub fn new(allocator: &'a Allocator, mut source: &'a str, source_type: SourceType) -> Self {
-        // Token's start and end are u32s, so limit for length of source is u32::MAX bytes.
+        // If source exceeds size limit, substitute a short source which will fail to parse.
-        // Only a debug assertion is required, as parser checks length of source before calling
+        // `Parser::parse` will convert error to `diagnostics::OverlongSource`.
-        // this method.
+        if source.len() > MAX_LEN {
-        debug_assert!(source.len() <= MAX_LEN, "Source length exceeds MAX_LEN");
+            source = "\0";
+        }
 
         let token = Token {
             // the first token is at the start of file, so is allows on a new line

crates/oxc_parser/src/lib.rs

@@ -155,12 +155,8 @@ pub struct Parser<'a> {
 impl<'a> Parser<'a> {
     /// Create a new parser
     pub fn new(allocator: &'a Allocator, source_text: &'a str, source_type: SourceType) -> Self {
-        // If source exceeds size limit, substitute a short source which will fail to parse.
-        // `parse()` will convert error to `diagnostics::OverlongSource`.
-        let source_text_for_lexer = if source_text.len() > MAX_LEN { "\0" } else { source_text };
-
         Self {
-            lexer: Lexer::new(allocator, source_text_for_lexer, source_type),
+            lexer: Lexer::new(allocator, source_text, source_type),
             source_type,
             source_text,
             errors: vec![],
@@ -254,7 +250,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Check if source length exceeds MAX_LEN, if the file cannot be parsed.
-    /// Original parsing error is not real - `Parser::new` substituted "\0" as the source text.
+    /// Original parsing error is not real - `Lexer::new` substituted "\0" as the source text.
     fn overlong_error(&self) -> Option<Error> {
         if self.source_text.len() > MAX_LEN {
             return Some(diagnostics::OverlongSource.into());