feat(codegen): json strings proposal (#1039)

crates/oxc_codegen/src/gen.rs

@@ -2,6 +2,7 @@ use oxc_allocator::{Box, Vec};
 #[allow(clippy::wildcard_imports)]
 use oxc_ast::ast::*;
 use oxc_syntax::{
+    identifier::{LS, PS},
     operator::{BinaryOperator, UnaryOperator},
     precedence::{GetPrecedence, Precedence},
     NumberBase,
@@ -91,10 +92,8 @@ impl<const MINIFY: bool> Gen<MINIFY> for Hashbang {
 
 impl<const MINIFY: bool> Gen<MINIFY> for Directive {
     fn gen(&self, p: &mut Codegen<{ MINIFY }>, _ctx: Context) {
-        p.print_indent();
-        p.print(b'\'');
-        p.print_str(self.directive.as_bytes());
-        p.print(b'\'');
+        // Use the string value instead of the raw self.directive because it can cannot escaped values.
+        print_str(self.expression.value.as_str(), p);
         p.print_semicolon();
     }
 }
@@ -1062,13 +1061,24 @@ impl<const MINIFY: bool> Gen<MINIFY> for RegExpLiteral {
     }
 }
 
+fn print_str<const MINIFY: bool>(s: &str, p: &mut Codegen<{ MINIFY }>) {
+    p.print(b'\'');
+    for c in s.chars() {
+        match c {
+            // Allow `U+2028` and `U+2029` in string literals
+            // <https://tc39.es/proposal-json-superset>
+            // <https://github.com/tc39/proposal-json-superset>
+            LS => p.print_str(b"\\u2028"),
+            PS => p.print_str(b"\\u2029"),
+            _ => p.print_str(c.escape_default().to_string().as_bytes()),
+        }
+    }
+    p.print(b'\'');
+}
+
 impl<const MINIFY: bool> Gen<MINIFY> for StringLiteral {
     fn gen(&self, p: &mut Codegen<{ MINIFY }>, _ctx: Context) {
-        p.print(b'\'');
-        for c in self.value.chars() {
-            p.print_str(c.escape_default().to_string().as_bytes());
-        }
-        p.print(b'\'');
+        print_str(self.value.as_str(), p);
     }
 }
 

crates/oxc_parser/src/lexer/mod.rs

@@ -291,11 +291,6 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    /// Add string to `SourceAtomSet` and get `TokenValue::Atom`
-    fn string_to_token_value(&mut self, s: &'a str) -> TokenValue<'a> {
-        TokenValue::String(s)
-    }
-
     fn set_numeric_value(&mut self, kind: Kind, src: &'a str) {
         let value = match kind {
             Kind::Decimal | Kind::Binary | Kind::Octal | Kind::Hex => {
@@ -549,7 +544,7 @@ impl<'a> Lexer<'a> {
             }
         }
         let (_, name) = self.identifier_tail(builder);
-        self.current.token.value = self.string_to_token_value(name);
+        self.current.token.value = TokenValue::String(name);
         Kind::PrivateIdentifier
     }
 
@@ -782,7 +777,7 @@ impl<'a> Lexer<'a> {
                 Some(c @ ('"' | '\'')) => {
                     if c == delimiter {
                         self.current.token.value =
-                            self.string_to_token_value(builder.finish_without_push(self));
+                            TokenValue::String(builder.finish_without_push(self));
                         return Kind::Str;
                     }
                     builder.push_matching(c);
@@ -797,8 +792,8 @@ impl<'a> Lexer<'a> {
                         self.error(diagnostics::InvalidEscapeSequence(range));
                     }
                 }
-                Some(other) => {
-                    builder.push_matching(other);
+                Some(c) => {
+                    builder.push_matching(c);
                 }
             }
         }
@@ -881,7 +876,7 @@ impl<'a> Lexer<'a> {
                 '$' if self.peek() == Some('{') => {
                     if is_valid_escape_sequence {
                         self.current.token.value =
-                            self.string_to_token_value(builder.finish_without_push(self));
+                            TokenValue::String(builder.finish_without_push(self));
                     }
                     self.current.chars.next();
                     return substitute;
@@ -889,7 +884,7 @@ impl<'a> Lexer<'a> {
                 '`' => {
                     if is_valid_escape_sequence {
                         self.current.token.value =
-                            self.string_to_token_value(builder.finish_without_push(self));
+                            TokenValue::String(builder.finish_without_push(self));
                     }
                     return tail;
                 }
@@ -936,7 +931,7 @@ impl<'a> Lexer<'a> {
         }
         let mut s = String::from_str_in(prev_str, self.allocator);
         s.push_str(builder.finish(self));
-        self.current.token.value = self.string_to_token_value(s.into_bump_str());
+        self.current.token.value = TokenValue::String(s.into_bump_str());
         Kind::Ident
     }
 
@@ -971,7 +966,7 @@ impl<'a> Lexer<'a> {
                         break;
                     }
                 }
-                self.current.token.value = self.string_to_token_value(builder.finish(self));
+                self.current.token.value = TokenValue::String(builder.finish(self));
                 Kind::JSXText
             }
             None => Kind::Eof,
@@ -995,7 +990,7 @@ impl<'a> Lexer<'a> {
                 Some(c @ ('"' | '\'')) => {
                     if c == delimiter {
                         self.current.token.value =
-                            self.string_to_token_value(builder.finish_without_push(self));
+                            TokenValue::String(builder.finish_without_push(self));
                         return Kind::Str;
                     }
                     builder.push_matching(c);
@@ -1194,11 +1189,19 @@ impl<'a> Lexer<'a> {
                 self.error(diagnostics::UnterminatedString(self.unterminated_range()));
             }
             Some(c) => match c {
-                // CharacterEscapeSequence
+                // \ LineTerminatorSequence
+                // LineTerminatorSequence ::
+                // <LF>
+                // <CR> [lookahead ≠ <LF>]
+                // <LS>
+                // <PS>
+                // <CR> <LF>
                 LF | LS | PS => {}
                 CR => {
                     self.next_eq(LF);
                 }
+                // SingleEscapeCharacter :: one of
+                //   ' " \ b f n r t v
                 '\'' | '"' | '\\' => text.push(c),
                 'b' => text.push('\u{8}'),
                 'f' => text.push(FF),

tasks/transform_conformance/babel.snap.md

@@ -1,8 +1,9 @@
-Passed: 162/1083
+Passed: 164/1083
 
 # All Passed:
 * babel-plugin-transform-numeric-separator
 * babel-plugin-transform-optional-catch-binding
+* babel-plugin-transform-json-strings
 * babel-plugin-transform-shorthand-properties
 * babel-plugin-transform-sticky-regex
 
@@ -560,10 +561,6 @@ Passed: 162/1083
 * transparent-expr-wrappers/ts-as-member-expression/input.ts
 * transparent-expr-wrappers/ts-parenthesized-expression-member-call/input.ts
 
-# babel-plugin-transform-json-strings (2/4)
-* json-strings/directive-line-separator/input.js
-* json-strings/directive-paragraph-separator/input.js
-
 # babel-plugin-transform-async-generator-functions (0/24)
 * async-generators/class-method/input.js
 * async-generators/class-private-method/input.js

tasks/transform_conformance/src/lib.rs

@@ -239,7 +239,7 @@ impl TestCase {
             .build(program);
         let transformed_code = Codegen::<false>::new(input.len(), CodegenOptions).build(program);
 
-        // Get output.js by using our codeg so code comparison can match.
+        // Get output.js by using our codegen so code comparison can match.
         let output = output_path.and_then(|path| fs::read_to_string(path).ok()).map_or_else(
             || {
                 // The transformation should be equal to input.js If output.js does not exist.