docs(parser): add module and struct level documentation (#5831)

crates/oxc_parser/examples/parser_tsx.rs

@@ -0,0 +1,40 @@
+use oxc_allocator::Allocator;
+use oxc_parser::{Parser, ParserReturn};
+use oxc_span::SourceType;
+
+fn main() {
+    let source_text = r"
+import React from 'react';
+
+/**
+ * A simple counter component
+ */
+export const Counter: React.FC = () => {
+    const [count, setCount] = React.useState(0);
+
+    return (
+        <div>
+            <p>Count: {count}</p>
+            <button onClick={() => setCount(count + 1)}>Increment</button>
+            <button onClick={() => setCount(count - 1)}>Decrement</button>
+        </div>
+    )
+}";
+
+    // Memory arena where AST nodes get stored
+    let allocator = Allocator::default();
+    // Infers TypeScript + JSX + ESM modules
+    let source_type = SourceType::from_path("Counter.tsx").unwrap();
+
+    let ParserReturn {
+        program,  // AST
+        errors,   // Syntax errors
+        panicked, // Parser encountered an error it couldn't recover from
+        trivias,  // Comments, whitespace, etc.
+    } = Parser::new(&allocator, source_text, source_type).parse();
+
+    assert!(!panicked);
+    assert!(errors.is_empty());
+    assert!(!program.body.is_empty());
+    assert_eq!(trivias.comments().count(), 1);
+}

crates/oxc_parser/src/lexer/kind.rs

@@ -2,6 +2,9 @@
 
 use std::fmt;
 
+/// Lexer token kind
+///
+/// Exported for other oxc crates to use. You generally don't need to use this directly.
 #[derive(Debug, Default, Clone, Copy, Eq, PartialEq)]
 #[non_exhaustive]
 pub enum Kind {
@@ -244,8 +247,11 @@ impl Kind {
         matches!(self, Ident) || self.is_all_keyword()
     }
 
-    /// Check the succeeding token of a `let` keyword
+    /// Check the succeeding token of a `let` keyword.
-    // let { a, b } = c, let [a, b] = c, let ident
+    ///
+    /// ```javascript
+    /// let { a, b } = c, let [a, b] = c, let ident
+    /// ```
     pub fn is_after_let(self) -> bool {
         self != Self::In && (matches!(self, LCurly | LBrack | Ident) || self.is_all_keyword())
     }

crates/oxc_parser/src/lib.rs

@@ -1,28 +1,50 @@
 //! Oxc Parser for JavaScript and TypeScript
 //!
-//! # Performance
+//! Oxc's [`Parser`] has full support for
-//!
+//! - The latest stable ECMAScript syntax
-//! The following optimization techniques are used:
+//! - TypeScript
-//! * AST is allocated in a memory arena ([bumpalo](https://docs.rs/bumpalo)) for fast AST drop
+//! - JSX and TSX
-//! * [oxc_span::Span] offsets uses `u32` instead of `usize`
+//! - [Stage 3 Decorators](https://github.com/tc39/proposal-decorator-metadata)
-//! * Scope binding, symbol resolution and complicated syntax errors are not done in the parser,
-//! they are delegated to the [semantic analyzer](https://docs.rs/oxc_semantic)
 //!
 //! # Usage
 //!
-//! The parser has a minimal API with three inputs and one return struct ([ParserReturn]).
+//! The parser has a minimal API with three inputs (a [memory arena](oxc_allocator::Allocator), a
+//! source string, and a [`SourceType`]) and one return struct (a [ParserReturn]).
 //!
 //! ```rust
 //! let parser_return = Parser::new(&allocator, &source_text, source_type).parse();
 //! ```
 //!
-//! # Example
+//! # Abstract Syntax Tree (AST)
-//! <https://github.com/Boshen/oxc/blob/main/crates/oxc_parser/examples/parser.rs>
+//! Oxc's AST is located in a separate [`oxc_ast`] crate. You can find type definitions for AST
+//! nodes [here][`oxc_ast::ast`].
+//!
+//! # Performance
+//!
+//! The following optimization techniques are used:
+//! * AST is allocated in a memory arena ([bumpalo](https://docs.rs/bumpalo)) for fast AST drop
+//! * [`oxc_span::Span`] offsets uses `u32` instead of `usize`
+//! * Scope binding, symbol resolution and complicated syntax errors are not done in the parser,
+//! they are delegated to the [semantic analyzer](https://docs.rs/oxc_semantic)
+//!
+//! <div class="warning">
+//! Because [`oxc_span::Span`] uses `u32` instead of `usize`, Oxc can only parse files up
+//! to 4 GiB in size. This shouldn't be a limitation in almost all cases.
+//! </div>
+//!
+//! # Examples
+//!
+//! <https://github.com/oxc-project/oxc/blob/main/crates/oxc_parser/examples/parser.rs>
 //!
 //! ```rust
 #![doc = include_str!("../examples/parser.rs")]
 //! ```
 //!
+//! ### Parsing TSX
+//! ```rust
+#![doc = include_str!("../examples/parser_tsx.rs")]
+//! ```
+//!
 //! # Visitor
 //!
 //! See [oxc_ast::Visit] and [oxc_ast::VisitMut]
@@ -91,39 +113,91 @@ pub const MAX_LEN: usize = if std::mem::size_of::<usize>() >= 8 {
     isize::MAX as usize
 };
 
-/// Return value of parser consisting of AST, errors and comments
+/// Return value of [`Parser::parse`] consisting of AST, errors and comments
 ///
-/// The parser always return a valid AST.
+/// ## AST Validity
-/// When `panicked = true`, then program will always be empty.
+///
-/// When `errors.len() > 0`, then program may or may not be empty due to error recovery.
+/// [`program`] will always contain a structurally valid AST, even if there are syntax errors.
+/// However, the AST may be semantically invalid. To ensure a valid AST,
+/// 1. Check that [`errors`] is empty
+/// 2. Run semantic analysis with [syntax error checking
+///    enabled](https://docs.rs/oxc_semantic/latest/oxc_semantic/struct.SemanticBuilder.html#method.with_check_syntax_error)
+///
+/// ## Errors
+/// Oxc's [`Parser`] is able to recover from some syntax errors and continue parsing. When this
+/// happens,
+/// 1. [`errors`] will be non-empty
+/// 2. [`program`] will contain a full AST
+/// 3. [`panicked`] will be false
+///
+/// When the parser cannot recover, it will abort and terminate parsing early. [`program`] will
+/// be empty and [`panicked`] will be `true`.
+///
+/// [`program`]: ParserReturn::program
+/// [`errors`]: ParserReturn::errors
+/// [`panicked`]: ParserReturn::panicked
 pub struct ParserReturn<'a> {
+    /// The parsed AST.
+    ///
+    /// Will be empty (e.g. no statements, directives, etc) if the parser panicked.
+    ///
+    /// ## Validity
+    /// It is possible for the AST to be present and semantically invalid. This will happen if
+    /// 1. The [`Parser`] encounters a recoverable syntax error
+    /// 2. The logic for checking the violation is in the semantic analyzer
+    ///
+    /// To ensure a valid AST, check that [`errors`](ParserReturn::errors) is empty. Then, run
+    /// semantic analysis with syntax error checking enabled.
     pub program: Program<'a>,
+
+    /// Syntax errors encountered while parsing.
+    ///
+    /// This list is not comprehensive. Oxc offloads more-expensive checks to [semantic
+    /// analysis](https://docs.rs/oxc_semantic), which can be enabled using
+    /// [`SemanticBuilder::with_check_syntax_error`](https://docs.rs/oxc_semantic/latest/oxc_semantic/struct.SemanticBuilder.html#method.with_check_syntax_error).
     pub errors: Vec<OxcDiagnostic>,
+
+    /// Comments and whitespace
     pub trivias: Trivias,
+
+    /// Whether the parser panicked and terminated early.
+    ///
+    /// This will be `false` if parsing was successful, or if parsing was able to recover from a
+    /// syntax error. When `true`, [`program`] will be empty and [`errors`] will contain at least
+    /// one error.
+    ///
+    /// [`program`]: ParserReturn::program
+    /// [`errors`]: ParserReturn::errors
     pub panicked: bool,
 }
 
 /// Parse options
+///
+/// You may provide options to the [`Parser`] using [`Parser::with_options`].
 #[derive(Debug, Clone, Copy)]
 pub struct ParseOptions {
     /// Whether to parse regular expressions or not.
     ///
-    /// Default: false
+    /// Default: `false`
     pub parse_regular_expression: bool,
 
-    /// Allow return outside of function
+    /// Allow [`return`] statements outside of functions.
     ///
-    /// By default, a return statement at the top level raises an error.
+    /// By default, a return statement at the top level raises an error (`false`).
-    /// Set this to true to accept such code.
+    /// Set this to `true` to accept such code.
+    ///
+    /// [`return`]: oxc_ast::ast::ReturnStatement
     pub allow_return_outside_function: bool,
 
-    /// Emit `ParenthesizedExpression` in AST.
+    /// Emit [`ParenthesizedExpression`]s in AST.
     ///
-    /// If this option is true, parenthesized expressions are represented by
+    /// If this option is `true`, parenthesized expressions are represented by
-    /// (non-standard) `ParenthesizedExpression` nodes that have a single `expression` property
+    /// (non-standard) [`ParenthesizedExpression`] nodes that have a single `expression` property
     /// containing the expression inside parentheses.
     ///
-    /// Default: true
+    /// Default: `true`
+    ///
+    /// [`ParenthesizedExpression`]: oxc_ast::ast::ParenthesizedExpression
     pub preserve_parens: bool,
 }
 
@@ -148,12 +222,18 @@ pub struct Parser<'a> {
 }
 
 impl<'a> Parser<'a> {
-    /// Create a new parser
+    /// Create a new [`Parser`]
+    ///
+    /// # Parameters
+    /// - `allocator`: [Memory arena](oxc_allocator::Allocator) for allocating AST nodes
+    /// - `source_text`: Source code to parse
+    /// - `source_type`: Source type (e.g. JavaScript, TypeScript, JSX, ESM Module, Script)
     pub fn new(allocator: &'a Allocator, source_text: &'a str, source_type: SourceType) -> Self {
         let options = ParseOptions::default();
         Self { allocator, source_text, source_type, options }
     }
 
+    /// Set parse options
     #[must_use]
     pub fn with_options(mut self, options: ParseOptions) -> Self {
         self.options = options;
@@ -200,6 +280,8 @@ mod parser_parse {
         ///
         /// Returns an empty `Program` on unrecoverable error,
         /// Recoverable errors are stored inside `errors`.
+        ///
+        /// See the [module-level documentation](crate) for examples and more information.
         pub fn parse(self) -> ParserReturn<'a> {
             let unique = UniquePromise::new();
             let parser = ParserImpl::new(
@@ -212,11 +294,25 @@ mod parser_parse {
             parser.parse()
         }
 
-        /// Parse `Expression`
+        /// Parse a single [`Expression`].
+        ///
+        /// # Example
+        ///
+        /// ```rust
+        /// use oxc_allocator::Allocator;
+        /// use oxc_ast::ast::Expression;
+        /// use oxc_parser::Parser;
+        /// use oxc_span::SourceType;
+        ///
+        /// let src = "let x = 1 + 2;";
+        /// let allocator = Allocator::new();
+        /// let source_type = SourceType::default();
+        ///
+        /// let expr: Expression<'_> = Parser::new(&allocator, src, source_type).parse_expression().unwrap();
+        /// ```
         ///
         /// # Errors
-        ///
+        /// If the source code being parsed has syntax errors.
-        /// * Syntax Error
         pub fn parse_expression(self) -> std::result::Result<Expression<'a>, Vec<OxcDiagnostic>> {
             let unique = UniquePromise::new();
             let parser = ParserImpl::new(