feat(tasks): benchmark NodeJS parser (#2770)

Add NodeJS parser to benchmarks.

Previous attempt #2724 did not work due CodSpeed producing very
inaccurate results (https://github.com/CodSpeedHQ/action/issues/96).

This version runs the actual benchmarks without CodSpeed's
instrumentation. Then another faux-benchmark runs within Codspeed's
instrumented action and just performs meaningless calculations in a loop
for as long as is required to take same amount of time as the original
uninstrumented benchmarks took.

It's unfortunate that we therefore don't get flame graphs on CodSpeed,
but this seems to be the best we can do for now.

.github/workflows/benchmark.yml

@@ -6,6 +6,8 @@ on:
     types: [opened, synchronize]
     paths:
       - '**/*.rs'
+      - 'napi/parser/**/*.js'
+      - 'napi/parser/**/*.mjs'
       - 'Cargo.lock'
       - '.github/workflows/benchmark.yml'
       - 'tasks/benchmark/codspeed/*.mjs'
@@ -15,6 +17,8 @@ on:
       - bench-*
     paths:
       - '**/*.rs'
+      - 'napi/parser/**/*.js'
+      - 'napi/parser/**/*.mjs'
       - 'Cargo.lock'
       - '.github/workflows/benchmark.yml'
       - 'tasks/benchmark/codspeed/*.mjs'
@@ -31,7 +35,7 @@ jobs:
       matrix:
         # Run each benchmark in own job.
         # Linter benchmark is by far the slowest, so split each fixture into own job.
-        component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap]
+        component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap, parser_napi]
         include:
           - component: linter
             fixture: 0
@@ -53,7 +57,7 @@ jobs:
       - name: Install Rust Toolchain
         uses: ./.github/actions/rustup
         with:
-          shared-key: 'benchmark'
+          shared-key: ${{ matrix.component == 'parser_napi' && 'benchmark_napi' || 'benchmark' }}
           save-cache: ${{ github.ref_name == 'main' }}
 
       - name: Install codspeed
@@ -77,12 +81,31 @@ jobs:
           pnpm install
           node capture.mjs &
 
-      - name: Build Benchmark
+      # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`,
+      # so instead run real benchmark without CodSpeed's instrumentation and save the results.
+      # Then "Run benchmark" step below runs a loop of some simple Rust code the number
+      # of times required to take same amount of time as the real benchmark took.
+      # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96
+      - name: Build NAPI Benchmark
+        if: ${{ matrix.component == 'parser_napi'}}
+        working-directory: ./napi/parser
+        run: |
+          corepack enable
+          pnpm install
+          pnpm build
+
+      - name: Run NAPI Benchmark
+        if: ${{ matrix.component == 'parser_napi'}}
+        working-directory: ./napi/parser
+        run: node parse.bench.mjs
+
+      - name: Build benchmark
         env:
           RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
         shell: bash
         run: |
-          cargo build --release -p oxc_benchmark --features codspeed --bench ${{ matrix.component }}
+          cargo build --release -p oxc_benchmark --bench ${{ matrix.component }} \
+            --features ${{ matrix.component == 'parser_napi' && 'codspeed_napi' || 'codspeed'}}
           mkdir -p target/codspeed/oxc_benchmark/
           mv target/release/deps/${{ matrix.component }}-* target/codspeed/oxc_benchmark
           rm -rf target/codspeed/oxc_benchmark/*.d

Cargo.lock

@@ -1358,6 +1358,8 @@ dependencies = [
  "oxc_span",
  "oxc_tasks_common",
  "oxc_transformer",
+ "serde",
+ "serde_json",
 ]
 
 [[package]]

napi/parser/package.json

@@ -7,11 +7,10 @@
     "bench": "vitest bench"
   },
   "devDependencies": {
-    "@codspeed/vitest-plugin": "^3.1.0",
     "@napi-rs/cli": "^2.18.0",
     "es-module-lexer": "^1.4.1",
     "flatbuffers": "^23.5.26",
-    "vitest": "^1.3.1"
+    "tinybench": "^2.6.0"
   },
   "engines": {
     "node": ">=14.*"

napi/parser/parse.bench.mjs

@@ -1,10 +1,11 @@
 import {fileURLToPath} from 'url';
 import {join as pathJoin} from 'path';
 import {readFile, writeFile} from 'fs/promises';
-import assert from 'assert';
-import {bench} from 'vitest';
+import {Bench} from 'tinybench';
 import {parseSync} from './index.js';
 
+const IS_CI = !!process.env.CI;
+
 const urls = [
     // TypeScript syntax (2.81MB)
     'https://raw.githubusercontent.com/microsoft/TypeScript/v5.3.3/src/compiler/checker.ts',
@@ -28,9 +29,9 @@ const files = await Promise.all(urls.map(async (url) => {
     let code;
     try {
         code = await readFile(path, 'utf8');
-        console.log('Found cached file:', filename);
+        if (IS_CI) console.log('Found cached file:', filename);
     } catch {
-        console.log('Downloading:', filename);
+        if (IS_CI) console.log('Downloading:', filename);
         const res = await fetch(url);
         code = await res.text();
         await writeFile(path, code);
@@ -39,10 +40,27 @@ const files = await Promise.all(urls.map(async (url) => {
     return {filename, code};
 }));
 
+const bench = new Bench();
+
 for (const {filename, code} of files) {
-    bench(`parser(napi)[${filename}]`, () => {
+    bench.add(`parser_napi[${filename}]`, () => {
         const res = parseSync(code, {sourceFilename: filename});
-        assert(res.errors.length === 0);
         JSON.parse(res.program);
     });
 }
+
+console.log('Warming up');
+await bench.warmup();
+console.log('Running benchmarks');
+await bench.run();
+console.table(bench.table());
+
+// If running on CI, save results to file
+if (IS_CI) {
+    const dataDir = process.env.DATA_DIR;
+    const results = bench.tasks.map(task => ({
+        filename: task.name.match(/\[(.+)\]$/)[1],
+        duration: task.result.period / 1000, // In seconds
+    }));
+    await writeFile(pathJoin(dataDir, 'results.json'), JSON.stringify(results));
+}

napi/parser/vitest.config.mjs

@@ -1,6 +0,0 @@
-import {defineConfig} from 'vitest/config';
-import codspeedPlugin from '@codspeed/vitest-plugin';
-
-export default defineConfig({
-    plugins: process.env.CI ? [codspeedPlugin()] : []
-});

tasks/benchmark/Cargo.toml

@@ -51,6 +51,12 @@ harness = false
 name    = "minifier"
 harness = false
 
+# Only run in CI
+[[bench]]
+name    = "parser_napi"
+harness = false
+bench   = false
+
 [dependencies]
 oxc_allocator    = { workspace = true }
 oxc_linter       = { workspace = true }
@@ -63,7 +69,10 @@ oxc_tasks_common = { workspace = true }
 oxc_transformer  = { workspace = true }
 oxc_codegen      = { workspace = true }
 
-criterion = { package = "criterion2", version = "0.6.0", default-features = false }
+criterion  = { package = "criterion2", version = "0.6.0", default-features = false }
+serde      = { workspace = true, optional = true }
+serde_json = { workspace = true, optional = true }
 
 [features]
-codspeed = ["criterion/codspeed"]
+codspeed      = ["criterion/codspeed"]
+codspeed_napi = ["criterion/codspeed", "dep:serde", "dep:serde_json"]

tasks/benchmark/benches/parser_napi.rs

@@ -0,0 +1,53 @@
+use std::{env, fs, path::PathBuf, time::Duration};
+
+use oxc_benchmark::{
+    black_box, criterion_group, criterion_main, BenchmarkId, Criterion, SamplingMode,
+};
+
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+struct BenchResult {
+    filename: String,
+    duration: f64,
+}
+
+/// This is a fake benchmark which is only here to get benchmarks for NAPI parser into CodSpeed.
+/// It's a workaround for CodSpeed's measurement of JS + NAPI being wildly inaccurate:
+/// https://github.com/CodSpeedHQ/action/issues/96
+/// So instead in CI we run the actual benchmark without CodSpeed's instrumentation
+/// (see `.github/workflows/benchmark.yml` and `napi/parser/parse.bench.mjs`).
+/// `parse.bench.mjs` writes the results of the benchmarks to a file `results.json`.
+/// This pseudo-benchmark reads that file and just performs meaningless calculations in a loop
+/// the number of times required to take same amount of time as the original benchmark.
+fn bench_parser_napi(criterion: &mut Criterion) {
+    let data_dir = env::var("DATA_DIR").unwrap();
+    let results_path: PathBuf = [&data_dir, "results.json"].iter().collect();
+    let results_file = fs::File::open(&results_path).unwrap();
+    let files: Vec<BenchResult> = serde_json::from_reader(results_file).unwrap();
+    fs::remove_file(&results_path).unwrap();
+
+    let mut group = criterion.benchmark_group("parser_napi");
+    // Reduce time to run benchmark as much as possible (10 is min for sample size)
+    group.sample_size(10);
+    group.warm_up_time(Duration::from_micros(1));
+    group.sampling_mode(SamplingMode::Flat);
+    for file in files {
+        let cycles = (file.duration * 266672645.0) as u64;
+        group.bench_function(BenchmarkId::from_parameter(&file.filename), |b| {
+            b.iter(|| {
+                let cycles = black_box(cycles);
+                let mut n: u64 = 0x1c2e9b89d37e0c1b;
+                for _ in 0..cycles {
+                    n = n.rotate_right(3);
+                    n = n ^ 0x18bb6752b938b511;
+                }
+                black_box(n);
+            });
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(parser, bench_parser_napi);
+criterion_main!(parser);