ci: add resolver benchmark (#546)

.github/workflows/benchmark.yml

@@ -7,6 +7,8 @@ on:
     paths:
       - '**.rs'
       - '.github/workflows/benchmark.yml'
+    branches-ignore:
+      - 'crates/oxc_resolver/'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
@@ -43,7 +45,7 @@ jobs:
         shell: bash
         run: sleep 15s
 
-      - name: Run Bench on PR Branch
+      - name: Run Bench on Main Branch
         run: cargo benchmark --save-baseline main
 
       - name: Checkout Main Branch
@@ -65,7 +67,7 @@ jobs:
         shell: bash
         run: sleep 15s
 
-      - name: Run Bench on Main Branch
+      - name: Run Bench on PR Branch
         run: cargo benchmark --save-baseline pr
 
       - name: Upload benchmark results

.github/workflows/benchmark_resolver.yml

@@ -0,0 +1,142 @@
+name: Benchmark Resolver
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+    paths:
+      - 'crates/oxc_resolver/'
+      - '.github/workflows/benchmark_resolver.yml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest] # `macos-latest` is too unstable to be useful for benchmark, the variance is always huge.
+    name: Run benchmark on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: main # Checkout main first because the cache is warm
+
+      - name: Install Rust Toolchain
+        uses: ./.github/actions/rustup
+        with:
+          shared-key: benchmark
+
+      - name: Compile
+        run: cargo build --release --bench resolver
+
+      - name: Sleep for CPU cooldown (windows)
+        if: runner.os == 'Windows'
+        shell: powershell
+        run: Start-Sleep -s 15
+
+      - name: Sleep for CPU cooldown
+        if: runner.os != 'Windows'
+        shell: bash
+        run: sleep 15s
+
+      - name: Run Bench on Main Branch
+        run: cargo bench --bench resolver -- --save-baseline main
+
+      - name: Checkout Main Branch
+        uses: actions/checkout@v3
+        with:
+          clean: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Compile
+        run: cargo build --release --bench resolver
+
+      - name: Sleep for CPU cooldown (windows)
+        if: runner.os == 'Windows'
+        shell: powershell
+        run: Start-Sleep -s 15
+
+      - name: Sleep for CPU cooldown
+        if: runner.os != 'Windows'
+        shell: bash
+        run: sleep 15s
+
+      - name: Run Bench on PR Branch
+        run: cargo bench --bench resolver -- --save-baseline pr
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v3
+        with:
+          name: benchmark-resolver-${{ matrix.os }}
+          path: ./target/criterion
+
+  benchmark-compare:
+    runs-on: ubuntu-latest
+    name: Compare Benchmarks
+    needs:
+      - benchmark
+
+    steps:
+      - name: Install critcmp
+        uses: taiki-e/install-action@v2
+        with:
+          tool: critcmp
+
+      - name: Linux | Download PR benchmark results
+        uses: actions/download-artifact@v3
+        with:
+          name: benchmark-resolver-ubuntu-latest
+          path: ./target/criterion
+
+      - name: Linux | Compare benchmark results
+        shell: bash
+        run: |
+          echo "## Benchmark Resolver" >> summary.md
+          echo "### Linux" >> summary.md
+          echo "\`\`\`" >> summary.md
+          critcmp main pr >> summary.md
+          echo "\`\`\`" >> summary.md
+          echo "" >> summary.md
+
+      - name: Linux | Cleanup benchmark results
+        run: rm -rf ./target/criterion
+
+      - name: Windows | Download PR benchmark results
+        uses: actions/download-artifact@v3
+        with:
+          name: benchmark-resolver-windows-latest
+          path: ./target/criterion
+
+      - name: Windows | Compare benchmark results
+        shell: bash
+        run: |
+          echo "### Windows" >> summary.md
+          echo "\`\`\`" >> summary.md
+          critcmp main pr >> summary.md
+          echo "\`\`\`" >> summary.md
+          echo "" >> summary.md
+          cat summary.md > $GITHUB_STEP_SUMMARY
+
+      - name: Find Comment
+        # Check if the event is not triggered by a fork
+        if: github.event.pull_request.head.repo.full_name == github.repository
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Benchmark Resolver
+
+      - name: Create or update comment
+        # Check if the event is not triggered by a fork
+        if: github.event.pull_request.head.repo.full_name == github.repository
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          edit-mode: replace
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body-file: summary.md