oxc/.github/workflows/benchmark.yml

name: Benchmark

on:
  workflow_dispatch:
  pull_request:
    types: [opened, synchronize]
    paths:
      - '**/*.rs'
      - 'napi/parser/**/*.js'
      - 'napi/parser/**/*.mjs'
      - 'Cargo.lock'
      - '.github/workflows/benchmark.yml'
      - 'tasks/benchmark/codspeed/*.mjs'
  push:
    branches:
      - main
    paths:
      - '**/*.rs'
      - 'napi/parser/**/*.js'
      - 'napi/parser/**/*.mjs'
      - 'Cargo.lock'
      - '.github/workflows/benchmark.yml'
      - 'tasks/benchmark/codspeed/*.mjs'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  build:
    name: Build Benchmark
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Branch
        uses: taiki-e/checkout-action@v1

      - name: Install Rust Toolchain
        uses: ./.github/actions/rustup
        with:
          shared-key: 'benchmark'
          save-cache: ${{ github.ref_name == 'main' }}

      - name: Build benchmark
        env:
          RUSTFLAGS: "-C debuginfo=1 -C strip=none -g --cfg codspeed"
        shell: bash
        run: |
          cargo build --release -p oxc_benchmark --benches --features codspeed
          rm target/release/deps/*.d
          mkdir -p target/codspeed/oxc_benchmark/
          mv target/release/deps/lexer-* target/codspeed/oxc_benchmark/lexer
          mv target/release/deps/parser-* target/codspeed/oxc_benchmark/parser
          mv target/release/deps/transformer-* target/codspeed/oxc_benchmark/transformer
          mv target/release/deps/semantic-* target/codspeed/oxc_benchmark/semantic
          mv target/release/deps/minifier-* target/codspeed/oxc_benchmark/minifier
          mv target/release/deps/codegen_sourcemap-* target/codspeed/oxc_benchmark/codegen_sourcemap
          mv target/release/deps/sourcemap-* target/codspeed/oxc_benchmark/sourcemap
          mv target/release/deps/linter-* target/codspeed/oxc_benchmark/linter          

      - name: Upload Binary
        uses: actions/upload-artifact@v4
        with:
          if-no-files-found: error
          name: benchmark
          path: ./target/codspeed/oxc_benchmark

  benchmark:
    needs: build
    name: Benchmark
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        component:
          - lexer
          - parser
          - transformer
          - semantic
          - minifier
          - codegen_sourcemap
          - sourcemap
          - linter

    steps:
      - name: Checkout Branch
        uses: taiki-e/checkout-action@v1

      - name: Download Artifacts
        uses: actions/download-artifact@v4
        with:
          name: benchmark
          path: ./target/codspeed/oxc_benchmark

      - name: Fix permission loss
        shell: bash
        run: |
          ls ./target/codspeed/oxc_benchmark
          chmod +x ./target/codspeed/oxc_benchmark/${{ matrix.component }}          

      - name: Install codspeed
        uses: taiki-e/install-action@v2
        with:
          tool: cargo-codspeed

      - name: Install Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          registry-url: 'https://registry.npmjs.org'

      - name: Start bench results interceptor server
        working-directory: ./tasks/benchmark/codspeed
        env:
          COMPONENT: ${{ matrix.component }}
        run: |
          corepack enable
          pnpm install
          node capture.mjs &          

      - name: Run benchmark
        uses: CodSpeedHQ/action@v2
        timeout-minutes: 30
        with:
          # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
          token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
          upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload
          run: cargo codspeed run ${{ matrix.component }}

      - name: Upload bench data artefact
        uses: actions/upload-artifact@v4
        with:
          name: result-${{ matrix.component }}
          path: ${{ env.DATA_DIR }} # # env.DEFAULT_PORT from capture.mjs
          if-no-files-found: error
          retention-days: 1

  # benchmark-napi:
    # name: Benchmark NAPI parser
    # runs-on: ubuntu-latest
    # if: false
    # steps:
      # - name: Checkout Branch
        # uses: taiki-e/checkout-action@v1

      # - name: Install Rust Toolchain
        # uses: ./.github/actions/rustup
        # with:
          # shared-key: 'benchmark_napi'
          # save-cache: ${{ github.ref_name == 'main' }}

      # - name: Install codspeed
        # uses: taiki-e/install-action@v2
        # with:
          # tool: cargo-codspeed

      # - name: Install Node.js
        # uses: actions/setup-node@v4
        # with:
          # node-version: 20
          # registry-url: 'https://registry.npmjs.org'

      # - name: Start bench results interceptor server
        # working-directory: ./tasks/benchmark/codspeed
        # env:
          # COMPONENT: 'parser_napi'
        # run: |
          # corepack enable
          # pnpm install
          # node capture.mjs &

      # # CodSpeed gets measurements completely off for NAPI if run in `CodSpeedHQ/action`,
      # # so instead run real benchmark without CodSpeed's instrumentation and save the results.
      # # Then "Run Rust benchmark" step below runs a loop of some simple Rust code the number
      # # of times required to take same amount of time as the real benchmark took.
      # # This is all a workaround for https://github.com/CodSpeedHQ/action/issues/96
      # - name: Build NAPI Benchmark
        # working-directory: ./napi/parser
        # run: |
          # corepack enable
          # pnpm install
          # pnpm build

      # - name: Run NAPI Benchmark
        # working-directory: ./napi/parser
        # run: node parse.bench.mjs

      # - name: Build Rust benchmark
        # env:
          # RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed"
        # shell: bash
        # run: |
          # cargo build --release -p oxc_benchmark --bench parser_napi --no-default-features --features codspeed_napi
          # mkdir -p target/codspeed/oxc_benchmark/
          # mv target/release/deps/parser_napi-* target/codspeed/oxc_benchmark
          # rm -rf target/codspeed/oxc_benchmark/*.d

      # - name: Run Rust benchmark
        # uses: CodSpeedHQ/action@v2
        # timeout-minutes: 30
        # with:
          # run: cargo codspeed run
          # # Dummy token for tokenless runs, to suppress logging hash of metadata JSON (see `upload.mjs`)
          # token: ${{ secrets.CODSPEED_TOKEN || 'dummy' }}
          # upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload

      # - name: Upload bench data artefact
        # uses: actions/upload-artifact@v4
        # with:
          # name: 'parser_napi'
          # path: ${{ env.DATA_DIR }}
          # if-no-files-found: error
          # retention-days: 1

  upload:
    name: Upload benchmarks
    # needs: [benchmark, benchmark-napi]
    needs: [benchmark]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Branch
        uses: taiki-e/checkout-action@v1

      - name: Install Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          registry-url: 'https://registry.npmjs.org'

      - name: Create temp dir
        working-directory: ./tasks/benchmark/codspeed
        run: |
          corepack enable
          pnpm install
          node create_temp_dir.mjs          

      - name: Download artefacts
        uses: actions/download-artifact@v4
        with:
          merge-multiple: true
          pattern: result-*
          path: ${{ env.DATA_DIR }} # env.DEFAULT_PORT from create_temp_dir.mjs

      - name: Upload to Codspeed
        working-directory: ./tasks/benchmark/codspeed
        env:
          CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }}
        run: node upload.mjs