From 93e1ea44197b449755ea79db7271e357e2aa089c Mon Sep 17 00:00:00 2001 From: overlookmotel Date: Mon, 18 Mar 2024 02:45:44 +0000 Subject: [PATCH] feat(tasks): shard benchmarks in CI (#2751) This PR shards benchmarks when running on CI. Each benchmark (parser, minifier etc) runs as a separate job, and then a final job combines the results and uploads to Codspeed. A bit of a hacky implementation. Uses a small NodeJS HTTP server to intercept the results from `codspeed-runner`, and then another NodeJS script to combine them all together, and upload to CodSpeed. I will submit PRs on Codspeed's runner + action to do it properly, but as I imagine it'll be a slow process getting that merged upstream, I wanted to see if it worked first. We can replace this once it's supported upstream. Sharding only reduces total time to run the benchmarks by about 70 secs at present, because linter benchmark takes 6 mins alone and holds up the whole process (all the rest are done in ~2 mins). If we can split up the linter benchmark, we can likely get total run time down to around 3 mins. I'll try that in a follow-on PR. I guess the other upside is we can now add as many benchmarks as we like with impunity - they'll run in parallel, and so won't slow things down overall. --- .github/workflows/benchmark.yml | 77 ++- .gitignore | 1 + tasks/benchmark/codspeed/capture.mjs | 73 +++ tasks/benchmark/codspeed/create_temp_dir.mjs | 13 + tasks/benchmark/codspeed/package.json | 9 + tasks/benchmark/codspeed/pnpm-lock.yaml | 580 +++++++++++++++++++ tasks/benchmark/codspeed/upload.mjs | 135 +++++ 7 files changed, 879 insertions(+), 9 deletions(-) create mode 100644 tasks/benchmark/codspeed/capture.mjs create mode 100644 tasks/benchmark/codspeed/create_temp_dir.mjs create mode 100644 tasks/benchmark/codspeed/package.json create mode 100644 tasks/benchmark/codspeed/pnpm-lock.yaml create mode 100644 tasks/benchmark/codspeed/upload.mjs diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2e964f9f3..3448f0eec 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -8,6 +8,7 @@ on: - '**/*.rs' - 'Cargo.lock' - '.github/workflows/benchmark.yml' + - 'tasks/benchmark/codspeed/*.mjs' push: branches: - main @@ -16,6 +17,7 @@ on: - '**/*.rs' - 'Cargo.lock' - '.github/workflows/benchmark.yml' + - 'tasks/benchmark/codspeed/*.mjs' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} @@ -25,11 +27,14 @@ jobs: benchmark: name: Benchmark runs-on: ubuntu-latest + strategy: + matrix: + component: [lexer, parser, transformer, semantic, linter, minifier, codegen_sourcemap] steps: - name: Checkout Branch uses: actions/checkout@v4 with: - progress: false + show-progress: false persist-credentials: false - name: Install Rust Toolchain @@ -43,20 +48,29 @@ jobs: with: tool: cargo-codspeed + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + registry-url: 'https://registry.npmjs.org' + + - name: Start bench results interceptor server + working-directory: ./tasks/benchmark/codspeed + env: + COMPONENT: ${{ matrix.component }} + run: | + corepack enable + pnpm install + node capture.mjs & + - name: Build Benchmark env: RUSTFLAGS: "-C debuginfo=2 -C strip=none -g --cfg codspeed" shell: bash run: | - cargo build --release -p oxc_benchmark --features codspeed --bench lexer --bench parser --bench transformer --bench semantic --bench linter --bench minifier --bench codegen_sourcemap + cargo build --release -p oxc_benchmark --features codspeed --bench ${{ matrix.component }} mkdir -p target/codspeed/oxc_benchmark/ - mv target/release/deps/lexer-* target/codspeed/oxc_benchmark - mv target/release/deps/parser-* target/codspeed/oxc_benchmark - mv target/release/deps/transformer-* target/codspeed/oxc_benchmark - mv target/release/deps/semantic-* target/codspeed/oxc_benchmark - mv target/release/deps/linter-* target/codspeed/oxc_benchmark - mv target/release/deps/minifier-* target/codspeed/oxc_benchmark - mv target/release/deps/codegen_sourcemap-* target/codspeed/oxc_benchmark + mv target/release/deps/${{ matrix.component }}-* target/codspeed/oxc_benchmark rm -rf target/codspeed/oxc_benchmark/*.d - name: Run benchmark @@ -65,3 +79,48 @@ jobs: with: run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} + upload-url: http://localhost:${{ env.INTERCEPT_PORT }}/upload + + - name: Upload bench data artefact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.component }} + path: ${{ env.DATA_DIR }} + if-no-files-found: error + retention-days: 1 + + upload: + name: Upload benchmarks + needs: benchmark + runs-on: ubuntu-latest + steps: + - name: Checkout Branch + uses: actions/checkout@v4 + with: + show-progress: false + persist-credentials: false + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + registry-url: 'https://registry.npmjs.org' + + - name: Create temp dir + working-directory: ./tasks/benchmark/codspeed + run: | + corepack enable + pnpm install + node create_temp_dir.mjs + + - name: Download artefacts + uses: actions/download-artifact@v4 + with: + path: ${{ env.DATA_DIR }} + merge-multiple: true + + - name: Upload to Codspeed + working-directory: ./tasks/benchmark/codspeed + env: + CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }} + run: node upload.mjs diff --git a/.gitignore b/.gitignore index 2e57dcf26..569b67a04 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ target/ /node_modules/ /website/node_modules/ /benchmark/node_modules/ +/tasks/benchmark/codspeed/node_modules/ /editors/vscode/node_modules/ /editors/vscode/icon.png diff --git a/tasks/benchmark/codspeed/capture.mjs b/tasks/benchmark/codspeed/capture.mjs new file mode 100644 index 000000000..9e11fe6ca --- /dev/null +++ b/tasks/benchmark/codspeed/capture.mjs @@ -0,0 +1,73 @@ +/* + * HTTP server to intercept benchmark data from Codspeed runner. + * + * Codspeed runner makes 2 API calls: + * 1. Uploading metadata + * 2. Uploading archive of CPU profile files + * + * Server starts on an available port, saves the files sent by Codspeed runner to a directory, + * then shuts itself down. + */ + +import fs from 'fs'; +import {pipeline} from 'stream/promises'; +import express from 'express'; + +const DEFAULT_PORT = 3000, + LISTEN_ATTEMPTS = 10; + +// Create directory for saving assets +const rand = Math.round(Math.random() * 1000000000000000000).toString(16), + dataDir = `/tmp/oxc_bench_data_${rand}`; +fs.mkdirSync(dataDir); + +const component = process.env.COMPONENT; + +const app = express(); + +app.post('/upload', (req, res, next) => { + saveBody(req, 'metadata.json', next, () => { + res.json({ + status: 'success', + uploadUrl: `http://localhost:${port}/upload_archive`, + runId: 'dummy_value', + }); + }); +}); + +app.put('/upload_archive', (req, res, next) => { + saveBody(req, 'archive.tar.gz', next, () => { + res.send('OK'); + server.close(() => {}); + }); +}); + +function saveBody(req, filename, onError, done) { + (async () => { + const stream = fs.createWriteStream(`${dataDir}/${component}_${filename}`); + await pipeline(req, stream); + done(); + })().catch(onError); +} + +// Open server on a port which is not already in use +let server, + port = DEFAULT_PORT; +for (let i = 0; i < LISTEN_ATTEMPTS; i++) { + console.log(`Starting server on port ${port}`); + try { + await new Promise((resolve, reject) => { + server = app.listen(port, resolve); + server.on('error', reject); + }); + break; + } catch (err) { + if (err?.code !== 'EADDRINUSE') throw err; + console.log(`Port ${port} in use. Trying again.`); + port = DEFAULT_PORT + Math.round(Math.random() * 5000); + } +} +console.log(`Server listening on port ${port}`); + +// Output data dir path + port to env vars +fs.appendFileSync(process.env.GITHUB_ENV, `DATA_DIR=${dataDir}\nINTERCEPT_PORT=${port}\n`); diff --git a/tasks/benchmark/codspeed/create_temp_dir.mjs b/tasks/benchmark/codspeed/create_temp_dir.mjs new file mode 100644 index 000000000..f07559d8b --- /dev/null +++ b/tasks/benchmark/codspeed/create_temp_dir.mjs @@ -0,0 +1,13 @@ +/* + * Create temp dir for downloading artefacts to. + */ + +import fs from 'fs'; + +// Create directory for saving assets +const rand = Math.round(Math.random() * 1000000000000000000).toString(16), + dataDir = `/tmp/oxc_bench_data_${rand}`; +fs.mkdirSync(dataDir); + +// Output dir path to env var +fs.appendFileSync(process.env.GITHUB_ENV, `DATA_DIR=${dataDir}\n`); diff --git a/tasks/benchmark/codspeed/package.json b/tasks/benchmark/codspeed/package.json new file mode 100644 index 000000000..5d1bf33fb --- /dev/null +++ b/tasks/benchmark/codspeed/package.json @@ -0,0 +1,9 @@ +{ + "name": "oxc_benchmark", + "version": "0.0.0", + "devDependencies": { + "axios": "^1.6.8", + "express": "^4.18.3", + "tar": "^6.2.0" + } +} diff --git a/tasks/benchmark/codspeed/pnpm-lock.yaml b/tasks/benchmark/codspeed/pnpm-lock.yaml new file mode 100644 index 000000000..4b3cd163f --- /dev/null +++ b/tasks/benchmark/codspeed/pnpm-lock.yaml @@ -0,0 +1,580 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +devDependencies: + axios: + specifier: ^1.6.8 + version: 1.6.8 + express: + specifier: ^4.18.3 + version: 4.18.3 + tar: + specifier: ^6.2.0 + version: 6.2.0 + +packages: + + /accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + dev: true + + /array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + dev: true + + /asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: true + + /axios@1.6.8: + resolution: {integrity: sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==} + dependencies: + follow-redirects: 1.15.6 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + dev: true + + /body-parser@1.20.2: + resolution: {integrity: sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.11.0 + raw-body: 2.5.2 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + dev: true + + /bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + dev: true + + /call-bind@1.0.7: + resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} + engines: {node: '>= 0.4'} + dependencies: + es-define-property: 1.0.0 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.2.4 + set-function-length: 1.2.2 + dev: true + + /chownr@2.0.0: + resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} + engines: {node: '>=10'} + dev: true + + /combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + dependencies: + delayed-stream: 1.0.0 + dev: true + + /content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + dependencies: + safe-buffer: 5.2.1 + dev: true + + /content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + dev: true + + /cookie-signature@1.0.6: + resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} + dev: true + + /cookie@0.5.0: + resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} + engines: {node: '>= 0.6'} + dev: true + + /debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.0.0 + dev: true + + /define-data-property@1.1.4: + resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} + engines: {node: '>= 0.4'} + dependencies: + es-define-property: 1.0.0 + es-errors: 1.3.0 + gopd: 1.0.1 + dev: true + + /delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + dev: true + + /depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + dev: true + + /destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + dev: true + + /ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + dev: true + + /encodeurl@1.0.2: + resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} + engines: {node: '>= 0.8'} + dev: true + + /es-define-property@1.0.0: + resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==} + engines: {node: '>= 0.4'} + dependencies: + get-intrinsic: 1.2.4 + dev: true + + /es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + dev: true + + /escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + dev: true + + /etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + dev: true + + /express@4.18.3: + resolution: {integrity: sha512-6VyCijWQ+9O7WuVMTRBTl+cjNNIzD5cY5mQ1WM8r/LEkI2u8EYpOotESNwzNlyCn3g+dmjKYI6BmNneSr/FSRw==} + engines: {node: '>= 0.10.0'} + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.2 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.5.0 + cookie-signature: 1.0.6 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.2.0 + fresh: 0.5.2 + http-errors: 2.0.0 + merge-descriptors: 1.0.1 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.7 + proxy-addr: 2.0.7 + qs: 6.11.0 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.18.0 + serve-static: 1.15.0 + setprototypeof: 1.2.0 + statuses: 2.0.1 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + dev: true + + /finalhandler@1.2.0: + resolution: {integrity: sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==} + engines: {node: '>= 0.8'} + dependencies: + debug: 2.6.9 + encodeurl: 1.0.2 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.1 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + dev: true + + /follow-redirects@1.15.6: + resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: true + + /form-data@4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: true + + /forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + dev: true + + /fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + dev: true + + /fs-minipass@2.1.0: + resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} + engines: {node: '>= 8'} + dependencies: + minipass: 3.3.6 + dev: true + + /function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + dev: true + + /get-intrinsic@1.2.4: + resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} + engines: {node: '>= 0.4'} + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + has-proto: 1.0.3 + has-symbols: 1.0.3 + hasown: 2.0.2 + dev: true + + /gopd@1.0.1: + resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + dependencies: + get-intrinsic: 1.2.4 + dev: true + + /has-property-descriptors@1.0.2: + resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} + dependencies: + es-define-property: 1.0.0 + dev: true + + /has-proto@1.0.3: + resolution: {integrity: sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==} + engines: {node: '>= 0.4'} + dev: true + + /has-symbols@1.0.3: + resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} + engines: {node: '>= 0.4'} + dev: true + + /hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + dependencies: + function-bind: 1.1.2 + dev: true + + /http-errors@2.0.0: + resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} + engines: {node: '>= 0.8'} + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.1 + toidentifier: 1.0.1 + dev: true + + /iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + dependencies: + safer-buffer: 2.1.2 + dev: true + + /inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: true + + /ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + dev: true + + /media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + dev: true + + /merge-descriptors@1.0.1: + resolution: {integrity: sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==} + dev: true + + /methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + dev: true + + /mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + dev: true + + /mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.52.0 + dev: true + + /mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + dev: true + + /minipass@3.3.6: + resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} + engines: {node: '>=8'} + dependencies: + yallist: 4.0.0 + dev: true + + /minipass@5.0.0: + resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} + engines: {node: '>=8'} + dev: true + + /minizlib@2.1.2: + resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} + engines: {node: '>= 8'} + dependencies: + minipass: 3.3.6 + yallist: 4.0.0 + dev: true + + /mkdirp@1.0.4: + resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} + engines: {node: '>=10'} + hasBin: true + dev: true + + /ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + dev: true + + /ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + dev: true + + /negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + dev: true + + /object-inspect@1.13.1: + resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} + dev: true + + /on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + dependencies: + ee-first: 1.1.1 + dev: true + + /parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + dev: true + + /path-to-regexp@0.1.7: + resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} + dev: true + + /proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + dev: true + + /proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: true + + /qs@6.11.0: + resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==} + engines: {node: '>=0.6'} + dependencies: + side-channel: 1.0.6 + dev: true + + /range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + dev: true + + /raw-body@2.5.2: + resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} + engines: {node: '>= 0.8'} + dependencies: + bytes: 3.1.2 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + dev: true + + /safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + dev: true + + /safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + dev: true + + /send@0.18.0: + resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==} + engines: {node: '>= 0.8.0'} + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.0 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.1 + transitivePeerDependencies: + - supports-color + dev: true + + /serve-static@1.15.0: + resolution: {integrity: sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==} + engines: {node: '>= 0.8.0'} + dependencies: + encodeurl: 1.0.2 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.18.0 + transitivePeerDependencies: + - supports-color + dev: true + + /set-function-length@1.2.2: + resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==} + engines: {node: '>= 0.4'} + dependencies: + define-data-property: 1.1.4 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.2.4 + gopd: 1.0.1 + has-property-descriptors: 1.0.2 + dev: true + + /setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + dev: true + + /side-channel@1.0.6: + resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.7 + es-errors: 1.3.0 + get-intrinsic: 1.2.4 + object-inspect: 1.13.1 + dev: true + + /statuses@2.0.1: + resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} + engines: {node: '>= 0.8'} + dev: true + + /tar@6.2.0: + resolution: {integrity: sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==} + engines: {node: '>=10'} + dependencies: + chownr: 2.0.0 + fs-minipass: 2.1.0 + minipass: 5.0.0 + minizlib: 2.1.2 + mkdirp: 1.0.4 + yallist: 4.0.0 + dev: true + + /toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + dev: true + + /type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + dev: true + + /unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + dev: true + + /utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + dev: true + + /vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + dev: true + + /yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: true diff --git a/tasks/benchmark/codspeed/upload.mjs b/tasks/benchmark/codspeed/upload.mjs new file mode 100644 index 000000000..d132c87d1 --- /dev/null +++ b/tasks/benchmark/codspeed/upload.mjs @@ -0,0 +1,135 @@ +/* + * Combine benchmark data from different jobs, and upload to Codspeed. + */ + +import {createReadStream} from 'fs'; +import fs from 'fs/promises'; +import {join as pathJoin} from 'path'; +import {createHash} from 'crypto'; +import assert from 'assert'; +import tar from 'tar'; +import axios from 'axios'; + +const METADATA_SUFFIX = 'metadata.json', + ARCHIVE_SUFFIX = `archive.tar.gz`, + CODSPEED_UPLOAD_URL = 'https://api.codspeed.io/upload'; + +const dataDir = process.env.DATA_DIR, + token = process.env.CODSPEED_TOKEN; + +// Get list of components +const components = (await fs.readdir(dataDir)) + .filter(filename => filename.endsWith(METADATA_SUFFIX)) + .map(filename => filename.slice(0, -METADATA_SUFFIX.length - 1)); + +// Unzip tarballs +const unzipDir = pathJoin(dataDir, 'unzip'); +await fs.mkdir(unzipDir); + +for (const component of components) { + console.log(`Unzipping profile data: ${component}`); + const archivePath = pathJoin(dataDir, `${component}_${ARCHIVE_SUFFIX}`); + const componentUnzipDir = pathJoin(unzipDir, component); + await fs.mkdir(componentUnzipDir); + await tar.extract({file: archivePath, cwd: componentUnzipDir}); + await fs.rm(archivePath); +} + +// Move all `.out` files to one directory +console.log('Combining profiles'); + +const outDir = pathJoin(dataDir, 'out'); +await fs.mkdir(outDir); + +const pids = new Set(), + duplicates = []; +let highestPid = -1; +for (const component of components) { + const componentDir = pathJoin(unzipDir, component); + const outFiles = await fs.readdir(componentDir); + for (const filename of outFiles) { + if (!filename.endsWith('.out')) continue; + let pid = filename.slice(0, -4); + assert(/^\d+$/.test(pid), `Unexpected file: ${component}/${filename}`); + pid *= 1; + + const path = pathJoin(componentDir, filename); + if (pids.has(pid)) { + // Duplicate PID + duplicates.push({pid, path}); + } else { + pids.add(pid); + if (pid > highestPid) highestPid = pid; + + await fs.rename(path, pathJoin(outDir, `${pid}.out`)); + } + } +} + +// Alter PIDs for `.out` files with duplicate filenames +for (let {pid, path} of duplicates) { + let content = await fs.readFile(path, 'utf8'); + + const pidLine = `\npid: ${pid}\n`; + const index = content.indexOf(pidLine); + assert(index !== -1, `Could not locate PID in ${path}`); + const before = content.slice(0, index); + assert(before.split('\n').length === 3, `Unexpected formatting in ${path}`); + + pid = ++highestPid; + content = `${before}\npid: ${pid}\n${content.slice(index + pidLine.length)}`; + + await fs.writeFile(pathJoin(outDir, `${pid}.out`), content); + await fs.rm(path); +} + +// Add log files to output dir +for (const filename of ['runner.log', 'valgrind.log']) { + await fs.rename(pathJoin(unzipDir, components[0], filename), pathJoin(outDir, filename)); +} + +// ZIP combined profile directory +console.log('Zipping combined profile directory'); +const archivePath = pathJoin(dataDir, 'archive.tar.gz'); +await tar.create({file: archivePath, gzip: true, cwd: outDir}, ['./']); + +// Get size + MD5 hash of archive +console.log('Hashing ZIP'); +const {size} = await fs.stat(archivePath); + +const hash = createHash('md5'); +const inputStream = createReadStream(archivePath); +for await (const chunk of inputStream) { + hash.update(chunk); +} +const md5 = hash.digest('base64'); + +// Alter MD5 hash in metadata object +const metadata = JSON.parse( + await fs.readFile(pathJoin(dataDir, `${components[0]}_${METADATA_SUFFIX}`), 'utf8') +); +metadata.profileMd5 = md5; + +// Upload metadata to CodSpeed +console.log('Uploading metadata to CodSpeed'); +const {data} = await axios({ + method: 'post', + url: CODSPEED_UPLOAD_URL, + data: metadata, + headers: {Authorization: token}, +}); +assert(data?.status === 'success', 'Failed to upload metadata to Codspeed'); +const {uploadUrl} = data; + +// Upload profile ZIP to Codspeed +console.log('Uploading profile ZIP to CodSpeed'); +await axios({ + method: 'put', + url: uploadUrl, + data: createReadStream(archivePath), + headers: { + 'Content-Type': 'application/gzip', + 'Content-Length': size, + 'Content-MD5': md5, + } +});