-
-
Notifications
You must be signed in to change notification settings - Fork 484
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(tasks): shard benchmarks in CI (#2751)
This PR shards benchmarks when running on CI. Each benchmark (parser, minifier etc) runs as a separate job, and then a final job combines the results and uploads to Codspeed. A bit of a hacky implementation. Uses a small NodeJS HTTP server to intercept the results from `codspeed-runner`, and then another NodeJS script to combine them all together, and upload to CodSpeed. I will submit PRs on Codspeed's runner + action to do it properly, but as I imagine it'll be a slow process getting that merged upstream, I wanted to see if it worked first. We can replace this once it's supported upstream. Sharding only reduces total time to run the benchmarks by about 70 secs at present, because linter benchmark takes 6 mins alone and holds up the whole process (all the rest are done in ~2 mins). If we can split up the linter benchmark, we can likely get total run time down to around 3 mins. I'll try that in a follow-on PR. I guess the other upside is we can now add as many benchmarks as we like with impunity - they'll run in parallel, and so won't slow things down overall.
- Loading branch information
1 parent
e146b8d
commit 93e1ea4
Showing
7 changed files
with
879 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| /* | ||
| * HTTP server to intercept benchmark data from Codspeed runner. | ||
| * | ||
| * Codspeed runner makes 2 API calls: | ||
| * 1. Uploading metadata | ||
| * 2. Uploading archive of CPU profile files | ||
| * | ||
| * Server starts on an available port, saves the files sent by Codspeed runner to a directory, | ||
| * then shuts itself down. | ||
| */ | ||
|
|
||
| import fs from 'fs'; | ||
| import {pipeline} from 'stream/promises'; | ||
| import express from 'express'; | ||
|
|
||
| const DEFAULT_PORT = 3000, | ||
| LISTEN_ATTEMPTS = 10; | ||
|
|
||
| // Create directory for saving assets | ||
| const rand = Math.round(Math.random() * 1000000000000000000).toString(16), | ||
| dataDir = `/tmp/oxc_bench_data_${rand}`; | ||
| fs.mkdirSync(dataDir); | ||
|
|
||
| const component = process.env.COMPONENT; | ||
|
|
||
| const app = express(); | ||
|
|
||
| app.post('/upload', (req, res, next) => { | ||
| saveBody(req, 'metadata.json', next, () => { | ||
| res.json({ | ||
| status: 'success', | ||
| uploadUrl: `http://localhost:${port}/upload_archive`, | ||
| runId: 'dummy_value', | ||
| }); | ||
| }); | ||
| }); | ||
|
|
||
| app.put('/upload_archive', (req, res, next) => { | ||
| saveBody(req, 'archive.tar.gz', next, () => { | ||
| res.send('OK'); | ||
| server.close(() => {}); | ||
| }); | ||
| }); | ||
|
|
||
| function saveBody(req, filename, onError, done) { | ||
| (async () => { | ||
| const stream = fs.createWriteStream(`${dataDir}/${component}_${filename}`); | ||
| await pipeline(req, stream); | ||
| done(); | ||
| })().catch(onError); | ||
| } | ||
|
|
||
| // Open server on a port which is not already in use | ||
| let server, | ||
| port = DEFAULT_PORT; | ||
| for (let i = 0; i < LISTEN_ATTEMPTS; i++) { | ||
| console.log(`Starting server on port ${port}`); | ||
| try { | ||
| await new Promise((resolve, reject) => { | ||
| server = app.listen(port, resolve); | ||
| server.on('error', reject); | ||
| }); | ||
| break; | ||
| } catch (err) { | ||
| if (err?.code !== 'EADDRINUSE') throw err; | ||
| console.log(`Port ${port} in use. Trying again.`); | ||
| port = DEFAULT_PORT + Math.round(Math.random() * 5000); | ||
| } | ||
| } | ||
| console.log(`Server listening on port ${port}`); | ||
|
|
||
| // Output data dir path + port to env vars | ||
| fs.appendFileSync(process.env.GITHUB_ENV, `DATA_DIR=${dataDir}\nINTERCEPT_PORT=${port}\n`); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| /* | ||
| * Create temp dir for downloading artefacts to. | ||
| */ | ||
|
|
||
| import fs from 'fs'; | ||
|
|
||
| // Create directory for saving assets | ||
| const rand = Math.round(Math.random() * 1000000000000000000).toString(16), | ||
| dataDir = `/tmp/oxc_bench_data_${rand}`; | ||
| fs.mkdirSync(dataDir); | ||
|
|
||
| // Output dir path to env var | ||
| fs.appendFileSync(process.env.GITHUB_ENV, `DATA_DIR=${dataDir}\n`); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| { | ||
| "name": "oxc_benchmark", | ||
| "version": "0.0.0", | ||
| "devDependencies": { | ||
| "axios": "^1.6.8", | ||
| "express": "^4.18.3", | ||
| "tar": "^6.2.0" | ||
| } | ||
| } |
Oops, something went wrong.