feat(tasks): shard linter benchmarks in CI

.github/workflows/benchmark.yml

@@ -29,7 +29,20 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        component: [lexer, parser, transformer, semantic, linter, minifier, codegen_sourcemap]
+        # Run each benchmark in own job.
+        # Linter benchmark is by far the slowest, so split each fixture into own job.
+        component: [lexer, parser, transformer, semantic, minifier, codegen_sourcemap]
+        include:
+          - component: linter
+            fixture: 0
+          - component: linter
+            fixture: 1
+          - component: linter
+            fixture: 2
+          - component: linter
+            fixture: 3
+          - component: linter
+            fixture: 4
     steps:
       - name: Checkout Branch
         uses: actions/checkout@v4
@@ -58,6 +71,7 @@ jobs:
         working-directory: ./tasks/benchmark/codspeed
         env:
           COMPONENT: ${{ matrix.component }}
+          FIXTURE: ${{ matrix.fixture }}
         run: |
           corepack enable
           pnpm install
@@ -76,6 +90,8 @@ jobs:
       - name: Run benchmark
         uses: CodSpeedHQ/action@v2
         timeout-minutes: 30
+        env:
+          FIXTURE: ${{ matrix.fixture }}
         with:
           run: cargo codspeed run
           token: ${{ secrets.CODSPEED_TOKEN }}
@@ -84,7 +100,7 @@ jobs:
       - name: Upload bench data artefact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.component }}
+          name: ${{ matrix.component }}${{ matrix.fixture }}
           path: ${{ env.DATA_DIR }}
           if-no-files-found: error
           retention-days: 1

tasks/benchmark/benches/linter.rs

@@ -1,4 +1,4 @@
-use std::{path::PathBuf, rc::Rc};
+use std::{env, path::PathBuf, rc::Rc};
 
 use oxc_allocator::Allocator;
 use oxc_benchmark::{criterion_group, criterion_main, BenchmarkId, Criterion};
@@ -10,7 +10,16 @@ use oxc_tasks_common::TestFiles;
 
 fn bench_linter(criterion: &mut Criterion) {
     let mut group = criterion.benchmark_group("linter");
-    for file in TestFiles::complicated().files() {
+
+    // If `FIXTURE` env is set, only run the specified benchmark. This is used for sharding in CI.
+    let test_files = if let Ok(fixture_index) = env::var("FIXTURE") {
+        let fixture_index = fixture_index.parse::<usize>().unwrap();
+        TestFiles::complicated_one(fixture_index)
+    } else {
+        TestFiles::complicated()
+    };
+
+    for file in test_files.files() {
         let source_type = SourceType::from_path(&file.file_name).unwrap();
         group.bench_with_input(
             BenchmarkId::from_parameter(&file.file_name),

tasks/benchmark/codspeed/capture.mjs

@@ -21,7 +21,8 @@ const rand = Math.round(Math.random() * 1000000000000000000).toString(16),
     dataDir = `/tmp/oxc_bench_data_${rand}`;
 fs.mkdirSync(dataDir);
 
-const component = process.env.COMPONENT;
+let component = process.env.COMPONENT;
+if (process.env.FIXTURE) component += process.env.FIXTURE;
 
 const app = express();
 

tasks/common/src/test_file.rs

@@ -42,7 +42,18 @@ impl TestFiles {
     }
 
     pub fn complicated() -> Self {
-        let files = [
+        let files = Self::complicated_urls().into_iter().map(TestFile::new).collect();
+        Self { files }
+    }
+
+    pub fn complicated_one(index: usize) -> Self {
+        let url = Self::complicated_urls()[index];
+        let file = TestFile::new(url);
+        Self { files: vec![file] }
+    }
+
+    fn complicated_urls() -> [&'static str; 5] {
+        [
             // TypeScript syntax (2.81MB)
             "https://raw.githubusercontent.com/microsoft/TypeScript/v5.3.3/src/compiler/checker.ts",
             // Real world app tsx (1.0M)
@@ -54,10 +65,6 @@ impl TestFiles {
             // ES5 (3.9M)
             "https://cdn.jsdelivr.net/npm/antd@5.12.5/dist/antd.js",
         ]
-        .into_iter()
-        .map(TestFile::new)
-        .collect();
-        Self { files }
     }
 }