Add support for lit

benchmark-action · Jan 19, 2025 · 56b5ee8 · 56b5ee8
1 parent e3c6616
commit 56b5ee8
Show file tree

Hide file tree

Showing 14 changed files with 292 additions and 0 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -404,6 +404,42 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           comment-on-alert: true
       - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark'
+
+  lit-benchmark:
+    name: Run lit benchmark example
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+      - run: npm ci
+      - run: npm run build
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - run: pip install lit
+      - name: Run benchmark
+        run: lit examples/lit --resultdb-output examples/lit/output.json
+      - name: Save previous data.js
+        run: |
+          git fetch origin gh-pages
+          git checkout gh-pages
+          cp ./dev/bench/data.js before_data.js
+          git checkout -
+      - name: Store benchmark result
+        uses: ./
+        with:
+          name: lit benchmark
+          tool: 'lit'
+          output-file-path: examples/lit/output.json
+          skip-fetch-gh-pages: true
+          fail-on-alert: true
+          summary-always: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'lit benchmark'
 
   only-alert-with-cache:
     name: Run alert check with actions/cache

diff --git a/.github/workflows/lit.yml b/.github/workflows/lit.yml
@@ -0,0 +1,52 @@
+name: lit Example
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Run lit benchmark example
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - run: pip install lit
+      - name: Run benchmark
+        run: lit examples/lit --resultdb-output examples/lit/output.json
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: lit benchmark result
+          tool: 'lit'
+          output-file-path: examples/lit/output.json
+          # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@tgymnich'
+
+      - name: Store benchmark result - separate results repo
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: lit benchmark result
+          tool: 'lit'
+          output-file-path: examples/lit/output.json
+          # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
+          github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@tgymnich'
+          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
diff --git a/README.md b/README.md
@@ -26,6 +26,7 @@ This action currently supports the following tools:
 - [Benchmark.Net][benchmarkdotnet] for .Net projects
 - [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects
 - [JMH][jmh] for Java projects
+- [lit][lit] for projects using the LLVM Integrated Tester
 - Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter'
 
 Multiple languages in the same repository are supported for polyglot projects.
@@ -51,6 +52,8 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor
 | .Net         | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) |
 | Java         | [![Java Example Workflow][java-badge]][java-workflow-example] | [examples/java](./examples/java) |
 | Luau         | Coming soon | Coming soon |
+| lit          | [lit Example Workflow][lit-workflow-example]                                            | [examples/lit](./examples/lit)                 |
+
 
 All benchmark charts from above workflows are gathered in GitHub pages:
 
@@ -362,6 +365,7 @@ and store it to file. Then specify the file path to `output-file-path` input.
 - [BenchmarkTools.jl for Julia projects](./examples/julia/README.md)
 - [Benchmark.Net for .Net projects](./examples/benchmarkdotnet/README.md)
 - [benchmarkluau for Luau projects](#) - Examples for this are still a work in progress.
+- [lit](./examples/lit/README.md)
 
 These examples are run in workflows of this repository as described in the 'Examples' section above.
 
@@ -660,6 +664,7 @@ Every release will appear on your GitHub notifications page.
 [catch2-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22
 [julia-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Julia+Example+with+BenchmarkTools.jl%22
 [java-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22JMH+Example%22
+[lit-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22lit+Example%22
 [help-watch-release]: https://docs.github.com/en/github/receiving-notifications-about-activity-on-github/watching-and-unwatching-releases-for-a-repository
 [help-github-token]: https://docs.github.com/en/actions/security-guides/automatic-token-authentication
 [minimal-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Example+for+minimal+setup%22
@@ -673,4 +678,5 @@ Every release will appear on your GitHub notifications page.
 [benchmarkdotnet]: https://benchmarkdotnet.org
 [benchmarkdotnet-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/benchmarkdotnet.yml/badge.svg
 [benchmarkdotnet-workflow-example]: https://github.com/rhysd/github-action-benchmark/actions?query=workflow%3A%22Benchmark.Net+Example%22
+[lit]: https://llvm.org/docs/CommandGuide/lit.html
 [job-summaries]: https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/
diff --git a/examples/lit/README.md b/examples/lit/README.md
@@ -0,0 +1,62 @@
+# lit - LLVM Integrated Tester Example
+
+- [lit docs](https://llvm.org/docs/CommandGuide/lit.html)
+- [Workflow for this example](../../.github/workflows/lit.yml)
+
+This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark)
+with [lit](https://llvm.org/docs/CommandGuide/lit.html).
+
+## Run benchmarks
+
+Install dependencies with `venv` package using Python3.
+
+```sh
+$ python -m venv venv
+$ source venv/bin/activate
+$ pip install lit
+```
+
+Setup your test suite. In this example it consists of a configuration file for lit `lit.cfg` and two tests `a.txt` and `b.txt` containing one RUN line each.
+
+e.g
+
+```python
+import lit.formats
+
+config.name = "time-tests"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+```
+
+```
+# RUN: true
+```
+
+```
+# RUN: sleep 1
+```
+
+And run the test suite with `--resultdb-output` in workflow. The JSON file will be an input to github-action-benchmark.
+
+e.g.
+
+```yaml
+- name: Run benchmark
+  run: lit examples/lit --resultdb-output output.json
+```
+
+## Process benchmark results
+
+Store the benchmark results with step using the action. Please set `tool` to `lit` input and pass the path to the output file.
+
+```yaml
+- name: Store benchmark result
+  uses: benchmark-action/github-action-benchmark@v1
+  with:
+    tool: 'lit'
+    output-file-path: output.json
+```
+
+Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage.
diff --git a/examples/lit/a.txt b/examples/lit/a.txt
@@ -0,0 +1 @@
+# RUN: true
diff --git a/examples/lit/b.txt b/examples/lit/b.txt
@@ -0,0 +1 @@
+# RUN: sleep 1
diff --git a/examples/lit/lit.cfg b/examples/lit/lit.cfg
@@ -0,0 +1,9 @@
+# -*- Python -*-
+
+import lit.formats
+
+config.name = "time-tests"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/src/config.ts b/src/config.ts
@@ -40,6 +40,7 @@ export const VALID_TOOLS = [
     'benchmarkdotnet',
     'customBiggerIsBetter',
     'customSmallerIsBetter',
+    'lit',
 ] as const;
 const RE_UINT = /^\d+$/;
 

diff --git a/src/default_index_html.ts b/src/default_index_html.ts
@@ -123,6 +123,7 @@ export const DEFAULT_INDEX_HTML = String.raw`<!DOCTYPE html>
           benchmarkdotnet: '#178600',
           customBiggerIsBetter: '#38ff38',
           customSmallerIsBetter: '#ff3838',
+          lit: '#185619',
           _: '#333333'
         };
 

diff --git a/src/extract.ts b/src/extract.ts
@@ -222,6 +222,24 @@ export interface BenchmarkDotNetBenchmarkJson {
     Benchmarks: BenchmarkDotnetBenchmark[];
 }
 
+export interface LitBenchmarkJson {
+    __version__: number[];
+    elapsed: number;
+    tests: Array<{
+        artifacts: {
+            artifact_content_in_request: {
+                contents: string;
+            };
+        };
+        duration: string;
+        expected: boolean;
+        start_time: string;
+        status: string;
+        summary_html: string;
+        testId: string;
+    }>;
+}
+
 function getHumanReadableUnitValue(seconds: number): [number, string] {
     if (seconds < 1.0e-6) {
         return [seconds * 1e9, 'nsec'];
@@ -690,6 +708,28 @@ function extractLuauBenchmarkResult(output: string): BenchmarkResult[] {
     return results;
 }
 
+function extractLitBenchmarkResult(output: string): BenchmarkResult[] {
+    let json: LitBenchmarkJson;
+    try {
+        json = JSON.parse(output);
+    } catch (err: any) {
+        throw new Error(
+            `Output file for 'lit' must be JSON file generated by --resultdb-output option: ${err.message}`,
+        );
+    }
+    return json.tests.map((b) => {
+        const name = b.testId;
+        const value = parseFloat(b.duration.substring(0, b.duration.length - 2));
+        const unit = 's';
+        const extra = `
+        expected: ${b.expected}
+        start_time: ${b.start_time}
+        status: ${b.status}
+        summary_html: ${b.summary_html}`;
+        return { name, value, unit, extra };
+    });
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool, githubToken, ref } = config;
@@ -732,6 +772,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
         case 'benchmarkluau':
             benches = extractLuauBenchmarkResult(output);
             break;
+        case 'lit':
+            benches = extractLitBenchmarkResult(output);
+            break;
         default:
             throw new Error(`FATAL: Unexpected tool: '${tool}'`);
     }

diff --git a/src/write.ts b/src/write.ts
@@ -81,6 +81,8 @@ function biggerIsBetter(tool: ToolType): boolean {
             return false;
         case 'benchmarkdotnet':
             return false;
+        case 'lit':
+            return false;
         case 'customBiggerIsBetter':
             return true;
         case 'customSmallerIsBetter':

diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap
@@ -887,6 +887,43 @@ params={"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample
 }
 `;
 
+exports[`extractResult() extracts benchmark output from lit - lit_output.json 1`] = `
+{
+  "benches": [
+    {
+      "extra": "
+        expected: true
+        start_time: 2025-01-19T20:13:43.882058Z
+        status: PASS
+        summary_html: <p><text-artifact artifact-id="artifact-content-in-request"></p>",
+      "name": "time-tests :: b.txt",
+      "unit": "s",
+      "value": 1.0095539,
+    },
+    {
+      "extra": "
+        expected: true
+        start_time: 2025-01-19T20:13:43.882072Z
+        status: PASS
+        summary_html: <p><text-artifact artifact-id="artifact-content-in-request"></p>",
+      "name": "time-tests :: a.txt",
+      "unit": "s",
+      "value": 0.00366711,
+    },
+  ],
+  "commit": {
+    "author": null,
+    "committer": null,
+    "id": "123456789abcdef",
+    "message": "this is dummy",
+    "timestamp": "dummy timestamp",
+    "url": "https://github.com/dummy/repo",
+  },
+  "date": 1712131503296,
+  "tool": "lit",
+}
+`;
+
 exports[`extractResult() extracts benchmark output from pytest - pytest_output.json 1`] = `
 {
   "benches": [

diff --git a/test/data/extract/lit_output.json b/test/data/extract/lit_output.json
@@ -0,0 +1,37 @@
+{
+    "__version__": [
+      18,
+      1,
+      8
+    ],
+    "elapsed": 1.0956830978393555,
+    "tests": [
+      {
+        "artifacts": {
+          "artifact-content-in-request": {
+            "contents": "RXhpdCBDb2RlOiAwCgpDb21tYW5kIE91dHB1dCAoc3Rkb3V0KToKLS0KIyBSVU46IGF0IGxpbmUgMQpzbGVlcCAxCiMgZXhlY3V0ZWQgY29tbWFuZDogc2xlZXAgMQoKLS0K"
+          }
+        },
+        "duration": "1.009553909s",
+        "expected": true,
+        "start_time": "2025-01-19T20:13:43.882058Z",
+        "status": "PASS",
+        "summary_html": "<p><text-artifact artifact-id=\"artifact-content-in-request\"></p>",
+        "testId": "time-tests :: b.txt"
+      },
+      {
+        "artifacts": {
+          "artifact-content-in-request": {
+            "contents": "RXhpdCBDb2RlOiAwCgpDb21tYW5kIE91dHB1dCAoc3Rkb3V0KToKLS0KIyBSVU46IGF0IGxpbmUgMQp0cnVlCiMgZXhlY3V0ZWQgY29tbWFuZDogdHJ1ZQoKLS0K"
+          }
+        },
+        "duration": "0.003667116s",
+        "expected": true,
+        "start_time": "2025-01-19T20:13:43.882072Z",
+        "status": "PASS",
+        "summary_html": "<p><text-artifact artifact-id=\"artifact-content-in-request\"></p>",
+        "testId": "time-tests :: a.txt"
+      }
+    ]
+  }
+
diff --git a/test/extract.spec.ts b/test/extract.spec.ts
@@ -135,6 +135,10 @@ describe('extractResult()', function () {
             tool: 'benchmarkdotnet',
             file: 'benchmarkdotnet.json',
         },
+        {
+            tool: 'lit',
+            file: 'lit_output.json',
+        },
         {
             tool: 'customBiggerIsBetter',
             file: 'customBiggerIsBetter_output.json',