diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 274cf633b..b8716cf49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -404,6 +404,42 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' + + lit-benchmark: + name: Run lit benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: pip install lit + - name: Run benchmark + run: lit examples/lit --resultdb-output examples/lit/output.json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: lit benchmark + tool: 'lit' + output-file-path: examples/lit/output.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'lit benchmark' only-alert-with-cache: name: Run alert check with actions/cache diff --git a/.github/workflows/lit.yml b/.github/workflows/lit.yml new file mode 100644 index 000000000..9455a6e96 --- /dev/null +++ b/.github/workflows/lit.yml @@ -0,0 +1,52 @@ +name: lit Example +on: + push: + branches: + - master + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Run lit benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: pip install lit + - name: Run benchmark + run: lit examples/lit --resultdb-output examples/lit/output.json + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: lit benchmark result + tool: 'lit' + output-file-path: examples/lit/output.json + # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@tgymnich' + + - name: Store benchmark result - separate results repo + uses: benchmark-action/github-action-benchmark@v1 + with: + name: lit benchmark result + tool: 'lit' + output-file-path: examples/lit/output.json + # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096 + github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@tgymnich' + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/README.md b/README.md index 7dacb6c4d..1cedeb59b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ This action currently supports the following tools: - [Benchmark.Net][benchmarkdotnet] for .Net projects - [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects - [JMH][jmh] for Java projects +- [lit][lit] for projects using the LLVM Integrated Tester - Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter' Multiple languages in the same repository are supported for polyglot projects. @@ -51,6 +52,8 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor | .Net | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) | | Java | [![Java Example Workflow][java-badge]][java-workflow-example] | [examples/java](./examples/java) | | Luau | Coming soon | Coming soon | +| lit | [lit Example Workflow][lit-workflow-example] | [examples/lit](./examples/lit) | + All benchmark charts from above workflows are gathered in GitHub pages: @@ -362,6 +365,7 @@ and store it to file. Then specify the file path to `output-file-path` input. - [BenchmarkTools.jl for Julia projects](./examples/julia/README.md) - [Benchmark.Net for .Net projects](./examples/benchmarkdotnet/README.md) - [benchmarkluau for Luau projects](#) - Examples for this are still a work in progress. +- [lit](./examples/lit/README.md) These examples are run in workflows of this repository as described in the 'Examples' section above. @@ -660,6 +664,7 @@ Every release will appear on your GitHub notifications page. [catch2-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22 [julia-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Julia+Example+with+BenchmarkTools.jl%22 [java-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22JMH+Example%22 +[lit-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22lit+Example%22 [help-watch-release]: https://docs.github.com/en/github/receiving-notifications-about-activity-on-github/watching-and-unwatching-releases-for-a-repository [help-github-token]: https://docs.github.com/en/actions/security-guides/automatic-token-authentication [minimal-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Example+for+minimal+setup%22 @@ -673,4 +678,5 @@ Every release will appear on your GitHub notifications page. [benchmarkdotnet]: https://benchmarkdotnet.org [benchmarkdotnet-badge]: https://github.com/benchmark-action/github-action-benchmark/actions/workflows/benchmarkdotnet.yml/badge.svg [benchmarkdotnet-workflow-example]: https://github.com/rhysd/github-action-benchmark/actions?query=workflow%3A%22Benchmark.Net+Example%22 +[lit]: https://llvm.org/docs/CommandGuide/lit.html [job-summaries]: https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/ diff --git a/examples/lit/README.md b/examples/lit/README.md new file mode 100644 index 000000000..92ca3f375 --- /dev/null +++ b/examples/lit/README.md @@ -0,0 +1,62 @@ +# lit - LLVM Integrated Tester Example + +- [lit docs](https://llvm.org/docs/CommandGuide/lit.html) +- [Workflow for this example](../../.github/workflows/lit.yml) + +This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) +with [lit](https://llvm.org/docs/CommandGuide/lit.html). + +## Run benchmarks + +Install dependencies with `venv` package using Python3. + +```sh +$ python -m venv venv +$ source venv/bin/activate +$ pip install lit +``` + +Setup your test suite. In this example it consists of a configuration file for lit `lit.cfg` and two tests `a.txt` and `b.txt` containing one RUN line each. + +e.g + +```python +import lit.formats + +config.name = "time-tests" +config.suffixes = [".txt"] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None +``` + +``` +# RUN: true +``` + +``` +# RUN: sleep 1 +``` + +And run the test suite with `--resultdb-output` in workflow. The JSON file will be an input to github-action-benchmark. + +e.g. + +```yaml +- name: Run benchmark + run: lit examples/lit --resultdb-output output.json +``` + +## Process benchmark results + +Store the benchmark results with step using the action. Please set `tool` to `lit` input and pass the path to the output file. + +```yaml +- name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'lit' + output-file-path: output.json +``` + +Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. \ No newline at end of file diff --git a/examples/lit/a.txt b/examples/lit/a.txt new file mode 100644 index 000000000..c6b9f3b8c --- /dev/null +++ b/examples/lit/a.txt @@ -0,0 +1 @@ +# RUN: true \ No newline at end of file diff --git a/examples/lit/b.txt b/examples/lit/b.txt new file mode 100644 index 000000000..1b381ff95 --- /dev/null +++ b/examples/lit/b.txt @@ -0,0 +1 @@ +# RUN: sleep 1 \ No newline at end of file diff --git a/examples/lit/lit.cfg b/examples/lit/lit.cfg new file mode 100644 index 000000000..7ba99c345 --- /dev/null +++ b/examples/lit/lit.cfg @@ -0,0 +1,9 @@ +# -*- Python -*- + +import lit.formats + +config.name = "time-tests" +config.suffixes = [".txt"] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 0ce3bd5c0..0be639a03 100644 --- a/src/config.ts +++ b/src/config.ts @@ -40,6 +40,7 @@ export const VALID_TOOLS = [ 'benchmarkdotnet', 'customBiggerIsBetter', 'customSmallerIsBetter', + 'lit', ] as const; const RE_UINT = /^\d+$/; diff --git a/src/default_index_html.ts b/src/default_index_html.ts index a6246d51b..6207d6934 100644 --- a/src/default_index_html.ts +++ b/src/default_index_html.ts @@ -123,6 +123,7 @@ export const DEFAULT_INDEX_HTML = String.raw` benchmarkdotnet: '#178600', customBiggerIsBetter: '#38ff38', customSmallerIsBetter: '#ff3838', + lit: '#185619', _: '#333333' }; diff --git a/src/extract.ts b/src/extract.ts index ec7bc35e2..9db911567 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -222,6 +222,24 @@ export interface BenchmarkDotNetBenchmarkJson { Benchmarks: BenchmarkDotnetBenchmark[]; } +export interface LitBenchmarkJson { + __version__: number[]; + elapsed: number; + tests: Array<{ + artifacts: { + artifact_content_in_request: { + contents: string; + }; + }; + duration: string; + expected: boolean; + start_time: string; + status: string; + summary_html: string; + testId: string; + }>; +} + function getHumanReadableUnitValue(seconds: number): [number, string] { if (seconds < 1.0e-6) { return [seconds * 1e9, 'nsec']; @@ -690,6 +708,28 @@ function extractLuauBenchmarkResult(output: string): BenchmarkResult[] { return results; } +function extractLitBenchmarkResult(output: string): BenchmarkResult[] { + let json: LitBenchmarkJson; + try { + json = JSON.parse(output); + } catch (err: any) { + throw new Error( + `Output file for 'lit' must be JSON file generated by --resultdb-output option: ${err.message}`, + ); + } + return json.tests.map((b) => { + const name = b.testId; + const value = parseFloat(b.duration.substring(0, b.duration.length - 2)); + const unit = 's'; + const extra = ` + expected: ${b.expected} + start_time: ${b.start_time} + status: ${b.status} + summary_html: ${b.summary_html}`; + return { name, value, unit, extra }; + }); +} + export async function extractResult(config: Config): Promise { const output = await fs.readFile(config.outputFilePath, 'utf8'); const { tool, githubToken, ref } = config; @@ -732,6 +772,9 @@ export async function extractResult(config: Config): Promise { case 'benchmarkluau': benches = extractLuauBenchmarkResult(output); break; + case 'lit': + benches = extractLitBenchmarkResult(output); + break; default: throw new Error(`FATAL: Unexpected tool: '${tool}'`); } diff --git a/src/write.ts b/src/write.ts index 27e13798e..c9bf380a0 100644 --- a/src/write.ts +++ b/src/write.ts @@ -81,6 +81,8 @@ function biggerIsBetter(tool: ToolType): boolean { return false; case 'benchmarkdotnet': return false; + case 'lit': + return false; case 'customBiggerIsBetter': return true; case 'customSmallerIsBetter': diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap index 76d1e91a9..a8c0c2510 100644 --- a/test/__snapshots__/extract.spec.ts.snap +++ b/test/__snapshots__/extract.spec.ts.snap @@ -887,6 +887,43 @@ params={"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample } `; +exports[`extractResult() extracts benchmark output from lit - lit_output.json 1`] = ` +{ + "benches": [ + { + "extra": " + expected: true + start_time: 2025-01-19T20:13:43.882058Z + status: PASS + summary_html:

", + "name": "time-tests :: b.txt", + "unit": "s", + "value": 1.0095539, + }, + { + "extra": " + expected: true + start_time: 2025-01-19T20:13:43.882072Z + status: PASS + summary_html:

", + "name": "time-tests :: a.txt", + "unit": "s", + "value": 0.00366711, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "lit", +} +`; + exports[`extractResult() extracts benchmark output from pytest - pytest_output.json 1`] = ` { "benches": [ diff --git a/test/data/extract/lit_output.json b/test/data/extract/lit_output.json new file mode 100644 index 000000000..897c7bd35 --- /dev/null +++ b/test/data/extract/lit_output.json @@ -0,0 +1,37 @@ +{ + "__version__": [ + 18, + 1, + 8 + ], + "elapsed": 1.0956830978393555, + "tests": [ + { + "artifacts": { + "artifact-content-in-request": { + "contents": "RXhpdCBDb2RlOiAwCgpDb21tYW5kIE91dHB1dCAoc3Rkb3V0KToKLS0KIyBSVU46IGF0IGxpbmUgMQpzbGVlcCAxCiMgZXhlY3V0ZWQgY29tbWFuZDogc2xlZXAgMQoKLS0K" + } + }, + "duration": "1.009553909s", + "expected": true, + "start_time": "2025-01-19T20:13:43.882058Z", + "status": "PASS", + "summary_html": "

", + "testId": "time-tests :: b.txt" + }, + { + "artifacts": { + "artifact-content-in-request": { + "contents": "RXhpdCBDb2RlOiAwCgpDb21tYW5kIE91dHB1dCAoc3Rkb3V0KToKLS0KIyBSVU46IGF0IGxpbmUgMQp0cnVlCiMgZXhlY3V0ZWQgY29tbWFuZDogdHJ1ZQoKLS0K" + } + }, + "duration": "0.003667116s", + "expected": true, + "start_time": "2025-01-19T20:13:43.882072Z", + "status": "PASS", + "summary_html": "

", + "testId": "time-tests :: a.txt" + } + ] + } + \ No newline at end of file diff --git a/test/extract.spec.ts b/test/extract.spec.ts index 48a5c271a..e1819f23b 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -135,6 +135,10 @@ describe('extractResult()', function () { tool: 'benchmarkdotnet', file: 'benchmarkdotnet.json', }, + { + tool: 'lit', + file: 'lit_output.json', + }, { tool: 'customBiggerIsBetter', file: 'customBiggerIsBetter_output.json',