diff --git a/.github/workflows/longitudinal-benchmark.yml b/.github/workflows/longitudinal-benchmark.yml new file mode 100644 index 00000000000..c1946951149 --- /dev/null +++ b/.github/workflows/longitudinal-benchmark.yml @@ -0,0 +1,75 @@ +# Longitudinal Benchmarking +# +# This workflow will run the benchmarks defined in the environment variable BENCHMARKS. +# It will collect and aggreate the benchmark output, format it and feed it to github-action-benchmark. +# +# The benchmark charts are live at https://input-output-hk.github.io/plutus/dev/bench +# The benchmark data is available at https://input-output-hk.github.io/plutus/dev/bench/data.js + +name: Longitudinal Benchmarking + +on: + push: + branches: + - master + +permissions: + # Deployments permission to deploy GitHub pages website + deployments: write + # Contents permission to update benchmark contents in gh-pages branch + contents: write + +jobs: + new-benchmark: + name: Performance regression check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3.3.0 + + - name: Run benchmarks + env: + BENCHMARKS: "validation validation-decode" + run: | + for bench in $BENCHMARKS; do + 2>&1 cabal run "$bench" | tee "$bench-output.txt" + done + + read -r -d '' PYTHON_SCRIPT <<- END_SCRIPT + import json + result = [] + for benchmark in "$BENCHMARKS".split(): + with open(f"{benchmark}-output.txt", "r") as file: + name = "" + for line in file.readlines(): + if line.startswith("benchmarking"): + name = line.split()[1] + elif line.startswith("mean"): + parts = line.split() + mean = parts[1] + unit = parts[2] + result.append({ + "name": f"{benchmark}-{name}", + "unit": unit, + "value": float(mean) + }) + with open("output.json", "w") as file: + json.dump(result, file) + END_SCRIPT + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1.16.2 + with: + name: My Project Go Benchmark + tool: 'customSmallerIsBetter' + output-file-path: output.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy GitHub pages branch automatically + auto-push: true + # Enable alert commit comment + comment-on-alert: true + # Mention @input-output-hk/plutus-core in the commit comment + alert-comment-cc-users: '@input-output-hk/plutus-core' + # Percentage value like "110%". + # It is a ratio indicating how worse the current benchmark result is. + # For example, if we now get 110 ns/iter and previously got 100 ns/iter, it gets 110% worse. + alert-threshold: '105%' diff --git a/.github/workflows/new-benchmark.yml b/.github/workflows/new-benchmark.yml deleted file mode 100644 index f54c356f038..00000000000 --- a/.github/workflows/new-benchmark.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Longitudinal Benchmarking - -on: - push: - branches: - - master - -permissions: - # Deployments permission to deploy GitHub pages website - deployments: write - # Contents permission to update benchmark contents in gh-pages branch - contents: write - -jobs: - new-benchmark: - name: Performance regression check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3.3.0 - - - name: Run benchmark - run: | - # TODO run the actual benchmarks - cat << EOF> ./output.json - [ - { - "name": "My Custom Smaller Is Better Benchmark - CPU Load", - "unit": "Percent", - "value": 8 - }, - { - "name": "My Custom Smaller Is Better Benchmark - Memory Used", - "unit": "Megabytes", - "value": 5, - "range": "3", - "extra": "Value for Tooltip: 25\nOptional Num #2: 100\nAnything Else!" - } - ] - EOF - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1.16.2 - with: - name: My Project Go Benchmark - tool: 'customSmallerIsBetter' - output-file-path: output.json - github-token: ${{ secrets.GITHUB_TOKEN }} - # Push and deploy GitHub pages branch automatically - auto-push: true - # Enable alert commit comment - comment-on-alert: true - # Mention @input-output-hk/plutus-core in the commit comment - alert-comment-cc-users: '@input-output-hk/plutus-core' - # Percentage value like "110%". - # It is a ratio indicating how worse the current benchmark result is. - # For example, if we now get 110 ns/iter and previously got 100 ns/iter, it gets 110% worse. - alert-threshold: '105%'