benchmarks.yml

# This CI configuration for relative benchmarks is based on the research done
# for scikit-image's implementation available here:
# https://github.com/scikit-image/scikit-image/blob/9bdd010a8/.github/workflows/benchmarks.yml#L1
# Blog post with the rationale: https://labs.quansight.org/blog/2021/08/github-actions-benchmarks/

name: Benchmarks

on:
  pull_request:
    types: [labeled]
  schedule:
    - cron: "6 6 * * 0" # every sunday
  workflow_dispatch:
    inputs:
      base_ref:
        description: "Baseline commit or git reference"
        required: true
      contender_ref:
        description: "Contender commit or git reference"
        required: true

# This is the main configuration section that needs to be fine tuned to napari's needs
# All the *_THREADS options is just to make the benchmarks more robust by not using parallelism
env:
  OPENBLAS_NUM_THREADS: "1"
  MKL_NUM_THREADS: "1"
  OMP_NUM_THREADS: "1"
  ASV_OPTIONS: "--split --show-stderr --factor 1.5 --attribute timeout=900"
  # --split -> split final reports in tables
  # --show-stderr -> print tracebacks if errors occur
  # --factor 1.5 -> report anomaly if tested timings are beyond 1.5x base timings
  # --attribute timeout=300 -> override timeout attribute (default=60s) to allow slow tests to run
  # see https://asv.readthedocs.io/en/stable/commands.html#asv-continuous for more details!

jobs:
  benchmark:
    if: ${{ github.event.label.name == 'run-benchmarks' && github.event_name == 'pull_request' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
    name: ${{ matrix.benchmark-name }}
    runs-on: ${{ matrix.runs-on }}
    permissions:
      contents: read
      issues: write
    strategy:
      fail-fast: false
      matrix:
        include:
          - benchmark-name: Qt
            asv-command: continuous
            selection-regex: "^benchmark_qt_.*"
            runs-on: macos-latest
            # Qt tests run on macOS to avoid using Xvfb business
            # xvfb makes everything run, but some tests segfault :shrug:
            # Fortunately, macOS graphics stack does not need xvfb!
          - benchmark-name: non-Qt
            asv-command: continuous
            selection-regex: "^benchmark_(?!qt_).*"
            runs-on: ubuntu-latest

    steps:
      # We need the full repo to avoid this issue
      # https://github.com/actions/checkout/issues/23
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        name: Install Python
        with:
          python-version: "3.11"
          cache-dependency-path: pyproject.toml

      - uses: tlambert03/setup-qt-libs@v1

      - name: Setup asv
        run: python -m pip install asv[virtualenv]
        env:
          PIP_CONSTRAINT: resources/constraints/benchmark.txt

      - uses: octokit/request-action@v2.x
        id: latest_release
        with:
          route: GET /repos/{owner}/{repo}/releases/latest
          owner: napari
          repo: napari
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Run ${{ matrix.benchmark-name }} benchmarks
        id: run_benchmark
        env:
          # asv will checkout commits, which might contain LFS artifacts; ignore those errors since
          # they are probably just documentation PNGs not needed here anyway
          GIT_LFS_SKIP_SMUDGE: 1
          PIP_CONSTRAINT: ${{ github.workspace }}/resources/constraints/benchmark.txt
        run: |
          set -euxo pipefail

          # ID this runner
          asv machine --yes

          if [[ $GITHUB_EVENT_NAME == pull_request ]]; then
            EVENT_NAME="PR #${{ github.event.pull_request.number }}"
            BASE_REF=${{ github.event.pull_request.base.sha }}
            CONTENDER_REF=${GITHUB_SHA}
            echo "Baseline:  ${BASE_REF} (${{ github.event.pull_request.base.label }})"
            echo "Contender: ${CONTENDER_REF} (${{ github.event.pull_request.head.label }})"
          elif [[ $GITHUB_EVENT_NAME == schedule ]]; then
            EVENT_NAME="cronjob"
            BASE_REF="${{ fromJSON(steps.latest_release.outputs.data).target_commitish }}"
            CONTENDER_REF="${GITHUB_SHA}"
            echo "Baseline:  ${BASE_REF} (${{ fromJSON(steps.latest_release.outputs.data).tag_name }})"
            echo "Contender: ${CONTENDER_REF} (current main)"
          elif [[ $GITHUB_EVENT_NAME == workflow_dispatch ]]; then
            EVENT_NAME="manual trigger"
            BASE_REF="${{ github.event.inputs.base_ref }}"
            CONTENDER_REF="${{ github.event.inputs.contender_ref }}"
            echo "Baseline:  ${BASE_REF} (workflow input)"
            echo "Contender: ${CONTENDER_REF} (workflow input)"
          fi

          echo "EVENT_NAME=$EVENT_NAME" >> $GITHUB_ENV
          echo "BASE_REF=$BASE_REF" >> $GITHUB_ENV
          echo "CONTENDER_REF=$CONTENDER_REF" >> $GITHUB_ENV

          # Run benchmarks for current commit against base
          asv continuous $ASV_OPTIONS -b '${{ matrix.selection-regex }}' ${BASE_REF} ${CONTENDER_REF} \
          | sed -E "/Traceback | failed$|PERFORMANCE DECREASED/ s/^/::error:: /" \
          | tee asv_continuous.log

          # Report and export results for subsequent steps
          if grep "Traceback \|failed\|PERFORMANCE DECREASED" asv_continuous.log > /dev/null ; then
              exit 1
          fi

      - name: Report Failures as Issue
        if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && failure() }}
        uses: JasonEtco/create-an-issue@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PLATFORM: ${{ matrix.runs-on }}
          PYTHON: "3.9"
          BACKEND: ${{ matrix.benchmark-name }}
          RUN_ID: ${{ github.run_id }}
          TITLE: "[test-bot] Benchmark tests failing"
        with:
          filename: .github/TEST_FAIL_TEMPLATE.md
          update_existing: true

      - name: Add more info to artifact
        if: always()
        run: |
          # Copy the full `asv continuous` log
          cp asv_continuous.log .asv/results/asv_continuous_${{ matrix.benchmark-name }}.log
          # ensure that even if this isn't a PR, the benchmark_report workflow can run without error
          touch .asv/results/message_${{ matrix.benchmark-name }}.txt

          # Add the message that might be posted as a comment on the PR
          # We delegate the actual comment to `benchmarks_report.yml` due to
          # potential token permissions issues
          if [[ $GITHUB_EVENT_NAME == pull_request ]]; then

          echo "${{ github.event.pull_request.number }}" > .asv/results/pr_number
          echo \
          "The ${{ matrix.benchmark-name }} benchmark run requested by $EVENT_NAME ($CONTENDER_REF vs $BASE_REF) has" \
          "finished with status '${{ steps.run_benchmark.outcome }}'. See the" \
          "[CI logs and artifacts](||BENCHMARK_CI_LOGS_URL||) for further details." \
          > .asv/results/message_${{ matrix.benchmark-name }}.txt

          fi

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: asv-benchmark-results-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.benchmark-name }}
          path: .asv/results

  combine-artifacts:
    runs-on: ubuntu-latest
    needs: benchmark
    if: always()
    steps:
      - name: Download artifact
        uses: actions/download-artifact@v4
        with:
          pattern: asv-benchmark-results*
          path: asv_result
          merge-multiple: true
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: asv-benchmark-results-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
          path: asv_result