run_benchmarks.yml

name: Run Benchmarks

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - '*'

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
  cancel-in-progress: true

env:
  VCPKG_DEFAULT_BINARY_CACHE: ${{github.workspace}}/vcpkg_binary_cache

jobs:
  benchmark:
    runs-on: ubuntu-latest
    permissions:
      contents: write
      deployments: write
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: recursive
      - uses: lukka/get-cmake@latest
      - run: echo "VCPKG_COMMIT=$(git rev-parse :ext_libs/vcpkg)" >> $GITHUB_ENV
        shell: bash
      - run: mkdir -p ${{ env.VCPKG_DEFAULT_BINARY_CACHE }}
      # This cache will probably be missing the benchmark dep, but that will be quick to fetch
      - uses: actions/cache@v3
        env:
          cache-name: vcpkg-cache
        with:
          path: ${{ env.VCPKG_DEFAULT_BINARY_CACHE }}/*
          key: ${{ matrix.os }}-build-${{ env.cache-name }}-${{ hashFiles('vcpkg.json') }}-${{ env.VCPKG_COMMIT }}"
      - name: Configure
        run: cmake --preset vcpkg-release -DBUILD_BENCHMARKS=On
      - name: Compile
        run: cmake --build build --target vw-benchmarks.out
      - name: Run benchmarks
        run: >
          ./build/test/benchmarks/vw-benchmarks.out
          --benchmark_min_time=3
          --benchmark_format=json
          --benchmark_out_format=json
          --benchmark_out=benchmark_results.json
      - uses: benchmark-action/github-action-benchmark@v1
        with:
          tool: 'googlecpp'
          output-file-path: benchmark_results.json
          alert-threshold: '150%'
          fail-on-alert: true
          comment-on-alert: true
          auto-push: false
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Push results to gh-pages branch
        if: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
        run: git push 'https://VowpalWabbit:${{ secrets.GITHUB_TOKEN }}@github.com/VowpalWabbit/vowpal_wabbit.git' gh-pages:gh-pages