Try making the build_test_runtime matrix configurable. #7
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2024 The IREE Authors | |
| # | |
| # Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| # See https://llvm.org/LICENSE.txt for license information. | |
| # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| # A PR number if a pull request and otherwise the commit hash. This cancels | |
| # queued and in-progress runs for the same PR (presubmit) or commit | |
| # (postsubmit). The workflow name is prepended to avoid conflicts between | |
| # different workflows. | |
| group: ${{ github.workflow }}-${{ github.event.number || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| # This needs to be in env instead of the outputs of setup because it contains | |
| # the run attempt and we want that to be the current attempt, not whatever | |
| # attempt the setup step last ran in. | |
| GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }} | |
| jobs: | |
| setup: | |
| uses: ./.github/workflows/setup.yml | |
| build_for_benchmarks: | |
| needs: setup | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_for_benchmarks') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/build_all.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| write-caches: ${{ needs.setup.outputs.write-caches }} | |
| run-tests: false | |
| build_benchmark_tools: | |
| needs: [setup, build_for_benchmarks] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_benchmark_tools') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/build_benchmark_tools.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| install-dir: ${{ needs.build_for_benchmarks.outputs.install-dir }} | |
| install-dir-archive: ${{ needs.build_for_benchmarks.outputs.install-dir-archive }} | |
| install-dir-gcs-artifact: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact }} | |
| build_e2e_test_artifacts: | |
| needs: [setup, build_for_benchmarks] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_e2e_test_artifacts') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/build_e2e_test_artifacts.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| install-dir: ${{ needs.build_for_benchmarks.outputs.install-dir }} | |
| install-dir-archive: ${{ needs.build_for_benchmarks.outputs.install-dir-archive }} | |
| install-dir-gcs-artifact: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact }} | |
| benchmark-presets: ${{ needs.setup.outputs.benchmark-presets }} | |
| shard-count: "c2-standard-60=2,default=1" | |
| test_benchmark_suites: | |
| needs: [setup, build_for_benchmarks, build_e2e_test_artifacts] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_benchmark_suites') && needs.setup.outputs.benchmark-presets != '' | |
| strategy: | |
| matrix: | |
| target: | |
| - platform: linux | |
| arch: riscv_64 | |
| docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79" | |
| run_scripts: "./build_tools/cmake/build_riscv.sh && ./build_tools/cmake/test_riscv.sh" | |
| - platform: linux | |
| arch: riscv_32 | |
| docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79" | |
| run_scripts: "./build_tools/cmake/build_riscv.sh && ./build_tools/cmake/test_riscv.sh" | |
| - platform: linux | |
| arch: x86_64 | |
| docker_image: "gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" | |
| run_scripts: "./build_tools/cmake/test_benchmark_suites_on_linux.sh" | |
| # Requires Intel CascadeLake CPU. | |
| host_machine: c2s601t | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=${{ needs.setup.outputs.runner-env }} | |
| - ${{ matrix.target.host_machine || 'cpu' }} # Default to generic x86_64 VM. | |
| env: | |
| PLATFORM: ${{ matrix.target.platform }} | |
| ARCH: ${{ matrix.target.arch }} | |
| DOCKER_IMAGE: ${{ matrix.target.docker_image }} | |
| RUN_SCRIPTS: ${{ matrix.target.run_scripts }} | |
| INSTALL_DIR: ${{ needs.build_for_benchmarks.outputs.install-dir }} | |
| INSTALL_DIR_ARCHIVE: ${{ needs.build_for_benchmarks.outputs.install-dir-archive }} | |
| INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact }} | |
| TARGET_BUILD_DIR: build-${{ matrix.target.platform }}-${{ matrix.target.arch }} | |
| E2E_TEST_ARTIFACTS_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} | |
| E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@v4.1.7 | |
| - name: "Checking out runtime submodules" | |
| run: ./build_tools/scripts/git/update_runtime_submodules.sh | |
| - name: "Downloading install dir archive" | |
| run: gcloud storage cp "${INSTALL_DIR_GCS_ARTIFACT}" "${INSTALL_DIR_ARCHIVE}" | |
| - name: "Extracting install directory" | |
| run: tar -xf "${INSTALL_DIR_ARCHIVE}" | |
| # TODO(#11136): Only download the needed artifacts instead of everything. | |
| - name: "Downloading e2e test artifacts" | |
| run: | | |
| mkdir -p ${E2E_TEST_ARTIFACTS_DIR} | |
| gcloud storage cp -r "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}/*" "${E2E_TEST_ARTIFACTS_DIR}" | |
| - name: "Build iree-run-module and test benchmark suite modules" | |
| run: | | |
| ./build_tools/github_actions/docker_run.sh \ | |
| --env "IREE_TARGET_PLATFORM=${PLATFORM}" \ | |
| --env "IREE_TARGET_ARCH=${ARCH}" \ | |
| --env "IREE_TARGET_BUILD_DIR=${TARGET_BUILD_DIR}" \ | |
| --env "BUILD_PRESET=benchmark-suite-test" \ | |
| --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \ | |
| --env "E2E_TEST_ARTIFACTS_DIR=${E2E_TEST_ARTIFACTS_DIR}" \ | |
| "${DOCKER_IMAGE}" \ | |
| bash -euo pipefail -c \ | |
| "${RUN_SCRIPTS}" | |
| compilation_benchmarks: | |
| needs: [setup, build_e2e_test_artifacts] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'compilation_benchmarks') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/benchmark_compilation.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} | |
| e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} | |
| e2e-test-artifacts-build-log: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log }} | |
| e2e-test-artifacts-build-log-gcs-artifact: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log-gcs-artifact }} | |
| execution_benchmarks: | |
| needs: [setup, build_benchmark_tools, build_e2e_test_artifacts] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'execution_benchmarks') && needs.setup.outputs.benchmark-presets != '' | |
| uses: ./.github/workflows/benchmark_execution.yml | |
| with: | |
| runner-group: ${{ needs.setup.outputs.runner-group }} | |
| runner-env: ${{ needs.setup.outputs.runner-env }} | |
| e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }} | |
| e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }} | |
| benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }} | |
| process_benchmark_results: | |
| needs: [setup, compilation_benchmarks, execution_benchmarks] | |
| if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'process_benchmark_results') && needs.setup.outputs.benchmark-presets != '' | |
| runs-on: | |
| - self-hosted # must come first | |
| - runner-group=${{ needs.setup.outputs.runner-group }} | |
| - environment=${{ needs.setup.outputs.runner-env }} | |
| - cpu | |
| - os-family=Linux | |
| env: | |
| COMPILE_STATS_RESULTS: ${{ needs.compilation_benchmarks.outputs.compile-stats-results }} | |
| COMPILE_STATS_RESULTS_GCS_ARTIFACT: ${{ needs.compilation_benchmarks.outputs.compile-stats-results-gcs-artifact }} | |
| # Empty if no execution benchmark runs. | |
| EXECUTION_BENCHMARK_RESULTS_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-dir }} | |
| # Empty if no execution benchmark runs. | |
| EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-gcs-artifact-dir }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@v4.1.7 | |
| with: | |
| # We need the full history (and main branch) to generate the report. | |
| fetch-depth: 0 | |
| - name: Downloading compilation benchmark results | |
| run: | | |
| gcloud storage cp \ | |
| "${COMPILE_STATS_RESULTS_GCS_ARTIFACT}" \ | |
| "${COMPILE_STATS_RESULTS}" | |
| - name: Downloading execution benchmark results | |
| id: download-execution-results | |
| # Skip the download if there is no execution benchmark results (e.g. no | |
| # benchmark matches the preset/filter). In such case, no benchmark job | |
| # is run in benchmark_execution.yml and the output variables are empty. | |
| if: env.EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR != '' | |
| run: | | |
| gcloud storage cp -r \ | |
| "${EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR}/benchmark-results-*.json" \ | |
| "${EXECUTION_BENCHMARK_RESULTS_DIR}" | |
| echo "execution-benchmark-results-pattern=${EXECUTION_BENCHMARK_RESULTS_DIR}/benchmark-results-*.json" >> "${GITHUB_OUTPUT}" | |
| - name: Generating comment | |
| if: fromJson(needs.setup.outputs.is-pr) | |
| id: generate-comment | |
| env: | |
| # Wildcard pattern to match all execution benchmark results. Empty if | |
| # execution_benchmarks is skipped, which results in no match. | |
| EXECUTION_BENCHMARK_RESULTS_PATTERN: ${{ steps.download-execution-results.outputs.execution-benchmark-results-pattern }} | |
| IREE_BUILD_URL: https://github.com/iree-org/iree/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| BENCHMARK_COMMENT_ARTIFACT: benchmark-comment.json | |
| run: | | |
| build_tools/github_actions/docker_run.sh \ | |
| gcr.io/iree-oss/benchmark-report@sha256:7498c6f32f63f13faf085463cc38656d4297519c824e63e1c99c8c258147f6ff \ | |
| ./build_tools/benchmarks/generate_benchmark_comment.py \ | |
| --verbose \ | |
| --pr_number="${PR_NUMBER}" \ | |
| --pr_committish="${GITHUB_SHA}" \ | |
| --pr_base_branch="origin/${GITHUB_BASE_REF}" \ | |
| --comment_type="benchmark-summary" \ | |
| --build_url="${IREE_BUILD_URL}" \ | |
| --benchmark_files="${EXECUTION_BENCHMARK_RESULTS_PATTERN}" \ | |
| --compile_stats_files="${COMPILE_STATS_RESULTS}" \ | |
| --output="${BENCHMARK_COMMENT_ARTIFACT}" | |
| echo "benchmark-comment-artifact=${BENCHMARK_COMMENT_ARTIFACT}" >> "${GITHUB_OUTPUT}" | |
| - name: Uploading comment artifact | |
| # Due to security reasons, instead of posting the comment to PR, we only | |
| # upload the comment data in presubmit workflow and trigger the posting | |
| # workflow on the main branch. See post_benchmark_comment.yaml | |
| if: fromJson(needs.setup.outputs.is-pr) | |
| env: | |
| BENCHMARK_COMMENT_ARTIFACT: ${{ steps.generate-comment.outputs.benchmark-comment-artifact }} | |
| BENCHMARK_COMMENT_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.generate-comment.outputs.benchmark-comment-artifact }} | |
| run: | | |
| gcloud storage cp \ | |
| "${BENCHMARK_COMMENT_ARTIFACT}" \ | |
| "${BENCHMARK_COMMENT_GCS_ARTIFACT}" | |
| - name: Uploading results to dashboard | |
| if: github.ref_name == 'main' | |
| env: | |
| EXECUTION_BENCHMARK_RESULTS_PATTERN: ${{ steps.download-execution-results.outputs.execution-benchmark-results-pattern }} | |
| IREE_DASHBOARD_API_TOKEN: ${{ secrets.IREE_DASHBOARD_API_TOKEN }} | |
| run: | | |
| build_tools/github_actions/docker_run.sh \ | |
| --env "IREE_DASHBOARD_API_TOKEN=${IREE_DASHBOARD_API_TOKEN}" \ | |
| gcr.io/iree-oss/benchmark-report@sha256:7498c6f32f63f13faf085463cc38656d4297519c824e63e1c99c8c258147f6ff \ | |
| ./build_tools/benchmarks/upload_benchmarks_to_dashboard.py \ | |
| --verbose \ | |
| --benchmark_files="${EXECUTION_BENCHMARK_RESULTS_PATTERN}" \ | |
| --compile_stats_files="${COMPILE_STATS_RESULTS}" | |
| ############################################################################## | |
| # Depends on all the other jobs to provide a single anchor that indicates the | |
| # final status. Status reporting will become more sophisticated in the future | |
| # and we can hopefully avoid the need to explicitly list every single job... | |
| benchmark_summary: | |
| # Even if you have an explicit if condition, you still need to override | |
| # GitHub's default behavior of not running if any dependencies failed. | |
| if: always() | |
| runs-on: ubuntu-20.04 | |
| needs: | |
| - setup | |
| - build_for_benchmarks | |
| # Benchmark pipeline | |
| - build_benchmark_tools | |
| - build_e2e_test_artifacts | |
| - test_benchmark_suites | |
| - compilation_benchmarks | |
| - execution_benchmarks | |
| - process_benchmark_results | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@v4.1.7 | |
| - name: Getting failed jobs | |
| id: failed_jobs | |
| run: | | |
| echo '${{ toJson(needs) }}' | |
| FAILED_JOBS="$(echo '${{ toJson(needs) }}' \ | |
| | jq --raw-output \ | |
| 'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \ | |
| )" | |
| echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT | |
| if [[ "${FAILED_JOBS}" != "" ]]; then | |
| echo "The following jobs failed: ${FAILED_JOBS}" | |
| exit 1 | |
| fi | |
| - name: Show useful artifact links | |
| if: always() | |
| env: | |
| # If the job of an artifact is skipped or failed, we show "NOT_PRESENT". | |
| INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_for_benchmarks.outputs.install-dir-gcs-artifact || 'NOT_PRESENT' }} | |
| E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir || 'NOT_PRESENT' }} | |
| BENCHMARK_TOOLS_GCS_ARTIFACT_DIR: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir || 'NOT_PRESENT' }} | |
| EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-gcs-artifact-dir || 'NOT_PRESENT' }} | |
| COMPILATION_BENCHMARK_RESULTS_GCS_ARTIFACT: ${{ needs.compilation_benchmarks.outputs.compile-stats-results-gcs-artifact || 'NOT_PRESENT' }} | |
| run: | | |
| envsubst < ./.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md >> "${GITHUB_STEP_SUMMARY}" | |
| - name: Posting to Discord | |
| uses: sarisia/actions-status-discord@v1.14.3 | |
| if: failure() && github.ref_name == 'main' | |
| with: | |
| webhook: ${{ secrets.DISCORD_WEBHOOK }} | |
| description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}" | |
| url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" |