diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh index 3a2d1337a0..a34c0a55a8 100755 --- a/.github/actions/compute-matrix/compute-matrix.sh +++ b/.github/actions/compute-matrix/compute-matrix.sh @@ -2,22 +2,48 @@ set -euo pipefail -# Check for the correct number of arguments -if [ $# -ne 2 ]; then - echo "Usage: $0 MATRIX_FILE MATRIX_QUERY" - echo "MATRIX_FILE: The path to the matrix file." - echo "MATRIX_QUERY: The jq query used to specify the desired matrix. e.g., '.pull-request.nvcc'" +write_output() { + local key="$1" + local value="$(echo "$2" | jq -c .)" + echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}" +} + +explode_std_versions() { + jq -r 'map(. as $o | {std: $o.std[]} + del($o.std))' +} + +extract_matrix() { + local file="$1" + local type="$2" + local matrix=$(yq -o=json "$file" | jq -r ".$type") + write_output "DEVCONTAINER_VERSION" "$(yq -o json "$file" | jq -r '.devcontainer_version')" + local nvcc_full_matrix="$(echo "$matrix" | jq -r '.nvcc' | explode_std_versions )" + write_output "NVCC_FULL_MATRIX" "$nvcc_full_matrix" + write_output "CUDA_VERSIONS" "$(echo "$nvcc_full_matrix" | jq -r '[.[] | .cuda] | unique')" + write_output "HOST_COMPILERS" "$(echo "$nvcc_full_matrix" | jq -r '[.[] | .compiler.name] | unique')" + write_output "PER_CUDA_COMPILER_MATRIX" "$(echo "$nvcc_full_matrix" | jq -r ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')" + write_output "NVRTC_MATRIX" "$(echo "$matrix" | jq '.nvrtc' | explode_std_versions)" +} + +main() { + if [ "$1" == "-v" ]; then + set -x + shift + fi + + if [ $# -ne 2 ] || [ "$2" != "pull_request" ]; then + echo "Usage: $0 [-v] MATRIX_FILE MATRIX_TYPE" + echo " -v : Enable verbose output" + echo " MATRIX_FILE : The path to the matrix file." + echo " MATRIX_TYPE : The desired matrix. Supported values: 'pull_request'" exit 1 -fi + fi -# Get realpath before changing directory -MATRIX_FILE=$(realpath "$1") -MATRIX_QUERY="$2" + echo "Input matrix file:" >&2 + cat "$1" >&2 + echo "Matrix Type: $2" >&2 -# Ensure the script is being executed in its containing directory -cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; + extract_matrix "$1" "$2" +} -echo "Input matrix file:" >&2 -cat "$MATRIX_FILE" >&2 -echo "Query: $MATRIX_QUERY" >&2 -echo $(yq -o=json "$MATRIX_FILE" | jq -c -r "$MATRIX_QUERY | map(. as \$o | {std: .std[]} + del(\$o.std))") +main "$@" diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ea2037323b..d7fe05d1a1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -32,145 +32,98 @@ concurrency: cancel-in-progress: true jobs: - - get-devcontainer-version: - name: Get devcontainer version - runs-on: ubuntu-latest - outputs: - DEVCONTAINER_VERSION: ${{ steps.set-outputs.outputs.DEVCONTAINER_VERSION }} - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - name: Get devcontainer version - id: set-outputs - run: | - DEVCONTAINER_VERSION=$(yq -o json ci/matrix.yaml | jq -r '.devcontainer_version') - echo "DEVCONTAINER_VERSION=$DEVCONTAINER_VERSION" | tee -a "$GITHUB_OUTPUT" - - compute-nvcc-matrix: - name: Compute NVCC matrix + compute-matrix: + name: Compute matrix runs-on: ubuntu-latest outputs: - FULL_MATRIX: ${{ steps.set-outputs.outputs.FULL_MATRIX }} - CUDA_VERSIONS: ${{ steps.set-outputs.outputs.CUDA_VERSIONS }} - HOST_COMPILERS: ${{ steps.set-outputs.outputs.HOST_COMPILERS }} - PER_CUDA_COMPILER_MATRIX: ${{ steps.set-outputs.outputs.PER_CUDA_COMPILER_MATRIX }} + DEVCONTAINER_VERSION: ${{steps.set-outputs.outputs.DEVCONTAINER_VERSION}} + NVCC_FULL_MATRIX: ${{steps.set-outputs.outputs.NVCC_FULL_MATRIX}} + CUDA_VERSIONS: ${{steps.set-outputs.outputs.CUDA_VERSIONS}} + HOST_COMPILERS: ${{steps.set-outputs.outputs.HOST_COMPILERS}} + PER_CUDA_COMPILER_MATRIX: ${{steps.set-outputs.outputs.PER_CUDA_COMPILER_MATRIX}} + NVRTC_MATRIX: ${{steps.set-outputs.outputs.matrix}} steps: - name: Checkout repo uses: actions/checkout@v3 - - name: Get full nvcc matrix - id: compute-nvcc-matrix - uses: ./.github/actions/compute-matrix - with: - matrix_file: './ci/matrix.yaml' - matrix_query: '.pull_request.nvcc' - - name: Set outputs + - name: Compute matrix outputs id: set-outputs run: | - FULL_MATRIX='${{steps.compute-nvcc-matrix.outputs.matrix}}' - echo "FULL_MATRIX=$FULL_MATRIX" | tee -a "$GITHUB_OUTPUT" - CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique') - echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT" - HOST_COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique') - echo "HOST_COMPILERS=$HOST_COMPILERS" | tee -a "$GITHUB_OUTPUT" - PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add') - echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT" - - compute-nvrtc-matrix: - name: Compute NVRTC matrix - runs-on: ubuntu-latest - outputs: - NVRTC_MATRIX: ${{ steps.set-outputs.outputs.NVRTC_MATRIX }} - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - name: Get full nvrtc matrix - id: compute-nvrtc-matrix - uses: ./.github/actions/compute-matrix - with: - matrix_file: './ci/matrix.yaml' - matrix_query: '.pull_request.nvrtc' - - name: Set outputs - id: set-outputs - run: | - NVRTC_MATRIX='${{steps.compute-nvrtc-matrix.outputs.matrix}}' - echo "NVRTC_MATRIX=$NVRTC_MATRIX" | tee -a "$GITHUB_OUTPUT" - + .github/actions/compute-matrix/compute-matrix.sh ci/matrix.yaml pull_request + nvrtc: name: NVRTC CUDA${{matrix.cuda}} C++${{matrix.std}} + needs: compute-matrix if: ${{ !contains(github.event.head_commit.message, 'skip-tests') }} - needs: [compute-nvrtc-matrix, get-devcontainer-version] uses: ./.github/workflows/run-as-coder.yml strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.compute-nvrtc-matrix.outputs.NVRTC_MATRIX) }} + include: ${{ fromJSON(needs.compute-matrix.outputs.NVRTC_MATRIX) }} with: name: NVRTC CUDA${{matrix.cuda}} C++${{matrix.std}} runner: linux-${{matrix.cpu}}-gpu-v100-latest-1 - image: rapidsai/devcontainers:${{needs.get-devcontainer-version.outputs.DEVCONTAINER_VERSION}}-cpp-gcc12-cuda${{matrix.cuda}}-${{matrix.os}} + image: rapidsai/devcontainers:${{needs.compute-matrix.outputs.DEVCONTAINER_VERSION}}-cpp-gcc12-cuda${{matrix.cuda}}-${{matrix.os}} command: | ./ci/nvrtc_libcudacxx.sh g++ ${{matrix.std}} ${{matrix.gpu_build_archs}} - thrust: name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} - needs: [compute-nvcc-matrix, get-devcontainer-version] + needs: compute-matrix uses: ./.github/workflows/dispatch-build-and-test.yml strategy: fail-fast: false matrix: - cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} - compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.HOST_COMPILERS) }} + cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} + compiler: ${{ fromJSON(needs.compute-matrix.outputs.HOST_COMPILERS) }} with: - per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} + per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} build_script: "./ci/build_thrust.sh" test_script: "./ci/test_thrust.sh" - devcontainer_version: ${{ needs.get-devcontainer-version.outputs.DEVCONTAINER_VERSION }} + devcontainer_version: ${{ needs.compute-matrix.outputs.DEVCONTAINER_VERSION }} cub: name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} - needs: [compute-nvcc-matrix, get-devcontainer-version] + needs: compute-matrix uses: ./.github/workflows/dispatch-build-and-test.yml strategy: fail-fast: false matrix: - cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} - compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.HOST_COMPILERS) }} + cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} + compiler: ${{ fromJSON(needs.compute-matrix.outputs.HOST_COMPILERS) }} with: - per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} + per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} build_script: "./ci/build_cub.sh" test_script: "./ci/test_cub.sh" - devcontainer_version: ${{ needs.get-devcontainer-version.outputs.DEVCONTAINER_VERSION }} + devcontainer_version: ${{ needs.compute-matrix.outputs.DEVCONTAINER_VERSION }} libcudacxx: name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }} - needs: [compute-nvcc-matrix, get-devcontainer-version] + needs: compute-matrix uses: ./.github/workflows/dispatch-build-and-test.yml strategy: fail-fast: false matrix: - cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} - compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.HOST_COMPILERS) }} + cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} + compiler: ${{ fromJSON(needs.compute-matrix.outputs.HOST_COMPILERS) }} with: - per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} + per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }} build_script: "./ci/build_libcudacxx.sh" test_script: "./ci/test_libcudacxx.sh" - devcontainer_version: ${{ needs.get-devcontainer-version.outputs.DEVCONTAINER_VERSION }} + devcontainer_version: ${{ needs.compute-matrix.outputs.DEVCONTAINER_VERSION }} examples: name: CCCL Examples + needs: compute-matrix if: ${{ !contains(github.event.head_commit.message, 'skip-tests') }} - needs: [compute-nvcc-matrix, get-devcontainer-version] strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.FULL_MATRIX) }} + include: ${{ fromJSON(needs.compute-matrix.outputs.NVCC_FULL_MATRIX) }} uses: ./.github/workflows/run-as-coder.yml with: name: CCCL Examples CUDA${{matrix.cuda}} ${{matrix.compiler.name}}${{matrix.compiler.version}} runner: linux-${{matrix.cpu}}-gpu-v100-latest-1 - image: rapidsai/devcontainers:${{needs.get-devcontainer-version.outputs.DEVCONTAINER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-${{matrix.os}} + image: rapidsai/devcontainers:${{needs.compute-matrix.outputs.DEVCONTAINER_VERSION}}-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-${{matrix.os}} command: | cmake -S . --preset=examples -DCCCL_EXAMPLE_CPM_TAG=${GITHUB_SHA} -DCMAKE_CUDA_COMPILER=nvcc ctest --preset=examples