Add support for nvcc-specific matrix. (#243)

* Add support for nvcc-specific matrix. * Remove errnoeous runs-on * Fix path to compute-matrix script. * debug * debug. * debug. * lol, use hyphen not underscore. * Get absolute path for matrix file. * Fix reference to matrix_file var. * Remove extraneous matrix_query * Use composite action instead of reusable workflow for matrix. * Fix quotes around composite action inputs. * Need to checkout repo to get local composite action. * Fix quotes. * Add shell to composite action. * Specify shell in step. * Correctly use inputs for composite action. * debug * Fix expansion. * Remove debug. * Remove debug. * Escape quotes in jq query string. * Use underscore instead of hyphen to avoid needing quotes in query. * Use alternative syntax for jq query. * Missing quote. * Add post processing to set nvcc-specific matrix outputs. * Quotes around matrix output. * Single quotes. * Remove old compute-matrix reusable workflow. * Fix usage message. * Update devcontainer script to reference nvcc field. * Removed unused GITHUB_OUTPUT var. * EOF * Fix quotes.
NVIDIA · Jul 26, 2023 · 3ed55e5 · 3ed55e5
1 parent 025364c
commit 3ed55e5
Show file tree

Hide file tree

Showing 6 changed files with 108 additions and 106 deletions.
diff --git a/.devcontainer/make_devcontainers.sh b/.devcontainer/make_devcontainers.sh
@@ -20,7 +20,7 @@ IMAGE_ROOT="rapidsai/devcontainers:23.06-cpp-"
 matrix_json=$(yq -o json ../ci/matrix.yaml)
 
 # Get unique combinations of cuda version, compiler name/version, and Ubuntu version
-combinations=$(echo "$matrix_json" | jq -c '[."pull-request"[] | {cuda: .cuda, compiler_name: .compiler.name, compiler_version: .compiler.version, os: .os}] | unique | .[]')
+combinations=$(echo "$matrix_json" | jq -c '[.pull_request.nvcc[] | {cuda: .cuda, compiler_name: .compiler.name, compiler_version: .compiler.version, os: .os}] | unique | .[]')
 
 # For each unique combination
 for combination in $combinations; do

diff --git a/.github/actions/compute-matrix/action.yml b/.github/actions/compute-matrix/action.yml
@@ -0,0 +1,25 @@
+
+name: Compute Matrix
+description: "Compute the matrix for a given matrix type from the specified matrix file"
+
+inputs:
+  matrix_query:
+    description: "The jq query used to specify the desired matrix. e.g., .pull_request.nvcc"
+    required: true
+  matrix_file:
+    description: 'The file containing the matrix'
+    required: true
+outputs:
+  matrix:
+    description: 'The requested matrix'
+    value: ${{ steps.compute-matrix.outputs.MATRIX }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Compute matrix
+      id: compute-matrix
+      run: |
+        MATRIX=$(./.github/actions/compute-matrix/compute-matrix.sh ${{inputs.matrix_file}}  ${{inputs.matrix_query}} )
+        echo "matrix=$MATRIX" | tee -a $GITHUB_OUTPUT
+      shell: bash -euxo pipefail {0}
diff --git a/.github/actions/compute-matrix/compute-matrix.sh b/.github/actions/compute-matrix/compute-matrix.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Check for the correct number of arguments
+if [ $# -ne 2 ]; then
+    echo "Usage: $0 MATRIX_FILE MATRIX_QUERY"
+    echo "MATRIX_FILE: The path to the matrix file."
+    echo "MATRIX_QUERY: The jq query used to specify the desired matrix. e.g., '.pull-request.nvcc'"
+    exit 1
+fi
+
+# Get realpath before changing directory
+MATRIX_FILE=$(realpath "$1")
+MATRIX_QUERY="$2"
+
+# Ensure the script is being executed in its containing directory
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+
+echo "Input matrix file:" >&2
+cat "$MATRIX_FILE" >&2
+echo "Query: $MATRIX_QUERY" >&2
+echo $(yq -o=json "$MATRIX_FILE" | jq -c -r "$MATRIX_QUERY | map(. as \$o | {std: .std[]} + del(\$o.std))")
diff --git a/.github/workflows/compute-matrix.yml b/.github/workflows/compute-matrix.yml
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -4,7 +4,7 @@ name: pr
 
 defaults:
   run:
-    shell: bash
+    shell: bash -euo pipefail {0}
 
 on:
   push:
@@ -18,51 +18,72 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  compute-matrix:
-    uses: ./.github/workflows/compute-matrix.yml
-    with:
-      matrix_file: "./ci/matrix.yaml"
-      matrix_type: "pull-request"
+  compute-nvcc-matrix:
+    name: Compute NVCC matrix
+    runs-on: ubuntu-latest
+    outputs:
+      CUDA_VERSIONS: ${{ steps.set-outputs.outputs.CUDA_VERSIONS }}
+      HOST_COMPILERS: ${{ steps.set-outputs.outputs.HOST_COMPILERS }}
+      PER_CUDA_COMPILER_MATRIX: ${{ steps.set-outputs.outputs.PER_CUDA_COMPILER_MATRIX }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Get full nvcc matrix
+        id: compute-nvcc-matrix
+        uses: ./.github/actions/compute-matrix
+        with:
+          matrix_file: './ci/matrix.yaml'
+          matrix_query: '.pull_request.nvcc'
+      - name: Set outputs
+        id: set-outputs
+        run: |
+          FULL_MATRIX='${{steps.compute-nvcc-matrix.outputs.matrix}}'
+          CUDA_VERSIONS=$(echo $FULL_MATRIX | jq -c '[.[] | .cuda] | unique')
+          echo "CUDA_VERSIONS=$CUDA_VERSIONS" | tee -a "$GITHUB_OUTPUT"
+          HOST_COMPILERS=$(echo $FULL_MATRIX | jq -c '[.[] | .compiler.name] | unique')
+          echo "HOST_COMPILERS=$HOST_COMPILERS" | tee -a "$GITHUB_OUTPUT"
+          PER_CUDA_COMPILER_MATRIX=$(echo $FULL_MATRIX | jq -c ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')
+          echo "PER_CUDA_COMPILER_MATRIX=$PER_CUDA_COMPILER_MATRIX" | tee -a "$GITHUB_OUTPUT"
 
   thrust:
     name: Thrust CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
-    needs: compute-matrix
+    needs: compute-nvcc-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
       fail-fast: false
       matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+        cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.HOST_COMPILERS) }}
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_thrust.sh"
       test_script: "./ci/test_thrust.sh"
 
   cub:
     name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
-    needs: compute-matrix
+    needs: compute-nvcc-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
       fail-fast: false
       matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+        cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.HOST_COMPILERS) }}
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_cub.sh"
       test_script: "./ci/test_cub.sh"
 
   libcudacxx:
     name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
-    needs: compute-matrix
+    needs: compute-nvcc-matrix
     uses: ./.github/workflows/dispatch-build-and-test.yml
     strategy:
       fail-fast: false
       matrix: 
-        cuda_version: ${{ fromJSON(needs.compute-matrix.outputs.CUDA_VERSIONS) }} 
-        compiler: ${{ fromJSON(needs.compute-matrix.outputs.COMPILERS) }}
+        cuda_version: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.CUDA_VERSIONS) }} 
+        compiler: ${{ fromJSON(needs.compute-nvcc-matrix.outputs.HOST_COMPILERS) }}
     with:
-      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
+      per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-nvcc-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_libcudacxx.sh"
       test_script: "./ci/test_libcudacxx.sh" 
 

diff --git a/ci/matrix.yaml b/ci/matrix.yaml
@@ -21,24 +21,23 @@ gpus:
 #    This field is unique as it will generate an independent build/test job for each value
 
 # Configurations that will run for every PR
-pull-request:
-# gcc
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '6', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14], jobs: ['build']}
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']}
-# clang
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
-- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']}
+pull_request:
+  nvcc:
+    - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '6', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14], jobs: ['build']}
+    - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+    - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+    - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']}
+    - {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+    - {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']}