From d74e254b36685599e0fa9b36362c778cb4076e3d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 11 Jan 2024 11:41:43 -0600 Subject: [PATCH 01/11] use CUDA 12.2 for building and testing wheels and conda packages --- .github/workflows/build.yaml | 12 ++++++------ .github/workflows/pr.yaml | 16 ++++++++-------- .github/workflows/test.yaml | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0ba4825e9..27556c26d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: if: github.ref_type == 'branch' needs: [python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -60,7 +60,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -68,7 +68,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -78,7 +78,7 @@ jobs: wheel-publish: needs: wheel-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@test-cuda-12.2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7d22e56a2..5bfa1a543 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,32 +20,32 @@ jobs: - wheel-build - wheel-tests secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@test-cuda-12.2 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@test-cuda-12.2 conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@test-cuda-12.2 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@test-cuda-12.2 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-cuda-12.2 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -55,14 +55,14 @@ jobs: wheel-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@test-cuda-12.2 with: build_type: pull-request script: ci/build_wheel.sh wheel-tests: needs: wheel-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: build_type: pull-request script: ci/test_wheel.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6f55ce705..5d7ec87b2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-cuda-12.2 with: build_type: nightly branch: ${{ inputs.branch }} From 9ba0dea9d9bce12a4dec01e1abd123f670d92f43 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 11 Jan 2024 11:44:54 -0600 Subject: [PATCH 02/11] update dependencies.yaml --- .../all_cuda-122_arch-x86_64.yaml | 55 +++++++++++++++++++ dependencies.yaml | 11 ++-- 2 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 conda/environments/all_cuda-122_arch-x86_64.yaml diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml new file mode 100644 index 000000000..4bebf8bb0 --- /dev/null +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -0,0 +1,55 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +- nvidia +dependencies: +- GPUtil>=1.4.0 +- c-compiler +- click +- cmake>=3.23.1,!=3.25.0 +- cuda-cudart-dev +- cuda-nvcc +- cuda-version=12.2 +- cupy>=12.0.0 +- cxx-compiler +- gcc_linux-64=11.* +- imagecodecs>=2021.6.8 +- ipython +- jbig +- lazy_loader>=0.1 +- libcufile-dev +- libnvjpeg-dev +- libnvjpeg-static +- libwebp-base +- matplotlib-base +- nbsphinx +- ninja +- numpy>=1.21.3 +- numpydoc +- openslide-python>=1.3.0 +- pip +- pooch>=1.6.0 +- pre-commit +- psutil>=5.8.0 +- pydata-sphinx-theme +- pytest-cov>=2.12.1 +- pytest-lazy-fixture>=0.6.3 +- pytest-xdist +- pytest>=6.2.4 +- python>=3.8,<3.11 +- recommonmark +- scikit-image>=0.19.0,<0.22.0a0 +- scipy +- sphinx<6 +- sysroot_linux-64==2.17 +- tifffile>=2022.7.28 +- xz +- yasm +- zlib +- zstd +- pip: + - opencv-python-headless>=4.6 +name: all_cuda-122_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index d95d62cd2..658971ae8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: conda matrix: - cuda: ["11.8", "12.0"] + cuda: ["11.8", "12.0", "12.2"] arch: [x86_64] includes: - build @@ -114,7 +114,6 @@ dependencies: arch: x86_64 cuda: "11.8" packages: - - cuda-version=11.8 - nvcc_linux-64=11.8 - libcufile=1.4.0.31 - libcufile-dev=1.4.0.31 @@ -124,14 +123,12 @@ dependencies: arch: aarch64 cuda: "11.8" packages: - - cuda-version=11.8 - nvcc_linux-aarch64=11.8 - libnvjpeg=11.6.0.55 - libnvjpeg-dev=11.6.0.55 - matrix: - cuda: "12.0" + cuda: "12.*" packages: - - cuda-version=12.0 - cuda-nvcc - libcufile-dev - libnvjpeg-dev @@ -165,6 +162,10 @@ dependencies: cuda: "12.0" packages: - cuda-version=12.0 + - matrix: + cuda: "12.2" + packages: + - cuda-version=12.2 cuda: specific: - output_types: conda From 44ea0f4dfe92c21ee87d23484da369619c9a4e0c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 11 Jan 2024 13:44:33 -0600 Subject: [PATCH 03/11] remove 1.20 env file --- .../all_cuda-120_arch-x86_64.yaml | 55 ------------------- dependencies.yaml | 2 +- 2 files changed, 1 insertion(+), 56 deletions(-) delete mode 100644 conda/environments/all_cuda-120_arch-x86_64.yaml diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml deleted file mode 100644 index 1084ea629..000000000 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- conda-forge -- nvidia -dependencies: -- GPUtil>=1.4.0 -- c-compiler -- click -- cmake>=3.23.1,!=3.25.0 -- cuda-cudart-dev -- cuda-nvcc -- cuda-version=12.0 -- cupy>=12.0.0 -- cxx-compiler -- gcc_linux-64=11.* -- imagecodecs>=2021.6.8 -- ipython -- jbig -- lazy_loader>=0.1 -- libcufile-dev -- libnvjpeg-dev -- libnvjpeg-static -- libwebp-base -- matplotlib-base -- nbsphinx -- ninja -- numpy>=1.21.3 -- numpydoc -- openslide-python>=1.3.0 -- pip -- pooch>=1.6.0 -- pre-commit -- psutil>=5.8.0 -- pydata-sphinx-theme -- pytest-cov>=2.12.1 -- pytest-lazy-fixture>=0.6.3 -- pytest-xdist -- pytest>=6.2.4 -- python>=3.8,<3.11 -- recommonmark -- scikit-image>=0.19.0,<0.22.0a0 -- scipy -- sphinx<6 -- sysroot_linux-64==2.17 -- tifffile>=2022.7.28 -- xz -- yasm -- zlib -- zstd -- pip: - - opencv-python-headless>=4.6 -name: all_cuda-120_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 658971ae8..0acc704bf 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: conda matrix: - cuda: ["11.8", "12.0", "12.2"] + cuda: ["11.8", "12.2"] arch: [x86_64] includes: - build From d4cab193d7c8c4b0f80b9b2a3b0dd71d2a33490a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jan 2024 16:32:15 -0600 Subject: [PATCH 04/11] try adding cuda_version to py_build --- dependencies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.yaml b/dependencies.yaml index 0acc704bf..2736d8158 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -38,6 +38,7 @@ files: table: build-system includes: - build + - cuda_version py_run: output: pyproject pyproject_dir: python/cucim From 651adce35745f2dcacebc3066e7faaaf04b28bd9 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Wed, 24 Jan 2024 16:52:46 -0800 Subject: [PATCH 05/11] Pin `cuda-version` during `*cucim` install --- ci/test_python.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/test_python.sh b/ci/test_python.sh index 3739acf2e..41c62285f 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -35,6 +35,7 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ + "cuda-version=${RAPIDS_CUDA_VERSION%.*}" \ "libcucim=${RAPIDS_VERSION_NUMBER}" \ "cucim=${RAPIDS_VERSION_NUMBER}" From 2bed0b39706c8bb11af4e6b6cdb5678e3e97f838 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Mon, 29 Jan 2024 13:47:13 -0800 Subject: [PATCH 06/11] Drop `cuda-version` install workaround --- ci/test_python.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index 41c62285f..3739acf2e 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -35,7 +35,6 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - "cuda-version=${RAPIDS_CUDA_VERSION%.*}" \ "libcucim=${RAPIDS_VERSION_NUMBER}" \ "cucim=${RAPIDS_VERSION_NUMBER}" From a378184418bcba422b20740da80f05689252c1e6 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Mon, 29 Jan 2024 14:22:59 -0800 Subject: [PATCH 07/11] Remove leftover CUDA 11.8 workaround attempt Co-authored-by: Bradley Dice --- dependencies.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 6504b67ca..7505e5aa5 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -38,7 +38,6 @@ files: table: build-system includes: - build - - cuda_version py_run: output: pyproject pyproject_dir: python/cucim From e075700473260e53f7765ed6016b876b1b1146bb Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 29 Jan 2024 16:25:31 -0600 Subject: [PATCH 08/11] Run ci/release/update-version.sh 24.04.00. --- ci/test_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index 3739acf2e..1de0fb135 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -7,7 +7,7 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh -export RAPIDS_VERSION_NUMBER="24.02" +export RAPIDS_VERSION_NUMBER="24.04" rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ From b318986d617452f80e85571e553656166dcae023 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 6 Feb 2024 08:07:35 -0600 Subject: [PATCH 09/11] Ignore run-exports from CUDA 12 compiler. --- conda/recipes/cucim/meta.yaml | 2 ++ conda/recipes/libcucim/meta.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/conda/recipes/cucim/meta.yaml b/conda/recipes/cucim/meta.yaml index 47606ab77..2e6aa06b3 100644 --- a/conda/recipes/cucim/meta.yaml +++ b/conda/recipes/cucim/meta.yaml @@ -20,6 +20,8 @@ build: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} {% endif %} script_env: - AWS_ACCESS_KEY_ID diff --git a/conda/recipes/libcucim/meta.yaml b/conda/recipes/libcucim/meta.yaml index bd591c9e2..0326a2784 100644 --- a/conda/recipes/libcucim/meta.yaml +++ b/conda/recipes/libcucim/meta.yaml @@ -22,6 +22,8 @@ build: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} {% endif %} script_env: - AWS_ACCESS_KEY_ID From eca560e8bd1985ba0da446dfd3a296ad0ae0a5de Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 6 Feb 2024 09:14:03 -0600 Subject: [PATCH 10/11] Loosen run-exports from -dev libraries. --- conda/recipes/libcucim/meta.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conda/recipes/libcucim/meta.yaml b/conda/recipes/libcucim/meta.yaml index 0326a2784..0253403eb 100644 --- a/conda/recipes/libcucim/meta.yaml +++ b/conda/recipes/libcucim/meta.yaml @@ -24,6 +24,8 @@ build: - {{ compiler('cuda11') }} {% else %} - {{ compiler('cuda') }} + - libcufile-dev # [linux64] + - libnvjpeg-dev {% endif %} script_env: - AWS_ACCESS_KEY_ID @@ -81,7 +83,10 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit - - libcufile {{ cuda11_libcufile_run_version }} # [linux64] + - libcufile {{ cuda11_libcufile_run_version }} # [linux64] + {% else %} + - libcufile # [linux64] + - libnvjpeg {% endif %} - {{ pin_compatible('libwebp-base', max_pin='x.x') }} - jbig From b56a9ace0abd644500b15871ea9bbec76263cca7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Feb 2024 00:06:24 -0600 Subject: [PATCH 11/11] Add cuda-cudart-dev with run-exports ignored. --- conda/recipes/cucim/meta.yaml | 7 +++++++ conda/recipes/libcucim/meta.yaml | 3 +++ 2 files changed, 10 insertions(+) diff --git a/conda/recipes/cucim/meta.yaml b/conda/recipes/cucim/meta.yaml index 2e6aa06b3..04a94830a 100644 --- a/conda/recipes/cucim/meta.yaml +++ b/conda/recipes/cucim/meta.yaml @@ -22,6 +22,7 @@ build: - {{ compiler('cuda11') }} {% else %} - {{ compiler('cuda') }} + - cuda-cudart-dev {% endif %} script_env: - AWS_ACCESS_KEY_ID @@ -57,6 +58,9 @@ requirements: host: - click - cuda-version ={{ cuda_version }} + {% if cuda_major != "11" %} + - cuda-cudart-dev + {% endif %} - cupy >=12.0.0 - libcucim ={{ version }} - numpy 1.21 @@ -65,6 +69,9 @@ requirements: - scipy run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + {% if cuda_major != "11" %} + - cuda-cudart + {% endif %} - {{ pin_compatible('numpy') }} - click - cupy >=12.0.0 diff --git a/conda/recipes/libcucim/meta.yaml b/conda/recipes/libcucim/meta.yaml index 0253403eb..e2a9b2c78 100644 --- a/conda/recipes/libcucim/meta.yaml +++ b/conda/recipes/libcucim/meta.yaml @@ -24,6 +24,7 @@ build: - {{ compiler('cuda11') }} {% else %} - {{ compiler('cuda') }} + - cuda-cudart-dev - libcufile-dev # [linux64] - libnvjpeg-dev {% endif %} @@ -68,6 +69,7 @@ requirements: - libnvjpeg {{ cuda11_libnvjpeg_host_version }} - libnvjpeg-dev {{ cuda11_libnvjpeg_host_version }} {% else %} + - cuda-cudart-dev - libcufile-dev # [linux64] - libnvjpeg-dev - libnvjpeg-static @@ -85,6 +87,7 @@ requirements: - cudatoolkit - libcufile {{ cuda11_libcufile_run_version }} # [linux64] {% else %} + - cuda-cudart - libcufile # [linux64] - libnvjpeg {% endif %}