build wheels on CI

.github/workflows/build.yaml

@@ -66,3 +66,23 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  wheel-build:
+    needs: [cpp-build]
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel.sh
+  wheel-publish:
+    needs: wheel-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-23.12
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: cucim

.github/workflows/pr.yaml

@@ -17,6 +17,8 @@ jobs:
       - conda-python-build
       - conda-python-tests
       - docs-build
+      - wheel-build
+      - wheel-tests
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-23.12
   checks:
@@ -50,3 +52,17 @@ jobs:
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
       run_script: "ci/build_docs.sh"
+  wheel-build:
+    needs: checks
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12
+    with:
+      build_type: pull-request
+      script: ci/build_wheel.sh
+  wheel-tests:
+    needs: wheel-build
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12
+    with:
+      build_type: pull-request
+      script: ci/test_wheel.sh

.github/workflows/test.yaml

@@ -22,3 +22,12 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  wheel-tests:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12
+    with:
+      build_type: nightly
+      branch: ${{ inputs.branch }}
+      date: ${{ inputs.date }}
+      sha: ${{ inputs.sha }}
+      script: ci/test_wheel.sh

CMakeLists.txt

@@ -23,7 +23,7 @@ cmake_minimum_required(VERSION 3.18)
 
 # Set VERSION and BUILD
 unset(VERSION CACHE)
-file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION)
+file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION_CPP VERSION)
 set(PROJECT_VERSION_BUILD dev)
 
 # Append local cmake module path

CONTRIBUTING.md

@@ -263,19 +263,21 @@ conda install -y -c ${CONDA_BLD_DIR} -c conda-forge \
 
 ## Building a package (for distribution. Including a wheel package for pip)
 
-**Build**
-
-You can execute the following command to build a wheel file for pip.
+**Wheel Build**
 
+If you are using CUDA 12.x, please update pyproject.toml as follows before building the wheel
 ```bash
-./run build_package
+sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/cucim/pyproject.toml
 ```
+This will switch the CuPy dependency to one based on CUDA 12.x instead of 11.x.
 
-The command would use `./temp` folder as a local build folder and build a distribution package into `dist` folder using [dockcross](https://github.com/dockcross/dockcross)'s manylinux2014 docker image.
+The wheel can then be build using.
 
-`./run build_package` will reuse local `./temp` folder to reduce the build time.
+```bash
+python -m pip wheel python/cucim/ -w dist -vvv --no-deps --disable-pip-version-check
+```
 
-If C++ code or dependent packages are updated so the build is failing somehow, please retry it after deleting the `temp` folder under the repository root.
+**Note:** It is possible to build the wheel in this way even without compiling the C++ library first, but in that case the `cucim.clara` module will not be importable.
 
 **Install**
 

ci/build_cpp.sh

@@ -9,8 +9,10 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
+version=$(rapids-generate-version)
+
 rapids-logger "Begin cpp build"
 
-rapids-conda-retry mambabuild conda/recipes/libcucim
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcucim
 
 rapids-upload-conda-to-s3 cpp

ci/build_python.sh

@@ -9,13 +9,27 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
+package_name="cucim"
+package_dir="python/cucim/src"
+
+version=$(rapids-generate-version)
+# for CMake VERSION need to truncate any trailing 'a'
+version_cpp=${version%a*}
+
+commit=$(git rev-parse HEAD)
+
+echo "${version}" > VERSION
+echo "${version_cpp}" > "${package_dir}/VERSION"
+echo "${version_cpp}" > VERSION_CPP
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
+
 rapids-logger "Begin py build"
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 # TODO: Remove `--no-test` flag once importing on a CPU
 # node works correctly
-rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/cucim

ci/build_wheel.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name="cucim"
+package_dir="python/cucim"
+package_src_dir="${package_dir}/src/${package_name}"
+
+CMAKE_BUILD_TYPE="release"
+
+source rapids-configure-sccache
+source rapids-date-string
+
+version=$(rapids-generate-version)
+commit=$(git rev-parse HEAD)
+
+# for CMake VERSION_CPP need to truncate any trailing 'a'
+version_cpp=${version%a*}
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+
+# Patch project metadata files to include the CUDA version suffix and version override.
+pyproject_file="${package_dir}/pyproject.toml"
+
+PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
+# update package name to have the cuda suffix
+sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
+echo "${version}" > VERSION
+echo "${version_cpp}" > VERSION_CPP
+echo "${version_cpp}" > "${package_dir}/VERSION"
+sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py"
+
+if [[ ${PACKAGE_CUDA_SUFFIX} == "-cu12" ]]; then
+    # change pyproject.toml to use CUDA 12.x version of cupy
+    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
+fi
+
+echo "The package name and/or version was modified in the package source. The git diff is:"
+git diff
+
+pip install --upgrade pip
+
+# Install pip build dependencies (not yet using pyproject.toml)
+rapids-dependency-file-generator \
+  --file_key "py_build" \
+  --output "requirements" \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee build_requirements.txt
+pip install -r build_requirements.txt
+
+# Building the libjpeg-turbo dependency requires YASM
+# Also need to install openslide dev libraries on the system
+if ! command -v apt &> /dev/null
+then
+    echo "apt package manager not found, attempting to use yum"
+    yum install yasm openslide-devel -y
+else
+    echo "apt package manager was found"
+    apt update
+    apt install yasm libopenslide-dev -y
+fi
+
+# First build the C++ lib using CMake via the run script
+echo "libcucim version: `cat VERSION_CPP`"
+./run build_local all ${CMAKE_BUILD_TYPE}
+
+cd "${package_dir}"
+
+python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+
+mkdir -p final_dist
+python -m auditwheel repair -w final_dist dist/*
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist

ci/release/update-version.sh

@@ -21,6 +21,7 @@ CURRENT_LONG_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}.${CURRENT_PATCH}
 #Get <major>.<minor> for next version
 NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}')
 NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}')
+NEXT_PATCH=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[3]}')
 NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
 
 echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG"
@@ -34,12 +35,12 @@ function sed_runner() {
 sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/source/conf.py
 sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf.py
 
-sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cucim/pyproject.toml
-sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" VERSION
-sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" python/cucim/VERSION
-sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cucim/src/cucim/__init__.pyi
-sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION
-sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION
+# Centralized version file update
+echo "${NEXT_FULL_TAG}" > VERSION
+# update VERSION files used by C++/CMake (TODO: avoid these duplicate copies)
+NEXT_CPP_LONG_TAG=${NEXT_MAJOR}.${NEXT_MINOR}.${NEXT_PATCH}  # no leading 'v' or trailing 'a'
+echo "${NEXT_CPP_LONG_TAG}" > VERSION_CPP
+
 sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md
 sed_runner "s/v${CURRENT_LONG_TAG}/v${NEXT_FULL_TAG}/g" python/cucim/docs/getting_started/index.md
 sed_runner "s#cucim.kit.cuslide@${CURRENT_LONG_TAG}.so#cucim.kit.cuslide@${NEXT_FULL_TAG}.so#g" python/cucim/docs/getting_started/index.md

ci/test_wheel.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -eou pipefail
+
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install $(echo ./dist/cucim*.whl)[test]
+
+# Run smoke tests for aarch64 pull requests
+if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
+    python ./ci/wheel_smoke_test.py
+else
+    # TODO: revisit enabling imagecodecs package during testing
+    python -m pytest ./python/cucim
+fi

ci/wheel_smoke_test.py

@@ -0,0 +1,21 @@
+import cupy as cp
+
+import cucim
+import cucim.skimage
+
+
+if __name__ == "__main__":
+    # verify that all top-level modules are available
+    assert cucim.is_available('clara')
+    assert cucim.is_available('core')
+    assert cucim.is_available('skimage')
+
+    # generate a synthetic image and apply a filter
+    img = cucim.skimage.data.binary_blobs(length=512, n_dim=2)
+    assert isinstance(img, cp.ndarray)
+    assert img.dtype.kind == 'b'
+    assert img.shape == (512, 512)
+
+    eroded = cucim.skimage.morphology.binary_erosion(
+        img, cp.ones((3, 3), dtype=bool)
+    )

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -24,13 +24,15 @@ dependencies:
 - libnvjpeg-dev=11.6.0.55
 - libnvjpeg=11.6.0.55
 - libwebp-base
+- matplotlib-base
 - nbsphinx
 - ninja
 - numpy>=1.21.3
 - numpydoc
 - nvcc_linux-64=11.8
 - openslide-python>=1.1.2
 - pip
+- pooch>=1.6.0
 - pre-commit
 - psutil>=5.8.0
 - pydata-sphinx-theme

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -24,12 +24,14 @@ dependencies:
 - libnvjpeg-dev
 - libnvjpeg-static
 - libwebp-base
+- matplotlib-base
 - nbsphinx
 - ninja
 - numpy>=1.21.3
 - numpydoc
 - openslide-python>=1.1.2
 - pip
+- pooch>=1.6.0
 - pre-commit
 - psutil>=5.8.0
 - pydata-sphinx-theme

conda/recipes/cucim/meta.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2021-2023, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %}
 {% set py_version = environ['CONDA_PY'] %}
 {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %}
 {% set cuda_major = cuda_version.split('.')[0] %}
@@ -12,7 +12,7 @@ package:
   version: {{ version }}
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}

conda/recipes/libcucim/meta.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2021-2023, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %}
 {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %}
 {% set cuda_major = cuda_version.split('.')[0] %}
 {% set date_string = environ['RAPIDS_DATE_STRING'] %}
@@ -11,7 +11,7 @@ package:
   version: {{ version }}
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}

cpp/cmake/deps/openslide.cmake

@@ -22,7 +22,9 @@ if (NOT TARGET deps::openslide)
         set(OPENSLIDE_LIB_PATH "$ENV{CONDA_PREFIX}/lib/libopenslide.so")
     elseif (EXISTS /usr/lib/x86_64-linux-gnu/libopenslide.so)
         set(OPENSLIDE_LIB_PATH /usr/lib/x86_64-linux-gnu/libopenslide.so)
-    else () # CentOS 6
+    elseif (EXISTS /usr/lib/aarch64-linux-gnu/libopenslide.so)
+        set(OPENSLIDE_LIB_PATH /usr/lib/aarch64-linux-gnu/libopenslide.so)
+    else () # CentOS (x86_64)
         set(OPENSLIDE_LIB_PATH /usr/lib64/libopenslide.so)
     endif ()
 

cpp/plugins/cucim.kit.cumed/CMakeLists.txt

@@ -23,7 +23,7 @@ cmake_minimum_required(VERSION 3.18)
 
 # Set VERSION
 unset(VERSION CACHE)
-file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION)
+file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION_CPP VERSION)
 
 # Append local cmake module path
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules")

cpp/plugins/cucim.kit.cuslide/CMakeLists.txt

@@ -23,7 +23,7 @@ cmake_minimum_required(VERSION 3.18)
 
 # Set VERSION
 unset(VERSION CACHE)
-file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION)
+file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION_CPP VERSION)
 
 # Append local cmake module path
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules")

cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake

@@ -22,7 +22,9 @@ if (NOT TARGET deps::openslide)
         set(OPENSLIDE_LIB_PATH "$ENV{CONDA_PREFIX}/lib/libopenslide.so")
     elseif (EXISTS /usr/lib/x86_64-linux-gnu/libopenslide.so)
         set(OPENSLIDE_LIB_PATH /usr/lib/x86_64-linux-gnu/libopenslide.so)
-    else () # CentOS 6
+    elseif (EXISTS /usr/lib/aarch64-linux-gnu/libopenslide.so)
+        set(OPENSLIDE_LIB_PATH /usr/lib/aarch64-linux-gnu/libopenslide.so)
+    else () # CentOS (x86_64)
         set(OPENSLIDE_LIB_PATH /usr/lib64/libopenslide.so)
     endif ()