From abded3a767d95f13fbea35af772109c25d6df385 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 10 Jun 2024 08:53:39 -0500 Subject: [PATCH] use rapids-build-backend (#1343) Contributes to https://github.com/rapidsai/build-planning/issues/31 Contributes to https://github.com/rapidsai/dependency-file-generator/issues/89 Proposes introducing `rapids-build-backend` as this project's build backend, to reduce the complexity of various CI/build scripts. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Lawrence Mitchell (https://github.com/wence-) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/dask-cuda/pull/1343 --- .pre-commit-config.yaml | 2 +- ci/build_docs.sh | 2 +- ci/build_python.sh | 10 ++---- ci/build_wheel.sh | 17 +-------- ci/check_style.sh | 2 +- ci/release/update-version.sh | 6 ++-- ci/test_python.sh | 2 +- .../all_cuda-114_arch-x86_64.yaml | 15 ++++---- .../all_cuda-118_arch-x86_64.yaml | 15 ++++---- .../all_cuda-122_arch-x86_64.yaml | 15 ++++---- conda/recipes/dask-cuda/meta.yaml | 1 + dask_cuda/_version.py | 14 ++++++-- dask_cuda/tests/test_version.py | 12 +++++++ dependencies.yaml | 35 +++++++++++++++---- pyproject.toml | 18 ++++++---- 15 files changed, 99 insertions(+), 67 deletions(-) create mode 100644 dask_cuda/tests/test_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 492c96f2c..b10be12af 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: args: ["--module=dask_cuda", "--ignore-missing-imports"] pass_filenames: false - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.8.0 + rev: v1.13.11 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 18d96c9af..c2a65a414 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -7,7 +7,7 @@ rapids-logger "Create test conda environment" rapids-dependency-file-generator \ --output conda \ - --file_key docs \ + --file-key docs \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n docs diff --git a/ci/build_python.sh b/ci/build_python.sh index e2429e98c..48cece328 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -13,18 +13,12 @@ export CMAKE_GENERATOR=Ninja rapids-print-env -package_name="dask_cuda" - -version=$(rapids-generate-version) -commit=$(git rev-parse HEAD) - -echo "${version}" | tr -d '"' > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_name}/_version.py" +rapids-generate-version > ./VERSION rapids-logger "Begin py build" conda config --set path_conflict prevent -RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ conda/recipes/dask-cuda rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9ec826733..828972dc2 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -6,22 +6,7 @@ set -euo pipefail source rapids-configure-sccache source rapids-date-string -version=$(rapids-generate-version) -commit=$(git rev-parse HEAD) - -echo "${version}" | tr -d '"' > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "dask_cuda/_version.py" - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" pyproject.toml +rapids-generate-version > ./VERSION python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check diff --git a/ci/check_style.sh b/ci/check_style.sh index 9bc26fe71..f8bc16525 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -8,7 +8,7 @@ rapids-logger "Create checks conda environment" rapids-dependency-file-generator \ --output conda \ - --file_key checks \ + --file-key checks \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n checks diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 0d1b8b1a5..ac834e5e8 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -36,8 +36,8 @@ function sed_runner() { echo "${NEXT_FULL_TAG}" | tr -d '"' > VERSION # Bump testing dependencies -sed_runner "s/ucx-py==.*/ucx-py==${NEXT_UCXPY_VERSION}.*/g" dependencies.yaml -sed_runner "s/ucxx==.*/ucxx==${NEXT_UCXPY_VERSION}.*/g" dependencies.yaml +sed_runner "s/ucx-py==.*/ucx-py==${NEXT_UCXPY_VERSION}.*,>=0.0.0a0/g" dependencies.yaml +sed_runner "s/ucxx==.*/ucxx==${NEXT_UCXPY_VERSION}.*,>=0.0.0a0/g" dependencies.yaml DEPENDENCIES=( cudf @@ -47,7 +47,7 @@ DEPENDENCIES=( ) for FILE in dependencies.yaml conda/environments/*.yaml; do for DEP in "${DEPENDENCIES[@]}"; do - sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}" + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" done done diff --git a/ci/test_python.sh b/ci/test_python.sh index b52cbb6d4..ef24c848f 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,7 +8,7 @@ set -euo pipefail rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ - --file_key test_python \ + --file-key test_python \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/conda/environments/all_cuda-114_arch-x86_64.yaml b/conda/environments/all_cuda-114_arch-x86_64.yaml index 57f475a26..c0fed8e57 100644 --- a/conda/environments/all_cuda-114_arch-x86_64.yaml +++ b/conda/environments/all_cuda-114_arch-x86_64.yaml @@ -10,10 +10,10 @@ dependencies: - click >=8.1 - cuda-version=11.4 - cudatoolkit -- cudf==24.8.* -- dask-cudf==24.8.* -- distributed-ucxx==0.39.* -- kvikio==24.8.* +- cudf==24.8.*,>=0.0.0a0 +- dask-cudf==24.8.*,>=0.0.0a0 +- distributed-ucxx==0.39.*,>=0.0.0a0 +- kvikio==24.8.*,>=0.0.0a0 - numactl-devel-cos7-x86_64 - numba>=0.57 - numpy>=1.23,<2.0a0 @@ -24,13 +24,14 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.12 -- rapids-dask-dependency==24.8.* +- rapids-build-backend>=0.3.0,<0.4.0dev0 +- rapids-dask-dependency==24.8.*,>=0.0.0a0 - setuptools>=64.0.0 - sphinx - sphinx-click>=2.7.1 - sphinx-rtd-theme>=0.5.1 - ucx-proc=*=gpu -- ucx-py==0.39.* -- ucxx==0.39.* +- ucx-py==0.39.*,>=0.0.0a0 +- ucxx==0.39.*,>=0.0.0a0 - zict>=2.0.0 name: all_cuda-114_arch-x86_64 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 627df99ca..d1f6933cd 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -10,10 +10,10 @@ dependencies: - click >=8.1 - cuda-version=11.8 - cudatoolkit -- cudf==24.8.* -- dask-cudf==24.8.* -- distributed-ucxx==0.39.* -- kvikio==24.8.* +- cudf==24.8.*,>=0.0.0a0 +- dask-cudf==24.8.*,>=0.0.0a0 +- distributed-ucxx==0.39.*,>=0.0.0a0 +- kvikio==24.8.*,>=0.0.0a0 - numactl-devel-cos7-x86_64 - numba>=0.57 - numpy>=1.23,<2.0a0 @@ -24,13 +24,14 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.12 -- rapids-dask-dependency==24.8.* +- rapids-build-backend>=0.3.0,<0.4.0dev0 +- rapids-dask-dependency==24.8.*,>=0.0.0a0 - setuptools>=64.0.0 - sphinx - sphinx-click>=2.7.1 - sphinx-rtd-theme>=0.5.1 - ucx-proc=*=gpu -- ucx-py==0.39.* -- ucxx==0.39.* +- ucx-py==0.39.*,>=0.0.0a0 +- ucxx==0.39.*,>=0.0.0a0 - zict>=2.0.0 name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index e122bd43c..4db52a6d6 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -11,10 +11,10 @@ dependencies: - cuda-nvcc-impl - cuda-nvrtc - cuda-version=12.2 -- cudf==24.8.* -- dask-cudf==24.8.* -- distributed-ucxx==0.39.* -- kvikio==24.8.* +- cudf==24.8.*,>=0.0.0a0 +- dask-cudf==24.8.*,>=0.0.0a0 +- distributed-ucxx==0.39.*,>=0.0.0a0 +- kvikio==24.8.*,>=0.0.0a0 - numactl-devel-cos7-x86_64 - numba>=0.57 - numpy>=1.23,<2.0a0 @@ -25,13 +25,14 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.12 -- rapids-dask-dependency==24.8.* +- rapids-build-backend>=0.3.0,<0.4.0dev0 +- rapids-dask-dependency==24.8.*,>=0.0.0a0 - setuptools>=64.0.0 - sphinx - sphinx-click>=2.7.1 - sphinx-rtd-theme>=0.5.1 - ucx-proc=*=gpu -- ucx-py==0.39.* -- ucxx==0.39.* +- ucx-py==0.39.*,>=0.0.0a0 +- ucxx==0.39.*,>=0.0.0a0 - zict>=2.0.0 name: all_cuda-122_arch-x86_64 diff --git a/conda/recipes/dask-cuda/meta.yaml b/conda/recipes/dask-cuda/meta.yaml index 357e6dede..877290d4a 100644 --- a/conda/recipes/dask-cuda/meta.yaml +++ b/conda/recipes/dask-cuda/meta.yaml @@ -29,6 +29,7 @@ requirements: host: - python - pip + - rapids-build-backend>=0.3.0,<0.4.0.dev0 run: - python {% for r in data.get("project", {}).get("dependencies", []) %} diff --git a/dask_cuda/_version.py b/dask_cuda/_version.py index c54072ba5..820bf10ba 100644 --- a/dask_cuda/_version.py +++ b/dask_cuda/_version.py @@ -15,6 +15,16 @@ import importlib.resources __version__ = ( - importlib.resources.files("dask_cuda").joinpath("VERSION").read_text().strip() + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/dask_cuda/tests/test_version.py b/dask_cuda/tests/test_version.py new file mode 100644 index 000000000..f30b2847d --- /dev/null +++ b/dask_cuda/tests/test_version.py @@ -0,0 +1,12 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. + +import dask_cuda + + +def test_version_constants_are_populated(): + # __git_commit__ will only be non-empty in a built distribution + assert isinstance(dask_cuda.__git_commit__, str) + + # __version__ should always be non-empty + assert isinstance(dask_cuda.__version__, str) + assert len(dask_cuda.__version__) > 0 diff --git a/dependencies.yaml b/dependencies.yaml index 3a21a1d34..c7f552836 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -74,6 +74,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: + - rapids-build-backend>=0.3.0,<0.4.0dev0 - setuptools>=64.0.0 cuda_version: specific: @@ -153,23 +154,23 @@ dependencies: - numpy>=1.23,<2.0a0 - pandas>=1.3 - pynvml>=11.0.0,<11.5 - - rapids-dask-dependency==24.8.* + - rapids-dask-dependency==24.8.*,>=0.0.0a0 - zict>=2.0.0 test_python: common: - output_types: [conda, requirements, pyproject] packages: - - cudf==24.8.* - - dask-cudf==24.8.* - - kvikio==24.8.* - pytest - pytest-cov - - ucx-py==0.39.* - output_types: [conda] packages: - - distributed-ucxx==0.39.* + - &cudf_conda cudf==24.8.*,>=0.0.0a0 + - &dask_cudf_conda dask-cudf==24.8.*,>=0.0.0a0 + - distributed-ucxx==0.39.*,>=0.0.0a0 + - &kvikio_conda kvikio==24.8.*,>=0.0.0a0 + - &ucx_py_conda ucx-py==0.39.*,>=0.0.0a0 - ucx-proc=*=gpu - - ucxx==0.39.* + - ucxx==0.39.*,>=0.0.0a0 specific: - output_types: conda matrices: @@ -181,3 +182,23 @@ dependencies: arch: aarch64 packages: - numactl-devel-cos7-aarch64 + - output_types: [requirements, pyproject] + matrices: + # kvikio should be added to the CUDA-version-specific matrices once there are wheels available + # ref: https://github.com/rapidsai/kvikio/pull/369 + - matrix: {cuda: "12.*"} + packages: + - cudf-cu12==24.8.*,>=0.0.0a0 + - dask-cudf-cu12==24.8.*,>=0.0.0a0 + - ucx-py-cu12==0.39.*,>=0.0.0a0 + - matrix: {cuda: "11.*"} + packages: + - cudf-cu11==24.8.*,>=0.0.0a0 + - dask-cudf-cu11==24.8.*,>=0.0.0a0 + - ucx-py-cu11==0.39.*,>=0.0.0a0 + - matrix: + packages: + - *cudf_conda + - *dask_cudf_conda + - *kvikio_conda + - *ucx_py_conda diff --git a/pyproject.toml b/pyproject.toml index e0f453818..126efba6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [build-system] -build-backend = "setuptools.build_meta" +build-backend = "rapids_build_backend.build" requires = [ + "rapids-build-backend>=0.3.0,<0.4.0dev0", "setuptools>=64.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. @@ -20,7 +21,7 @@ dependencies = [ "numpy>=1.23,<2.0a0", "pandas>=1.3", "pynvml>=11.0.0,<11.5", - "rapids-dask-dependency==24.8.*", + "rapids-dask-dependency==24.8.*,>=0.0.0a0", "zict>=2.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ @@ -49,12 +50,12 @@ docs = [ "sphinx-rtd-theme>=0.5.1", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. test = [ - "cudf==24.8.*", - "dask-cudf==24.8.*", - "kvikio==24.8.*", + "cudf==24.8.*,>=0.0.0a0", + "dask-cudf==24.8.*,>=0.0.0a0", + "kvikio==24.8.*,>=0.0.0a0", "pytest", "pytest-cov", - "ucx-py==0.39.*", + "ucx-py==0.39.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] @@ -129,6 +130,11 @@ filterwarnings = [ "ignore:Dask DataFrame implementation is deprecated:DeprecationWarning", ] +[tool.rapids-build-backend] +build-backend = "setuptools.build_meta" +dependencies-file = "dependencies.yaml" +disable-cuda = true + [tool.setuptools] license-files = ["LICENSE"]