From d31ad6085393fe96bdc228ea3174b3e7c6088ca3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 4 Jun 2024 20:15:38 -0700 Subject: [PATCH 01/27] use rapids-build-backend --- .pre-commit-config.yaml | 2 +- .readthedocs.yml | 3 -- ci/build_docs_pre_install.sh | 17 ----------- ci/build_python.sh | 8 ++--- ci/build_wheel.sh | 30 +------------------ ci/check_style.sh | 2 +- ci/test_python.sh | 2 +- conda/environments/builddocs.yml | 6 ++++ conda/recipes/ucx-py/meta.yaml | 2 +- dependencies.yaml | 50 +++++++++++++++++++++++--------- pyproject.toml | 20 +++++++++---- 11 files changed, 64 insertions(+), 78 deletions(-) delete mode 100755 ci/build_docs_pre_install.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6955dcbe..c3fa09fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: args: ["--config=.flake8.cython"] types: [cython] - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.8.0 + rev: v1.13.11 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/.readthedocs.yml b/.readthedocs.yml index f672b74d..d72bb73a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,9 +4,6 @@ build: os: "ubuntu-22.04" tools: python: "mambaforge-22.9" - jobs: - post_checkout: - - bash ci/build_docs_pre_install.sh conda: environment: conda/environments/builddocs.yml diff --git a/ci/build_docs_pre_install.sh b/ci/build_docs_pre_install.sh deleted file mode 100755 index 95a38f72..00000000 --- a/ci/build_docs_pre_install.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024 NVIDIA CORPORATION. -# -# [description] -# -# ucx-py's docs builds require installing the library. -# -# It does that by running 'pip install .' from the root of the repo. This script -# is used to modify readthedocs' local checkout of this project's source code prior -# to that 'pip install' being run. -# -# For more, see https://docs.readthedocs.io/en/stable/build-customization.html -# - -set -euo pipefail - -sed -r -i "s/\"libucx/\"libucx-cu12/g" ./pyproject.toml diff --git a/ci/build_python.sh b/ci/build_python.sh index 43dc784c..1e423cc1 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -11,16 +11,12 @@ source rapids-date-string rapids-print-env -version=$(rapids-generate-version) -commit=$(git rev-parse HEAD) - -echo "${version}" > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" ucp/_version.py +rapids-generate-version > ./VERSION rapids-logger "Begin py build" conda config --set path_conflict prevent -RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \ conda/recipes/ucx-py rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 5417f309..b1c4138f 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -22,38 +22,10 @@ rm -rf /usr/include/uct source rapids-configure-sccache source rapids-date-string -version=$(rapids-generate-version) -commit=$(git rev-parse HEAD) +rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="pyproject.toml" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" ucp/_version.py - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -sed -r -i "s/cudf==(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -sed -r -i "/\"libucx([=><]+)/ s/\"libucx/\"libucx${PACKAGE_CUDA_SUFFIX}/g" ${pyproject_file} - -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi - python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist diff --git a/ci/check_style.sh b/ci/check_style.sh index 79496726..d7d3651a 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -8,7 +8,7 @@ rapids-logger "Create checks conda environment" rapids-dependency-file-generator \ --output conda \ - --file_key checks \ + --file-key checks \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n checks diff --git a/ci/test_python.sh b/ci/test_python.sh index b54f0319..c42fb613 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -11,7 +11,7 @@ rapids-logger "Create test conda environment" rapids-dependency-file-generator \ --output conda \ - --file_key test_python \ + --file-key test_python \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-mamba-retry env create --yes -f env.yaml -n test diff --git a/conda/environments/builddocs.yml b/conda/environments/builddocs.yml index cd9766ab..fe806fab 100644 --- a/conda/environments/builddocs.yml +++ b/conda/environments/builddocs.yml @@ -16,5 +16,11 @@ dependencies: - pip - pip: - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + # NOTE: this correctly changes dependencies like 'libucx' to their suffixed, + # existing versions like 'libucx-cu12' via an environment variable set + # in the readthedocs UI: + # + # - RAPIDS_MATRIX_ENTRY="cuda=12.2" + # - ../../ - cython diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 1e6d6ab4..140a2788 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -18,7 +18,7 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} script: - - {{ PYTHON }} -m pip install . -vv + - {{ PYTHON }} -m pip install --config-settings rapidsai.disable-cuda=true . -vv ignore_run_exports_from: - ucx diff --git a/dependencies.yaml b/dependencies.yaml index 8878b254..4344226b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -6,6 +6,7 @@ files: - checks - cuda - cuda_version + - rapids_build_setuptools - py_version - run - test_python @@ -25,6 +26,14 @@ files: pyproject_dir: . extras: table: build-system + includes: + - rapids_build_setuptools + py_rapids_build: + output: pyproject + pyproject_dir: . + extras: + table: tool.rapids-build-backend + key: requires includes: - build_python - depends_on_ucx_build @@ -109,8 +118,13 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - setuptools>=64.0.0 - cython>=3.0.0 + rapids_build_setuptools: + common: + - output_types: [conda, requirements, pyproject] + packages: + - rapids-build-backend>=0.3.0,<0.4.0dev0 + - setuptools>=64.0.0 run: common: - output_types: [conda, requirements, pyproject] @@ -132,15 +146,13 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - libucx-cu12==1.15.0 + - libucx-cu12==1.15.0,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - libucx-cu11==1.15.0 - # NOTE: this fallback needs to be a real, suffixed version - # so 'pip install .' (e.g. as used in docs builds) will work + - libucx-cu11==1.15.0,>=0.0.0a0 - matrix: null packages: - - libucx==1.15.0 + - libucx==1.15.0,>=0.0.0a0 depends_on_ucx_run: common: - output_types: conda @@ -156,21 +168,19 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - libucx-cu12>=1.15.0,<1.16 + - libucx-cu12>=1.15.0,<1.16,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - libucx-cu11>=1.15.0,<1.16 - # NOTE: this fallback needs to be a real, suffixed version - # so "pip install ." (e.g. as used in docs builds) will work + - libucx-cu11>=1.15.0,<1.16,>=0.0.0a0 - matrix: null packages: - - libucx>=1.15.0,<1.16 + - libucx>=1.15.0,<1.16,>=0.0.0a0 test_python: common: - output_types: [conda, requirements, pyproject] packages: - cloudpickle - - cudf==24.8.* + - &cudf_conda cudf==24.8.*,>=0.0.0a0 - dask - distributed - numba>=0.57 @@ -180,6 +190,18 @@ dependencies: - output_types: [conda] packages: - cupy>=12.0.0 + specific: - output_types: [requirements, pyproject] - packages: - - cupy-cuda11x>=12.0.0 + matrices: + - matrix: {cuda: "12.*"} + packages: + - cudf-cu12==24.8.*,>=0.0.0a0 + - cupy-cuda12x>=12.0.0 + - matrix: {cuda: "11.*"} + packages: + - cudf-cu11==24.8.*,>=0.0.0a0 + - &cupy_cu11 cupy-cuda11x>=12.0.0 + - matrix: + packages: + - *cudf_conda + - *cupy_cu11 diff --git a/pyproject.toml b/pyproject.toml index 11527954..5ec9b92a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,9 @@ # limitations under the License. [build-system] -build-backend = "setuptools.build_meta" +build-backend = "rapids_build_backend.build" requires = [ - "cython>=3.0.0", - "libucx==1.15.0", + "rapids-build-backend>=0.3.0,<0.4.0dev0", "setuptools>=64.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. @@ -31,7 +30,7 @@ authors = [ license = { text = "BSD-3-Clause" } requires-python = ">=3.9" dependencies = [ - "libucx>=1.15.0,<1.16", + "libucx>=1.15.0,<1.16,>=0.0.0a0", "numpy>=1.23,<2.0a0", "pynvml>=11.4.1", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. @@ -50,7 +49,7 @@ classifiers = [ [project.optional-dependencies] test = [ "cloudpickle", - "cudf==24.8.*", + "cudf==24.8.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", "dask", "distributed", @@ -112,6 +111,17 @@ skip = [ [tool.pytest.ini_options] xfail_strict = true +[tool.rapids-build-backend] +build-backend = "setuptools.build_meta" +commit-files = [ + "ucp/COMMIT_FILE" +] +dependencies-file = "dependencies.yaml" +requires = [ + "cython>=3.0.0", + "libucx==1.15.0,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. + [tool.setuptools] license-files = ["LICENSE"] zip-safe = false From bfdc82376119e3015ac326bcb2c0293808fe00df Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 4 Jun 2024 21:06:32 -0700 Subject: [PATCH 02/27] fix dependencies --- conda/environments/builddocs.yml | 5 +++-- conda/recipes/ucx-py/meta.yaml | 1 + dependencies.yaml | 13 ------------- pyproject.toml | 6 ++---- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/conda/environments/builddocs.yml b/conda/environments/builddocs.yml index fe806fab..af07f1cc 100644 --- a/conda/environments/builddocs.yml +++ b/conda/environments/builddocs.yml @@ -17,9 +17,10 @@ dependencies: - pip: - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple # NOTE: this correctly changes dependencies like 'libucx' to their suffixed, - # existing versions like 'libucx-cu12' via an environment variable set + # existing versions like 'libucx-cu12' via an environment variables set # in the readthedocs UI: - # + # + # - RAPIDS_DISABLE_CUDA=true # - RAPIDS_MATRIX_ENTRY="cuda=12.2" # - ../../ diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 140a2788..248caf58 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -29,6 +29,7 @@ requirements: host: - python - pip + - rapids-build-backend >=0.3.0,<0.4.0dev0 - ucx # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead {% for r in data.get("build-system", {}).get("requires", []) if not r.startswith("libucx") %} diff --git a/dependencies.yaml b/dependencies.yaml index 4344226b..6c899215 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -6,7 +6,6 @@ files: - checks - cuda - cuda_version - - rapids_build_setuptools - py_version - run - test_python @@ -26,14 +25,6 @@ files: pyproject_dir: . extras: table: build-system - includes: - - rapids_build_setuptools - py_rapids_build: - output: pyproject - pyproject_dir: . - extras: - table: tool.rapids-build-backend - key: requires includes: - build_python - depends_on_ucx_build @@ -119,10 +110,6 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - cython>=3.0.0 - rapids_build_setuptools: - common: - - output_types: [conda, requirements, pyproject] - packages: - rapids-build-backend>=0.3.0,<0.4.0dev0 - setuptools>=64.0.0 run: diff --git a/pyproject.toml b/pyproject.toml index 5ec9b92a..7b618ddb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,8 @@ [build-system] build-backend = "rapids_build_backend.build" requires = [ + "cython>=3.0.0", + "libucx==1.15.0,>=0.0.0a0", "rapids-build-backend>=0.3.0,<0.4.0dev0", "setuptools>=64.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. @@ -117,10 +119,6 @@ commit-files = [ "ucp/COMMIT_FILE" ] dependencies-file = "dependencies.yaml" -requires = [ - "cython>=3.0.0", - "libucx==1.15.0,>=0.0.0a0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. [tool.setuptools] license-files = ["LICENSE"] From 745b40f3de4802972407b498113e70c864821ac5 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 4 Jun 2024 21:09:16 -0700 Subject: [PATCH 03/27] version tests --- tests/test_version.py | 5 +++++ ucp/_version.py | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/test_version.py b/tests/test_version.py index ffb5152e..1b77632e 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -9,6 +9,11 @@ def test_get_ucx_version(): assert ucp.core._ctx is None +def test_git_commit_constant(): + # __git_commit__ will only be non-empty in a built distribution + assert isinstance(ucp.__git_commit__, str) + + def test_version_constant(): assert isinstance(ucp.__version__, str) diff --git a/ucp/_version.py b/ucp/_version.py index ab2f6e9f..3aa963c1 100644 --- a/ucp/_version.py +++ b/ucp/_version.py @@ -14,5 +14,18 @@ import importlib.resources -__version__ = importlib.resources.files("ucp").joinpath("VERSION").read_text().strip() -__git_commit__ = "" +__version__ = ( + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() +) + +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] From 6cf9e4e64d1cf075708a3aff54b83d8ad71551e3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 5 Jun 2024 10:35:05 -0700 Subject: [PATCH 04/27] changes --- ci/build_wheel.sh | 2 +- dependencies.yaml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index b1c4138f..d787b8c1 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -26,7 +26,7 @@ rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +python -m pip wheel . -w dist -v --no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair \ diff --git a/dependencies.yaml b/dependencies.yaml index 6c899215..8f08e1f8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -133,13 +133,13 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - libucx-cu12==1.15.0,>=0.0.0a0 + - libucx-cu12==1.15.0 - matrix: {cuda: "11.*"} packages: - - libucx-cu11==1.15.0,>=0.0.0a0 + - libucx-cu11==1.15.0 - matrix: null packages: - - libucx==1.15.0,>=0.0.0a0 + - libucx==1.15.0 depends_on_ucx_run: common: - output_types: conda @@ -155,13 +155,13 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - libucx-cu12>=1.15.0,<1.16,>=0.0.0a0 + - libucx-cu12>=1.15.0,<1.16 - matrix: {cuda: "11.*"} packages: - - libucx-cu11>=1.15.0,<1.16,>=0.0.0a0 + - libucx-cu11>=1.15.0,<1.16 - matrix: null packages: - - libucx>=1.15.0,<1.16,>=0.0.0a0 + - libucx>=1.15.0,<1.16 test_python: common: - output_types: [conda, requirements, pyproject] From 74dbf00f53d71a80bf573f8450b76d5abff7e5af Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 5 Jun 2024 10:44:49 -0700 Subject: [PATCH 05/27] fix --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7b618ddb..72ac44f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ build-backend = "rapids_build_backend.build" requires = [ "cython>=3.0.0", - "libucx==1.15.0,>=0.0.0a0", + "libucx==1.15.0", "rapids-build-backend>=0.3.0,<0.4.0dev0", "setuptools>=64.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. @@ -32,7 +32,7 @@ authors = [ license = { text = "BSD-3-Clause" } requires-python = ">=3.9" dependencies = [ - "libucx>=1.15.0,<1.16,>=0.0.0a0", + "libucx>=1.15.0,<1.16", "numpy>=1.23,<2.0a0", "pynvml>=11.4.1", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. From 663320d9be296a8df9e9f3c2f4fb44e8cca5ce51 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 5 Jun 2024 10:57:11 -0700 Subject: [PATCH 06/27] just run wheel builds --- .github/workflows/pr.yaml | 48 +++++++++++++++++++-------------------- ci/build_wheel.sh | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4a142c90..2cf351bb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,28 +13,28 @@ jobs: pr-builder: needs: - checks - - conda-python-build - - conda-python-tests + # - conda-python-build + # - conda-python-tests - wheel-build - - wheel-tests + # - wheel-tests secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08 - conda-python-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 - with: - build_type: pull-request - conda-python-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 - with: - build_type: pull-request - container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" + # conda-python-build: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + # with: + # build_type: pull-request + # conda-python-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + # with: + # build_type: pull-request + # container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" wheel-build: needs: checks secrets: inherit @@ -42,11 +42,11 @@ jobs: with: build_type: pull-request script: ci/build_wheel.sh - wheel-tests: - needs: wheel-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 - with: - build_type: pull-request - script: ci/test_wheel.sh - container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" + # wheel-tests: + # needs: wheel-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + # with: + # build_type: pull-request + # script: ci/test_wheel.sh + # container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index d787b8c1..0097a7fd 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -26,7 +26,7 @@ rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -python -m pip wheel . -w dist -v --no-deps --disable-pip-version-check +python -m pip wheel . -w dist--no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair \ From 2f06f89fe36ed9713502a57792489c122a6a36da Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 5 Jun 2024 11:42:09 -0700 Subject: [PATCH 07/27] try moving libucx dependency --- dependencies.yaml | 7 +++++++ pyproject.toml | 4 +++- setup.py | 5 ++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 8f08e1f8..532d2597 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -27,6 +27,13 @@ files: table: build-system includes: - build_python + py_rapids_build: + output: pyproject + pyproject_dir: . + extras: + table: tool.rapids-build-backend + key: requires + includes: - depends_on_ucx_build py_run: output: pyproject diff --git a/pyproject.toml b/pyproject.toml index 72ac44f9..6760baa3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ build-backend = "rapids_build_backend.build" requires = [ "cython>=3.0.0", - "libucx==1.15.0", "rapids-build-backend>=0.3.0,<0.4.0dev0", "setuptools>=64.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. @@ -119,6 +118,9 @@ commit-files = [ "ucp/COMMIT_FILE" ] dependencies-file = "dependencies.yaml" +requires = [ + "libucx==1.15.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. [tool.setuptools] license-files = ["LICENSE"] diff --git a/setup.py b/setup.py index 2c238c3c..46db1f38 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,9 @@ import os from distutils.sysconfig import get_config_var, get_python_inc +# TODO: delete this before merging. Just checking if this has to be available +# when setup.py is run. +import libucx # noqa: F401 from Cython.Distutils.build_ext import new_build_ext from setuptools import setup from setuptools.extension import Extension @@ -25,7 +28,7 @@ def _find_libucx_libs_and_headers(): be compiled against those libucx-wheel-provided versions of the UCX libraries. """ try: - import libucx + import libucx # noqa: F811 except ImportError: return [], [] From 369d57ef5bfd884b41a5bdaea3b0c6b5b60335de Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 5 Jun 2024 11:56:06 -0700 Subject: [PATCH 08/27] put libucx back, it is needed in setup.py --- dependencies.yaml | 7 ------- pyproject.toml | 1 + setup.py | 3 --- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 532d2597..8f08e1f8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -27,13 +27,6 @@ files: table: build-system includes: - build_python - py_rapids_build: - output: pyproject - pyproject_dir: . - extras: - table: tool.rapids-build-backend - key: requires - includes: - depends_on_ucx_build py_run: output: pyproject diff --git a/pyproject.toml b/pyproject.toml index 6760baa3..6e9e68ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ build-backend = "rapids_build_backend.build" requires = [ "cython>=3.0.0", + "libucx==1.15.0", "rapids-build-backend>=0.3.0,<0.4.0dev0", "setuptools>=64.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/setup.py b/setup.py index 46db1f38..8e73cd74 100644 --- a/setup.py +++ b/setup.py @@ -9,9 +9,6 @@ import os from distutils.sysconfig import get_config_var, get_python_inc -# TODO: delete this before merging. Just checking if this has to be available -# when setup.py is run. -import libucx # noqa: F401 from Cython.Distutils.build_ext import new_build_ext from setuptools import setup from setuptools.extension import Extension From 5fd0b04c77df8f54485bb7653a360d4af5078611 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 10 Jun 2024 11:33:13 -0700 Subject: [PATCH 09/27] test with https://github.com/rapidsai/rapids-build-backend/pull/41 --- ci/build_python.sh | 13 +++++++++++++ ci/build_wheel.sh | 13 +++++++++++++ conda/recipes/ucx-py/meta.yaml | 2 +- dependencies.yaml | 2 +- pyproject.toml | 2 +- 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 1e423cc1..8d6fc3d2 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -11,6 +11,19 @@ source rapids-date-string rapids-print-env +# TODO: remove before merging (when new rapids-build-backend is released) +git clone \ + -b setuptools \ + https://github.com/jameslamb/rapids-build-backend.git \ + /tmp/delete-me/rapids-build-backend + +pushd /tmp/delete-me/rapids-build-backend +sed -e 's/^version =.*/version = "0.3.1"/' -i pyproject.toml +python -m pip install . +popd + +export PIP_FIND_LINKS="file:///tmp/delete-me/rapids-build-backend/dist" + rapids-generate-version > ./VERSION rapids-logger "Begin py build" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 0097a7fd..0a02482f 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -26,6 +26,19 @@ rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +# TODO: remove before merging (when new rapids-build-backend is released) +git clone \ + -b setuptools \ + https://github.com/jameslamb/rapids-build-backend.git \ + /tmp/delete-me/rapids-build-backend + +pushd /tmp/delete-me/rapids-build-backend +sed -e 's/^version =.*/version = "0.3.1"/' -i pyproject.toml +python -m pip install . +popd + +export PIP_FIND_LINKS="file:///tmp/delete-me/rapids-build-backend/dist" + python -m pip wheel . -w dist--no-deps --disable-pip-version-check mkdir -p final_dist diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 248caf58..25a5eaa7 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -29,7 +29,7 @@ requirements: host: - python - pip - - rapids-build-backend >=0.3.0,<0.4.0dev0 + - rapids-build-backend >=0.3.1,<0.4.0dev0 - ucx # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead {% for r in data.get("build-system", {}).get("requires", []) if not r.startswith("libucx") %} diff --git a/dependencies.yaml b/dependencies.yaml index 8f08e1f8..2660eb21 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -110,7 +110,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - cython>=3.0.0 - - rapids-build-backend>=0.3.0,<0.4.0dev0 + - rapids-build-backend>=0.3.1,<0.4.0dev0 - setuptools>=64.0.0 run: common: diff --git a/pyproject.toml b/pyproject.toml index 6e9e68ad..1a9b071f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ build-backend = "rapids_build_backend.build" requires = [ "cython>=3.0.0", "libucx==1.15.0", - "rapids-build-backend>=0.3.0,<0.4.0dev0", + "rapids-build-backend>=0.3.1,<0.4.0dev0", "setuptools>=64.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. From 893ff43c68f74e98cee657f902e8b28663c925f3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 10 Jun 2024 11:38:20 -0700 Subject: [PATCH 10/27] move other dependencies down to rapids-build-backend's table --- dependencies.yaml | 13 +++++++++++++ pyproject.toml | 3 +-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 2660eb21..7e8917ae 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -7,6 +7,7 @@ files: - cuda - cuda_version - py_version + - rapids_build_setuptools - run - test_python test_python: @@ -25,6 +26,14 @@ files: pyproject_dir: . extras: table: build-system + includes: + - rapids_build_setuptools + py_rapids_build: + output: pyproject + pyproject_dir: . + extras: + table: tool.rapids-build-backend + key: requires includes: - build_python - depends_on_ucx_build @@ -110,6 +119,10 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - cython>=3.0.0 + rapids_build_setuptools: + common: + - output_types: [conda, requirements, pyproject] + packages: - rapids-build-backend>=0.3.1,<0.4.0dev0 - setuptools>=64.0.0 run: diff --git a/pyproject.toml b/pyproject.toml index 1a9b071f..18cc55e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,6 @@ [build-system] build-backend = "rapids_build_backend.build" requires = [ - "cython>=3.0.0", - "libucx==1.15.0", "rapids-build-backend>=0.3.1,<0.4.0dev0", "setuptools>=64.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. @@ -120,6 +118,7 @@ commit-files = [ ] dependencies-file = "dependencies.yaml" requires = [ + "cython>=3.0.0", "libucx==1.15.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. From 2e2889fb9d37b8fabc6d5c19e3288c3e904a71f1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 10 Jun 2024 11:50:49 -0700 Subject: [PATCH 11/27] wheel, not install --- ci/build_python.sh | 2 +- ci/build_wheel.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 8d6fc3d2..3e00dd5f 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -19,7 +19,7 @@ git clone \ pushd /tmp/delete-me/rapids-build-backend sed -e 's/^version =.*/version = "0.3.1"/' -i pyproject.toml -python -m pip install . +python -m pip wheel --wheel-dir ./dist popd export PIP_FIND_LINKS="file:///tmp/delete-me/rapids-build-backend/dist" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 0a02482f..1d30260b 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -34,7 +34,7 @@ git clone \ pushd /tmp/delete-me/rapids-build-backend sed -e 's/^version =.*/version = "0.3.1"/' -i pyproject.toml -python -m pip install . +python -m pip wheel --wheel-dir ./dist popd export PIP_FIND_LINKS="file:///tmp/delete-me/rapids-build-backend/dist" From 6c9fb3fc9314c3aadb2d9b0e7e77a32a46c5c903 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 10 Jun 2024 12:10:51 -0700 Subject: [PATCH 12/27] fix pip wheel invocation --- ci/build_python.sh | 2 +- ci/build_wheel.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 3e00dd5f..5e3283ff 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -19,7 +19,7 @@ git clone \ pushd /tmp/delete-me/rapids-build-backend sed -e 's/^version =.*/version = "0.3.1"/' -i pyproject.toml -python -m pip wheel --wheel-dir ./dist +python -m pip wheel --wheel-dir ./dist . popd export PIP_FIND_LINKS="file:///tmp/delete-me/rapids-build-backend/dist" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 1d30260b..b643efea 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -34,7 +34,7 @@ git clone \ pushd /tmp/delete-me/rapids-build-backend sed -e 's/^version =.*/version = "0.3.1"/' -i pyproject.toml -python -m pip wheel --wheel-dir ./dist +python -m pip wheel --wheel-dir ./dist . popd export PIP_FIND_LINKS="file:///tmp/delete-me/rapids-build-backend/dist" From 15fde531adae09fa1f427fb7cfbcea1fc1233e87 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 10 Jun 2024 12:23:03 -0700 Subject: [PATCH 13/27] get better logs --- ci/build_python.sh | 2 +- ci/build_wheel.sh | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 5e3283ff..d7134dbf 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION. -set -euo pipefail +set -euo pipefail -x rapids-configure-conda-channels diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index b643efea..8193c216 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION. -set -euo pipefail +set -euo pipefail -x package_name="ucx-py" underscore_package_name=$(echo "${package_name}" | tr "-" "_") @@ -26,6 +26,12 @@ rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +cat /etc/xdg/pip/pip.conf < Date: Mon, 10 Jun 2024 12:31:48 -0700 Subject: [PATCH 14/27] fix pip wheel invocation --- ci/build_wheel.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 8193c216..e1593671 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -26,7 +26,7 @@ rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -cat /etc/xdg/pip/pip.conf < /etc/xdg/pip/pip.conf < Date: Mon, 10 Jun 2024 12:46:30 -0700 Subject: [PATCH 15/27] re-enable wheel tests --- .github/workflows/pr.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2cf351bb..3e57d74c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,7 +16,7 @@ jobs: # - conda-python-build # - conda-python-tests - wheel-build - # - wheel-tests + - wheel-tests secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 checks: @@ -42,11 +42,11 @@ jobs: with: build_type: pull-request script: ci/build_wheel.sh - # wheel-tests: - # needs: wheel-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 - # with: - # build_type: pull-request - # script: ci/test_wheel.sh - # container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" + wheel-tests: + needs: wheel-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + with: + build_type: pull-request + script: ci/test_wheel.sh + container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" From 473f306dd8b2f489d5cb9d2e076ddf91cf5822d6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 10 Jun 2024 13:02:13 -0700 Subject: [PATCH 16/27] move cudf dependency --- dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 7e8917ae..6fd64d4c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -180,7 +180,6 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - cloudpickle - - &cudf_conda cudf==24.8.*,>=0.0.0a0 - dask - distributed - numba>=0.57 @@ -189,6 +188,7 @@ dependencies: - pytest-rerunfailures - output_types: [conda] packages: + - &cudf_conda cudf==24.8.*,>=0.0.0a0 - cupy>=12.0.0 specific: - output_types: [requirements, pyproject] From 0dddea797da9f8fc0851db23be4d002ae8a7d28e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 11 Jun 2024 09:21:27 -0700 Subject: [PATCH 17/27] empty commit to re-trigger CI From 772ac92c4b2a956c61dd2666b9b0cc22b000fee2 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 12 Jun 2024 06:54:37 -0700 Subject: [PATCH 18/27] use rapids-build-backend packages --- ci/build_python.sh | 13 ------------- ci/build_wheel.sh | 19 ------------------- 2 files changed, 32 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index d7134dbf..e5df9dce 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -11,19 +11,6 @@ source rapids-date-string rapids-print-env -# TODO: remove before merging (when new rapids-build-backend is released) -git clone \ - -b setuptools \ - https://github.com/jameslamb/rapids-build-backend.git \ - /tmp/delete-me/rapids-build-backend - -pushd /tmp/delete-me/rapids-build-backend -sed -e 's/^version =.*/version = "0.3.1"/' -i pyproject.toml -python -m pip wheel --wheel-dir ./dist . -popd - -export PIP_FIND_LINKS="file:///tmp/delete-me/rapids-build-backend/dist" - rapids-generate-version > ./VERSION rapids-logger "Begin py build" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index e1593671..6017c855 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -26,25 +26,6 @@ rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -cat > /etc/xdg/pip/pip.conf < Date: Wed, 12 Jun 2024 07:09:18 -0700 Subject: [PATCH 19/27] re-enable conda builds --- .github/workflows/pr.yaml | 30 +++++++++++++++--------------- ci/build_python.sh | 2 +- ci/build_wheel.sh | 2 +- ci/release/update-version.sh | 2 +- conda/environments/builddocs.yml | 13 ++++++++----- conda/recipes/ucx-py/meta.yaml | 10 ++++++---- setup.py | 2 +- 7 files changed, 33 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3e57d74c..4a142c90 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,8 +13,8 @@ jobs: pr-builder: needs: - checks - # - conda-python-build - # - conda-python-tests + - conda-python-build + - conda-python-tests - wheel-build - wheel-tests secrets: inherit @@ -22,19 +22,19 @@ jobs: checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08 - # conda-python-build: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 - # with: - # build_type: pull-request - # conda-python-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 - # with: - # build_type: pull-request - # container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" + conda-python-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.08 + with: + build_type: pull-request + conda-python-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + with: + build_type: pull-request + container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" wheel-build: needs: checks secrets: inherit diff --git a/ci/build_python.sh b/ci/build_python.sh index e5df9dce..1e423cc1 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION. -set -euo pipefail -x +set -euo pipefail rapids-configure-conda-channels diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 6017c855..747018f3 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (c) 2023, NVIDIA CORPORATION. -set -euo pipefail -x +set -euo pipefail package_name="ucx-py" underscore_package_name=$(echo "${package_name}" | tr "-" "_") diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 3f2d2e43..b872e6c8 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -42,7 +42,7 @@ DEPENDENCIES=( ) for FILE in dependencies.yaml; do for DEP in "${DEPENDENCIES[@]}"; do - sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*/g" "${FILE}" + sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" done done diff --git a/conda/environments/builddocs.yml b/conda/environments/builddocs.yml index af07f1cc..596a92ce 100644 --- a/conda/environments/builddocs.yml +++ b/conda/environments/builddocs.yml @@ -16,12 +16,15 @@ dependencies: - pip - pip: - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - # NOTE: this correctly changes dependencies like 'libucx' to their suffixed, - # existing versions like 'libucx-cu12' via an environment variables set - # in the readthedocs UI: + # NOTE: this 'pip install' working correctly depends on two environment variables + # used by rapids-build-backend, set in the readthedocs UI # - # - RAPIDS_DISABLE_CUDA=true - # - RAPIDS_MATRIX_ENTRY="cuda=12.2" + # # do not try to run 'nvcc --version' to determine CUDA version + # RAPIDS_DISABLE_CUDA=true + # + # # install CUDA-12-specific dependencies + # # (e.g. libucx-cu12) + # RAPIDS_MATRIX_ENTRY="cuda=12.2" # - ../../ - cython diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 25a5eaa7..31d51ce3 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -29,17 +29,19 @@ requirements: host: - python - pip - - rapids-build-backend >=0.3.1,<0.4.0dev0 - ucx + {% for b_r in data["build-system"]["requires"] %} + - {{ b_r }} + {% endfor %} # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead - {% for r in data.get("build-system", {}).get("requires", []) if not r.startswith("libucx") %} - - {{ r }} + {% for rbb_r in data["tool.rapids-build-backend"]["requires"] if not r.startswith("libucx") %} + - {{ rbb_r }} {% endfor %} run: - python - ucx >=1.15.0,<1.16.0 # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead - {% for r in data.get("project", {}).get("dependencies", []) if not r.startswith("libucx") %} + {% for r in data["project"]["dependencies"] if not r.startswith("libucx") %} - {{ r }} {% endfor %} diff --git a/setup.py b/setup.py index 8e73cd74..2c238c3c 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def _find_libucx_libs_and_headers(): be compiled against those libucx-wheel-provided versions of the UCX libraries. """ try: - import libucx # noqa: F811 + import libucx except ImportError: return [], [] From d6241e5af9fd85b281b988645ce2be73e800746e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 12 Jun 2024 07:29:01 -0700 Subject: [PATCH 20/27] fix data-gathering in conda recipe --- conda/recipes/ucx-py/meta.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 31d51ce3..1f840729 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -34,7 +34,7 @@ requirements: - {{ b_r }} {% endfor %} # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead - {% for rbb_r in data["tool.rapids-build-backend"]["requires"] if not r.startswith("libucx") %} + {% for rbb_r in data["tool"]["rapids-build-backend"]["requires"] if not r.startswith("libucx") %} - {{ rbb_r }} {% endfor %} run: @@ -50,12 +50,12 @@ test: - ucp about: - home: {{ data.get("project", {}).get("urls", {}).get("Homepage", "") }} - license: {{ data.get("project", {}).get("license", {}).get("text", "") }} + home: {{ data["project"]["urls"]["Homepage"] }} + license: {{ data["project"]["license"]["text"] }} license_file: - {% for e in data.get("tool", {}).get("setuptools", {}).get("license-files", []) %} + {% for e in data["tool"]["setuptools"]["license-files"] %} - ../../../{{ e }} {% endfor %} - summary: {{ data.get("project", {}).get("description", "") }} - dev_url: {{ data.get("project", {}).get("urls", {}).get("Source", "") }} - doc_url: {{ data.get("project", {}).get("urls", {}).get("Documentation", "") }} + summary: {{ data["project"]["description"] }} + dev_url: {{ data["project"]["urls"]["Source"] }} + doc_url: {{ data["project"]["urls"]["Documentation"] }} From 39960ddd269bfbd3e4e40df82cf58d79b0017edc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 12 Jun 2024 07:40:12 -0700 Subject: [PATCH 21/27] fix variable name --- conda/recipes/ucx-py/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 1f840729..480fc075 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -34,7 +34,7 @@ requirements: - {{ b_r }} {% endfor %} # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead - {% for rbb_r in data["tool"]["rapids-build-backend"]["requires"] if not r.startswith("libucx") %} + {% for rbb_r in data["tool"]["rapids-build-backend"]["requires"] if not rbb_r.startswith("libucx") %} - {{ rbb_r }} {% endfor %} run: From 8a443becfd1b8f0814b4c2125722eb0c961a551a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 12 Jun 2024 10:52:31 -0500 Subject: [PATCH 22/27] Apply suggestions from code review Co-authored-by: Bradley Dice --- conda/recipes/ucx-py/meta.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/recipes/ucx-py/meta.yaml b/conda/recipes/ucx-py/meta.yaml index 480fc075..da667d48 100644 --- a/conda/recipes/ucx-py/meta.yaml +++ b/conda/recipes/ucx-py/meta.yaml @@ -30,19 +30,19 @@ requirements: - python - pip - ucx - {% for b_r in data["build-system"]["requires"] %} - - {{ b_r }} + {% for build_req in data["build-system"]["requires"] %} + - {{ build_req }} {% endfor %} # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead - {% for rbb_r in data["tool"]["rapids-build-backend"]["requires"] if not rbb_r.startswith("libucx") %} - - {{ rbb_r }} + {% for rbb_req in data["tool"]["rapids-build-backend"]["requires"] if not rbb_req.startswith("libucx") %} + - {{ rbb_req }} {% endfor %} run: - python - ucx >=1.15.0,<1.16.0 # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead - {% for r in data["project"]["dependencies"] if not r.startswith("libucx") %} - - {{ r }} + {% for req in data["project"]["dependencies"] if not req.startswith("libucx") %} + - {{ req }} {% endfor %} test: From 3238af2623f9f52434e9a8efbec8473c907ad24a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 12 Jun 2024 09:10:38 -0700 Subject: [PATCH 23/27] try library install in post_create_environment --- .readthedocs.yml | 2 ++ conda/environments/builddocs.yml | 13 ------------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index d72bb73a..99f55db9 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,6 +4,8 @@ build: os: "ubuntu-22.04" tools: python: "mambaforge-22.9" + post_create_environment: + - pip install -C rapidsai.disable-cuda=true -C rapidsai.matrix-entry="cuda-12.2" . conda: environment: conda/environments/builddocs.yml diff --git a/conda/environments/builddocs.yml b/conda/environments/builddocs.yml index 596a92ce..14419853 100644 --- a/conda/environments/builddocs.yml +++ b/conda/environments/builddocs.yml @@ -14,17 +14,4 @@ dependencies: - recommonmark - pandoc=<2.0.0 - pip -- pip: - - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - # NOTE: this 'pip install' working correctly depends on two environment variables - # used by rapids-build-backend, set in the readthedocs UI - # - # # do not try to run 'nvcc --version' to determine CUDA version - # RAPIDS_DISABLE_CUDA=true - # - # # install CUDA-12-specific dependencies - # # (e.g. libucx-cu12) - # RAPIDS_MATRIX_ENTRY="cuda=12.2" - # - - ../../ - cython From dbd572282c6abea9603bb6d6d7f3071eec299a53 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 12 Jun 2024 09:12:48 -0700 Subject: [PATCH 24/27] fix keys --- .readthedocs.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 99f55db9..236b7203 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,8 +4,9 @@ build: os: "ubuntu-22.04" tools: python: "mambaforge-22.9" - post_create_environment: - - pip install -C rapidsai.disable-cuda=true -C rapidsai.matrix-entry="cuda-12.2" . + jobs: + post_create_environment: + - pip install -C rapidsai.disable-cuda=true -C rapidsai.matrix-entry="cuda-12.2" . conda: environment: conda/environments/builddocs.yml From 51c0153e9c4750976454890982623e9fdf9fdb99 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 12 Jun 2024 09:16:32 -0700 Subject: [PATCH 25/27] consider rapids nightly index --- .readthedocs.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 236b7203..68e99e0a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,8 +5,12 @@ build: tools: python: "mambaforge-22.9" jobs: - post_create_environment: - - pip install -C rapidsai.disable-cuda=true -C rapidsai.matrix-entry="cuda-12.2" . + post_create_environment: | + pip install \ + --extra-index-url "https://pypi.anaconda.org/rapidsai-wheels-nightly/simple/" \ + -C rapidsai.disable-cuda=true \ + -C rapidsai.matrix-entry="cuda-12.2" \ + . conda: environment: conda/environments/builddocs.yml From ec63083f6a183fad4293b84fe01487d56f409fae Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 12 Jun 2024 09:22:46 -0700 Subject: [PATCH 26/27] fix formatting --- .readthedocs.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 68e99e0a..e2471eae 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,12 +5,13 @@ build: tools: python: "mambaforge-22.9" jobs: - post_create_environment: | - pip install \ - --extra-index-url "https://pypi.anaconda.org/rapidsai-wheels-nightly/simple/" \ - -C rapidsai.disable-cuda=true \ - -C rapidsai.matrix-entry="cuda-12.2" \ - . + post_create_environment: + - | + pip install \ + --extra-index-url "https://pypi.anaconda.org/rapidsai-wheels-nightly/simple/" \ + -C rapidsai.disable-cuda=true \ + -C rapidsai.matrix-entry="cuda-12.2" \ + . conda: environment: conda/environments/builddocs.yml From 5de66ecac646668c1c0eb6e6ab974affb107d5ea Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 12 Jun 2024 09:33:46 -0700 Subject: [PATCH 27/27] use same __version__ test as other libraries, fix rtd config --- .readthedocs.yml | 2 +- tests/test_version.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index e2471eae..427cf5b1 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,7 +10,7 @@ build: pip install \ --extra-index-url "https://pypi.anaconda.org/rapidsai-wheels-nightly/simple/" \ -C rapidsai.disable-cuda=true \ - -C rapidsai.matrix-entry="cuda-12.2" \ + -C rapidsai.matrix-entry="cuda=12.2" \ . conda: diff --git a/tests/test_version.py b/tests/test_version.py index 1b77632e..205e9fbb 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -9,13 +9,13 @@ def test_get_ucx_version(): assert ucp.core._ctx is None -def test_git_commit_constant(): +def test_version_constants_are_populated(): # __git_commit__ will only be non-empty in a built distribution assert isinstance(ucp.__git_commit__, str) - -def test_version_constant(): + # __version__ should always be non-empty assert isinstance(ucp.__version__, str) + assert len(ucp.__version__) > 0 def test_ucx_version_constant():