From 15c04666e20712f30882a903bfad79e208d8e852 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 24 May 2024 16:55:50 -0500 Subject: [PATCH] Use rapids-build-backend. (#1502) This PR uses `rapids-build-backend` to simplify wheel builds and reduce the complexity of various CI/build scripts. See also: - https://github.com/rapidsai/rapids-build-backend - https://github.com/rapidsai/build-planning/issues/31 - https://github.com/rapidsai/cudf/pull/15245 - #1529 Authors: - Bradley Dice (https://github.com/bdice) - James Lamb (https://github.com/jameslamb) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/rmm/pull/1502 --- .pre-commit-config.yaml | 2 +- build.sh | 6 ++- ci/build_python.sh | 13 ++----- ci/build_wheel_cpp.sh | 14 +------ ci/build_wheel_python.sh | 26 +------------ .../all_cuda-118_arch-x86_64.yaml | 4 +- .../all_cuda-122_arch-x86_64.yaml | 4 +- conda/recipes/rmm/meta.yaml | 1 + dependencies.yaml | 38 +++++++++++++++---- python/librmm/librmm/_version.py | 7 +++- python/librmm/pyproject.toml | 14 +++++-- python/rmm/pyproject.toml | 20 +++++++--- python/rmm/rmm/_version.py | 19 ++++++++-- python/rmm/rmm/tests/test_version.py | 24 ++++++++++++ 14 files changed, 121 insertions(+), 71 deletions(-) create mode 100644 python/rmm/rmm/tests/test_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d21fcebf8..4155cdc19 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.8.0 + rev: v1.13.11 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/build.sh b/build.sh index 70da14b9b..afd26e9e6 100755 --- a/build.sh +++ b/build.sh @@ -170,5 +170,9 @@ fi # Build and install the rmm Python package if (( NUMARGS == 0 )) || hasArg rmm; then echo "building and installing rmm..." - SKBUILD_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/rmm + SKBUILD_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" python -m pip install \ + --no-build-isolation \ + --no-deps \ + --config-settings rapidsai.disable-cuda=true \ + ${REPODIR}/python/rmm fi diff --git a/ci/build_python.sh b/ci/build_python.sh index 394b3a453..fcd2c55e7 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. set -euo pipefail @@ -13,20 +13,13 @@ export CMAKE_GENERATOR=Ninja rapids-print-env -package_name="rmm" -package_dir="python/rmm" - -version=$(rapids-generate-version) -commit=$(git rev-parse HEAD) - -echo "${version}" > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" +rapids-generate-version > ./VERSION rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) # This calls mambabuild when boa is installed (as is the case in the CI images) -RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild -c "${CPP_CHANNEL}" conda/recipes/rmm +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild -c "${CPP_CHANNEL}" conda/recipes/rmm rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index e61f6641c..2c5cc0560 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -3,27 +3,15 @@ set -euo pipefail -package_name="librmm" package_dir="python/librmm" source rapids-configure-sccache source rapids-date-string -version=$(rapids-generate-version) -commit=$(git rev-parse HEAD) +rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -pyproject_file="${package_dir}/pyproject.toml" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" - cd "${package_dir}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index debe9b0fe..92a0c8df2 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -9,33 +9,11 @@ package_dir="python/rmm" source rapids-configure-sccache source rapids-date-string -version=$(rapids-generate-version) -commit=$(git rev-parse HEAD) - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# This is the version of the suffix with a preceding hyphen. It is used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -pyproject_file="${package_dir}/pyproject.toml" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" - -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -sed -r -i "s/librmm==(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} -fi +rapids-generate-version > ./VERSION cd "${package_dir}" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="rmm_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/librmm_dist) python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links "${CPP_WHEELHOUSE}" diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 5806b8bd9..cb2d68d21 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -2,6 +2,7 @@ # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: - rapidsai +- rapidsai-nightly - conda-forge dependencies: - breathe>=4.35.0 @@ -32,7 +33,8 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.12 -- scikit-build-core>=0.7.0 +- rapids-build-backend >=0.3.0,<0.4.0.dev0 +- scikit-build-core >=0.7.0 - spdlog>=1.12.0,<1.13 - sphinx - sphinx-copybutton diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 63876715a..d5fe64a52 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -2,6 +2,7 @@ # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: - rapidsai +- rapidsai-nightly - conda-forge dependencies: - breathe>=4.35.0 @@ -31,7 +32,8 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.12 -- scikit-build-core>=0.7.0 +- rapids-build-backend >=0.3.0,<0.4.0.dev0 +- scikit-build-core >=0.7.0 - spdlog>=1.12.0,<1.13 - sphinx - sphinx-copybutton diff --git a/conda/recipes/rmm/meta.yaml b/conda/recipes/rmm/meta.yaml index fa8a82475..044defdd9 100644 --- a/conda/recipes/rmm/meta.yaml +++ b/conda/recipes/rmm/meta.yaml @@ -62,6 +62,7 @@ requirements: - cuda-python >=12.0,<13.0a0 {% endif %} - cython >=3.0.0 + - rapids-build-backend >=0.3.0,<0.4.0.dev0 - librmm ={{ version }} - python - scikit-build-core >=0.7.0 diff --git a/dependencies.yaml b/dependencies.yaml index 82d1c83e0..d112b3ba6 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -14,6 +14,7 @@ files: - develop - docs - py_version + - rapids_build_skbuild - run - test_python test_python: @@ -43,6 +44,14 @@ files: pyproject_dir: python/librmm extras: table: build-system + includes: + - rapids_build_skbuild + py_cpp_rapids_build: + output: pyproject + pyproject_dir: python/librmm + extras: + table: tool.rapids-build-backend + key: requires includes: - build py_build: @@ -50,6 +59,14 @@ files: pyproject_dir: python/rmm extras: table: build-system + includes: + - rapids_build_skbuild + py_rapids_build: + output: pyproject + pyproject_dir: python/rmm + extras: + table: tool.rapids-build-backend + key: requires includes: - build - cython_build @@ -70,8 +87,19 @@ files: - test_python channels: - rapidsai + - rapidsai-nightly - conda-forge dependencies: + rapids_build_skbuild: + common: + - output_types: conda + packages: + - rapids-build-backend >=0.3.0,<0.4.0.dev0 + - scikit-build-core >=0.7.0 + - output_types: [requirements, pyproject] + packages: + - rapids-build-backend>=0.3.0,<0.4.0.dev0 + - scikit-build-core[pyproject]>=0.7.0 build: common: - output_types: [conda, requirements, pyproject] @@ -83,11 +111,7 @@ dependencies: - c-compiler - cxx-compiler - fmt>=10.1.1,<11 - - scikit-build-core>=0.7.0 - spdlog>=1.12.0,<1.13 - - output_types: [requirements, pyproject] - packages: - - scikit-build-core[pyproject]>=0.7.0 specific: - output_types: conda matrices: @@ -137,14 +161,14 @@ dependencies: - matrix: cuda: "12.*" packages: - - librmm-cu12==24.8.* + - librmm-cu12==24.8.*,>=0.0.0a0 - matrix: cuda: "11.*" packages: - - librmm-cu11==24.8.* + - librmm-cu11==24.8.*,>=0.0.0a0 - matrix: packages: - - librmm==24.8.* + - librmm==24.8.*,>=0.0.0a0 checks: common: - output_types: [conda, requirements] diff --git a/python/librmm/librmm/_version.py b/python/librmm/librmm/_version.py index ea50101b2..d4be04086 100644 --- a/python/librmm/librmm/_version.py +++ b/python/librmm/librmm/_version.py @@ -15,6 +15,11 @@ import importlib.resources __version__ = ( - importlib.resources.files("librmm").joinpath("VERSION").read_text().strip() + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() ) __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/librmm/pyproject.toml b/python/librmm/pyproject.toml index 4b997be9a..04d544c33 100644 --- a/python/librmm/pyproject.toml +++ b/python/librmm/pyproject.toml @@ -13,10 +13,9 @@ # limitations under the License. [build-system] -build-backend = "scikit_build_core.build" +build-backend = "rapids_build_backend.build" requires = [ - "cmake>=3.26.4", - "ninja", + "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -44,6 +43,15 @@ Homepage = "https://github.com/rapidsai/rmm" [project.entry-points."cmake.prefix"] librmm = "librmm" +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +commit-file = "librmm/GIT_COMMIT" +dependencies-file = "../../dependencies.yaml" +requires = [ + "cmake>=3.26.4", + "ninja", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + [tool.scikit-build] build-dir = "build/{wheel_tag}" cmake.build-type = "Release" diff --git a/python/rmm/pyproject.toml b/python/rmm/pyproject.toml index d3a1ac635..a728d132c 100644 --- a/python/rmm/pyproject.toml +++ b/python/rmm/pyproject.toml @@ -13,13 +13,9 @@ # limitations under the License. [build-system] -build-backend = "scikit_build_core.build" +build-backend = "rapids_build_backend.build" requires = [ - "cmake>=3.26.4", - "cuda-python>=11.7.1,<12.0a0", - "cython>=3.0.0", - "librmm==24.8.*", - "ninja", + "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -125,6 +121,18 @@ provider = "scikit_build_core.metadata.regex" input = "rmm/VERSION" regex = "(?P.*)" +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +commit-file = "rmm/GIT_COMMIT" +dependencies-file = "../../dependencies.yaml" +requires = [ + "cmake>=3.26.4", + "cuda-python>=11.7.1,<12.0a0", + "cython>=3.0.0", + "librmm==24.8.*,>=0.0.0a0", + "ninja", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + [tool.pytest.ini_options] # treat warnings as errors filterwarnings = [ diff --git a/python/rmm/rmm/_version.py b/python/rmm/rmm/_version.py index 053e51ecd..7dd732b49 100644 --- a/python/rmm/rmm/_version.py +++ b/python/rmm/rmm/_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,19 @@ import importlib.resources __version__ = ( - importlib.resources.files("rmm").joinpath("VERSION").read_text().strip() + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/rmm/rmm/tests/test_version.py b/python/rmm/rmm/tests/test_version.py new file mode 100644 index 000000000..f74642829 --- /dev/null +++ b/python/rmm/rmm/tests/test_version.py @@ -0,0 +1,24 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import rmm + + +def test_version_constants_are_populated(): + # __git_commit__ will only be non-empty in a built distribution + assert isinstance(rmm.__git_commit__, str) + + # __version__ should always be non-empty + assert isinstance(rmm.__version__, str) + assert len(rmm.__version__) > 0