From 8d86032ce2a36325311003ef968f52790fc9c13c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 22 May 2024 15:41:23 -0400 Subject: [PATCH 01/10] Use rapids-build-backend --- .pre-commit-config.yaml | 2 +- ci/build_wheel.sh | 35 ----------- .../all_cuda-118_arch-aarch64.yaml | 7 ++- .../all_cuda-118_arch-x86_64.yaml | 7 ++- .../all_cuda-122_arch-aarch64.yaml | 7 ++- .../all_cuda-122_arch-x86_64.yaml | 7 ++- .../bench_ann_cuda-118_arch-aarch64.yaml | 3 +- .../bench_ann_cuda-118_arch-x86_64.yaml | 3 +- .../bench_ann_cuda-120_arch-aarch64.yaml | 3 +- .../bench_ann_cuda-120_arch-x86_64.yaml | 3 +- dependencies.yaml | 61 ++++++++++++++----- python/pylibraft/pylibraft/_version.py | 17 ++++-- python/pylibraft/pyproject.toml | 22 ++++--- python/raft-ann-bench/pyproject.toml | 8 ++- .../src/raft-ann-bench/_version.py | 15 ++++- python/raft-dask/pyproject.toml | 22 ++++--- python/raft-dask/raft_dask/_version.py | 17 ++++-- 17 files changed, 145 insertions(+), 94 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b89948ec1..ecbaceb20f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -95,7 +95,7 @@ repos: args: ["--toml", "pyproject.toml"] exclude: (?x)^(^CHANGELOG.md$) - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.8.0 + rev: v1.13.9 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index e3e7ce9c89..0930e98c85 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -19,42 +19,7 @@ git_commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${underscore_package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} echo "${version}" > VERSION -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -if [[ ${package_name} == "raft-dask" ]]; then - sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/distributed-ucxx==(.*)\"/distributed-ucxx${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -else - sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -fi - -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi cd "${package_dir}" diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 3d0f9347bc..1448373b07 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -20,7 +20,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.8.* +- dask-cuda==24.8.*,>=0.0.0a0 - distributed-ucxx==0.39.* - doxygen>=1.8.20 - gcc_linux-aarch64=11.* @@ -46,9 +46,10 @@ dependencies: - pydata-sphinx-theme - pytest-cov - pytest==7.* -- rapids-dask-dependency==24.8.* +- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-dask-dependency==24.8.*,>=0.0.0a0 - recommonmark -- rmm==24.8.* +- rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - scikit-learn - scipy diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index bedfc9a2a3..b7a76f9d49 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,7 +20,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.8.* +- dask-cuda==24.8.*,>=0.0.0a0 - distributed-ucxx==0.39.* - doxygen>=1.8.20 - gcc_linux-64=11.* @@ -46,9 +46,10 @@ dependencies: - pydata-sphinx-theme - pytest-cov - pytest==7.* -- rapids-dask-dependency==24.8.* +- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-dask-dependency==24.8.*,>=0.0.0a0 - recommonmark -- rmm==24.8.* +- rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - scikit-learn - scipy diff --git a/conda/environments/all_cuda-122_arch-aarch64.yaml b/conda/environments/all_cuda-122_arch-aarch64.yaml index 25543099f8..9b98d2f40f 100644 --- a/conda/environments/all_cuda-122_arch-aarch64.yaml +++ b/conda/environments/all_cuda-122_arch-aarch64.yaml @@ -21,7 +21,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.8.* +- dask-cuda==24.8.*,>=0.0.0a0 - distributed-ucxx==0.39.* - doxygen>=1.8.20 - gcc_linux-aarch64=11.* @@ -42,9 +42,10 @@ dependencies: - pydata-sphinx-theme - pytest-cov - pytest==7.* -- rapids-dask-dependency==24.8.* +- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-dask-dependency==24.8.*,>=0.0.0a0 - recommonmark -- rmm==24.8.* +- rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - scikit-learn - scipy diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 52adc54c55..9c9c1ce37f 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -21,7 +21,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-cuda==24.8.* +- dask-cuda==24.8.*,>=0.0.0a0 - distributed-ucxx==0.39.* - doxygen>=1.8.20 - gcc_linux-64=11.* @@ -42,9 +42,10 @@ dependencies: - pydata-sphinx-theme - pytest-cov - pytest==7.* -- rapids-dask-dependency==24.8.* +- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-dask-dependency==24.8.*,>=0.0.0a0 - recommonmark -- rmm==24.8.* +- rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - scikit-learn - scipy diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml index 729ae0c3d0..0169bb8693 100644 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml @@ -39,7 +39,8 @@ dependencies: - openblas - pandas - pyyaml -- rmm==24.8.* +- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-118_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 6a6eb09bee..0df6e2f321 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -39,7 +39,8 @@ dependencies: - openblas - pandas - pyyaml -- rmm==24.8.* +- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-118_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml index f368fa0c9c..ae71eda509 100644 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml @@ -35,7 +35,8 @@ dependencies: - openblas - pandas - pyyaml -- rmm==24.8.* +- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-aarch64==2.17 name: bench_ann_cuda-120_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml index 8f8cdf9b77..7686329abe 100644 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml @@ -35,7 +35,8 @@ dependencies: - openblas - pandas - pyyaml -- rmm==24.8.* +- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-64==2.17 name: bench_ann_cuda-120_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index c758620879..7d8abb9101 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -7,6 +7,7 @@ files: arch: [x86_64, aarch64] includes: - build + - rapids_build - build_pylibraft - cuda - cuda_version @@ -28,6 +29,7 @@ files: arch: [x86_64, aarch64] includes: - build + - rapids_build - cuda - cuda_version - develop @@ -66,6 +68,14 @@ files: table: build-system includes: - build + py_rapids_build_pylibraft: + output: pyproject + pyproject_dir: python/pylibraft + extras: + table: tool.rapids-build-backend + key: requires + includes: + - rapids_build - build_pylibraft py_run_pylibraft: output: pyproject @@ -91,6 +101,14 @@ files: table: build-system includes: - build + py_rapids_build_raft_dask: + output: pyproject + pyproject_dir: python/raft-dask + extras: + table: tool.rapids-build-backend + key: requires + includes: + - rapids_build - depends_on_ucx_build py_run_raft_dask: output: pyproject @@ -130,6 +148,16 @@ channels: - nvidia dependencies: build: + common: + - output_types: [conda] + packages: + - rapids-build-backend>=0.2.0,<0.3.0.dev0 + - scikit-build-core>=0.7.0 + - output_types: [requirements, pyproject] + packages: + - rapids-build-backend>=0.2.0,<0.3.0.dev0 + - scikit-build-core[pyproject]>=0.7.0 + rapids_build: common: - output_types: [conda, requirements, pyproject] packages: @@ -142,10 +170,6 @@ dependencies: - cxx-compiler - nccl>=2.9.9 - libucxx==0.39.* - - scikit-build-core>=0.7.0 - - output_types: [requirements, pyproject] - packages: - - scikit-build-core[pyproject]>=0.7.0 specific: - output_types: conda matrices: @@ -184,7 +208,7 @@ dependencies: common: - output_types: [conda] packages: - - &rmm_conda rmm==24.8.* + - &rmm_conda rmm==24.8.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -198,17 +222,21 @@ dependencies: cuda: "12.*" packages: - &cuda_python12 cuda-python>=12.0,<13.0a0 - - matrix: # All CUDA 11 versions + - matrix: + cuda: "11.*" packages: - &cuda_python11 cuda-python>=11.7.1,<12.0a0 + - matrix: + packages: + - &cuda_python cuda-python - output_types: [requirements, pyproject] matrices: - matrix: {cuda: "12.*"} packages: - - &rmm_cu12 rmm-cu12==24.8.* + - &rmm_cu12 rmm-cu12==24.8.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - &rmm_cu11 rmm-cu11==24.8.* + - &rmm_cu11 rmm-cu11==24.8.*,>=0.0.0a0 - {matrix: null, packages: [*rmm_conda] } checks: common: @@ -381,6 +409,7 @@ dependencies: packages: - wheel - setuptools + - rapids-build-backend>=0.2.0,<0.3.0.dev0 py_version: specific: - output_types: conda @@ -421,9 +450,13 @@ dependencies: cuda: "12.*" packages: - *cuda_python12 - - matrix: # All CUDA 11 versions + - matrix: + cuda: "11.*" packages: - *cuda_python11 + - matrix: + packages: + - *cuda_python - output_types: [requirements, pyproject] matrices: - matrix: {cuda: "12.*"} @@ -437,18 +470,18 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - dask-cuda==24.8.* + - dask-cuda==24.8.*,>=0.0.0a0 - joblib>=0.11 - numba>=0.57 - *numpy - - rapids-dask-dependency==24.8.* + - rapids-dask-dependency==24.8.*,>=0.0.0a0 - ucx-py==0.39.* - output_types: conda packages: - &ucx_py_conda ucx-py==0.39.* - output_types: pyproject packages: - - &pylibraft_conda pylibraft==24.8.* + - &pylibraft_conda pylibraft==24.8.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -460,11 +493,11 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - &pylibraft_cu12 pylibraft-cu12==24.8.* + - &pylibraft_cu12 pylibraft-cu12==24.8.*,>=0.0.0a0 - &ucx_py_cu12 ucx-py-cu12==0.39.* - matrix: {cuda: "11.*"} packages: - - &pylibraft_cu11 pylibraft-cu11==24.8.* + - &pylibraft_cu11 pylibraft-cu11==24.8.*,>=0.0.0a0 - &ucx_py_cu11 ucx-py-cu11==0.39.* - {matrix: null, packages: [*pylibraft_conda, *ucx_py_conda]} test_python_common: diff --git a/python/pylibraft/pylibraft/_version.py b/python/pylibraft/pylibraft/_version.py index 3b359e2ff3..0fa0ba80bc 100644 --- a/python/pylibraft/pylibraft/_version.py +++ b/python/pylibraft/pylibraft/_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,13 +13,22 @@ # limitations under the License. # - import importlib.resources __version__ = ( - importlib.resources.files("pylibraft") + importlib.resources.files(__package__) .joinpath("VERSION") .read_text() .strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__version__", "__git_commit__"] diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 9f77f22f6a..93dcb2f5d3 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -15,14 +15,10 @@ [build-system] requires = [ - "cmake>=3.26.4", - "cuda-python>=11.7.1,<12.0a0", - "cython>=3.0.0", - "ninja", - "rmm==24.8.*", + "rapids-build-backend>=0.2.0,<0.3.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -build-backend = "scikit_build_core.build" +build-backend = "rapids_build_backend.build" [project] name = "pylibraft" @@ -35,9 +31,9 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cuda-python>=11.7.1,<12.0a0", + "cuda-python", "numpy>=1.23,<2.0a0", - "rmm==24.8.*", + "rmm==24.8.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -117,6 +113,16 @@ provider = "scikit_build_core.metadata.regex" input = "pylibraft/VERSION" regex = "(?P.*)" +[tool.rapids-build-backend] +requires = [ + "cmake>=3.26.4", + "cuda-python", + "cython>=3.0.0", + "ninja", + "rmm==24.8.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +dependencies-file = "../../dependencies.yaml" + [tool.pytest.ini_options] filterwarnings = [ "error", diff --git a/python/raft-ann-bench/pyproject.toml b/python/raft-ann-bench/pyproject.toml index 9bb7ae0468..5ca3bf39f2 100644 --- a/python/raft-ann-bench/pyproject.toml +++ b/python/raft-ann-bench/pyproject.toml @@ -1,8 +1,9 @@ # Copyright (c) 2023, NVIDIA CORPORATION. [build-system] -build-backend = "setuptools.build_meta" +build-backend = "rapids_build_backend.build" requires = [ + "rapids-build-backend>=0.2.0,<0.3.0.dev0", "setuptools", "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -60,3 +61,8 @@ skip = [ [tool.setuptools.dynamic] version = { attr = "raft-ann-bench.__version__" } + +[tool.rapids-build-backend] +requires = [] +dependencies-file = "../../dependencies.yaml" +commit-files = ["src/raft-ann-bench/GIT_COMMIT"] diff --git a/python/raft-ann-bench/src/raft-ann-bench/_version.py b/python/raft-ann-bench/src/raft-ann-bench/_version.py index 6dbb8e81b0..0fa0ba80bc 100644 --- a/python/raft-ann-bench/src/raft-ann-bench/_version.py +++ b/python/raft-ann-bench/src/raft-ann-bench/_version.py @@ -13,13 +13,22 @@ # limitations under the License. # - import importlib.resources __version__ = ( - importlib.resources.files("raft-ann-bench") + importlib.resources.files(__package__) .joinpath("VERSION") .read_text() .strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__version__", "__git_commit__"] diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 589ff36bc6..5648ac4364 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -14,12 +14,9 @@ [build-system] -build-backend = "scikit_build_core.build" +build-backend = "rapids_build_backend.build" requires = [ - "cmake>=3.26.4", - "cython>=3.0.0", - "libucx==1.15.0", - "ninja", + "rapids-build-backend>=0.2.0,<0.3.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -34,13 +31,13 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "dask-cuda==24.8.*", + "dask-cuda==24.8.*,>=0.0.0a0", "distributed-ucxx==0.39.*", "joblib>=0.11", "numba>=0.57", "numpy>=1.23,<2.0a0", - "pylibraft==24.8.*", - "rapids-dask-dependency==24.8.*", + "pylibraft==24.8.*,>=0.0.0a0", + "rapids-dask-dependency==24.8.*,>=0.0.0a0", "ucx-py==0.39.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ @@ -118,3 +115,12 @@ wheel.packages = ["raft_dask"] provider = "scikit_build_core.metadata.regex" input = "raft_dask/VERSION" regex = "(?P.*)" + +[tool.rapids-build-backend] +requires = [ + "cmake>=3.26.4", + "cython>=3.0.0", + "libucx==1.15.0", + "ninja", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +dependencies-file = "../../dependencies.yaml" diff --git a/python/raft-dask/raft_dask/_version.py b/python/raft-dask/raft_dask/_version.py index 03cddef557..0fa0ba80bc 100644 --- a/python/raft-dask/raft_dask/_version.py +++ b/python/raft-dask/raft_dask/_version.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,13 +13,22 @@ # limitations under the License. # - import importlib.resources __version__ = ( - importlib.resources.files("raft_dask") + importlib.resources.files(__package__) .joinpath("VERSION") .read_text() .strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__version__", "__git_commit__"] From d5d747ffe296fc9d669c4ef6af91f7a3badce8e9 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 22 May 2024 15:46:59 -0400 Subject: [PATCH 02/10] disable-cuda --- build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 148d23c9c1..6a1af3e70e 100755 --- a/build.sh +++ b/build.sh @@ -496,18 +496,18 @@ fi # Build and (optionally) install the pylibraft Python package if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ - python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/pylibraft + python -m pip install --no-build-isolation --no-deps --package-config disable-cuda=true ${REPODIR}/python/pylibraft fi # Build and (optionally) install the raft-dask Python package if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ - python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/raft-dask + python -m pip install --no-build-isolation --no-deps --package-config disable-cuda=true ${REPODIR}/python/raft-dask fi # Build and (optionally) install the raft-ann-bench Python package if (( ${NUMARGS} == 0 )) || hasArg bench-ann; then - python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/raft-ann-bench -vvv + python -m pip install --no-build-isolation --no-deps --package-config disable-cuda=true ${REPODIR}/python/raft-ann-bench -vvv fi if hasArg docs; then From 7083248de2afd4f689763aaeb336c0613508acb1 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 22 May 2024 15:56:00 -0400 Subject: [PATCH 03/10] build-backend --- python/pylibraft/pyproject.toml | 1 + python/raft-ann-bench/pyproject.toml | 1 + python/raft-dask/pyproject.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 93dcb2f5d3..33525e0242 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -114,6 +114,7 @@ input = "pylibraft/VERSION" regex = "(?P.*)" [tool.rapids-build-backend] +build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cuda-python", diff --git a/python/raft-ann-bench/pyproject.toml b/python/raft-ann-bench/pyproject.toml index 5ca3bf39f2..55d5afc399 100644 --- a/python/raft-ann-bench/pyproject.toml +++ b/python/raft-ann-bench/pyproject.toml @@ -63,6 +63,7 @@ skip = [ version = { attr = "raft-ann-bench.__version__" } [tool.rapids-build-backend] +build-backend = "setuptools.build_meta" requires = [] dependencies-file = "../../dependencies.yaml" commit-files = ["src/raft-ann-bench/GIT_COMMIT"] diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 5648ac4364..ef7aaeadd4 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -117,6 +117,7 @@ input = "raft_dask/VERSION" regex = "(?P.*)" [tool.rapids-build-backend] +build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.0", From 2b768dc480b86c733dd64aed11cdb6b961a99d4b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 22 May 2024 16:33:33 -0400 Subject: [PATCH 04/10] config-setting --- build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 6a1af3e70e..37851bf584 100755 --- a/build.sh +++ b/build.sh @@ -496,18 +496,18 @@ fi # Build and (optionally) install the pylibraft Python package if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ - python -m pip install --no-build-isolation --no-deps --package-config disable-cuda=true ${REPODIR}/python/pylibraft + python -m pip install --no-build-isolation --no-deps --config-setting disable-cuda=true ${REPODIR}/python/pylibraft fi # Build and (optionally) install the raft-dask Python package if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ - python -m pip install --no-build-isolation --no-deps --package-config disable-cuda=true ${REPODIR}/python/raft-dask + python -m pip install --no-build-isolation --no-deps --config-setting disable-cuda=true ${REPODIR}/python/raft-dask fi # Build and (optionally) install the raft-ann-bench Python package if (( ${NUMARGS} == 0 )) || hasArg bench-ann; then - python -m pip install --no-build-isolation --no-deps --package-config disable-cuda=true ${REPODIR}/python/raft-ann-bench -vvv + python -m pip install --no-build-isolation --no-deps --config-setting disable-cuda=true ${REPODIR}/python/raft-ann-bench -vvv fi if hasArg docs; then From 08deb4918f6b8e1ad7d7fe931ed46dbd9f079cd9 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 22 May 2024 17:20:55 -0400 Subject: [PATCH 05/10] Fix conda recipe --- conda/recipes/pylibraft/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index cbeaec3b55..739fcd609c 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -56,6 +56,7 @@ requirements: - rmm ={{ minor_version }} - scikit-build-core >=0.7.0 - setuptools + - rapids-build-backend>=0.2.0,<0.3.0.dev0 run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} From 28675a68c361b28137fc7f6179bf2e485097d902 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 23 May 2024 13:19:33 -0400 Subject: [PATCH 06/10] Update RBB to 0.3.0 --- build.sh | 6 +++--- conda/environments/all_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-122_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-122_arch-x86_64.yaml | 2 +- conda/environments/bench_ann_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/bench_ann_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/bench_ann_cuda-120_arch-aarch64.yaml | 2 +- conda/environments/bench_ann_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/pylibraft/meta.yaml | 2 +- dependencies.yaml | 6 +++--- python/pylibraft/pyproject.toml | 2 +- python/raft-ann-bench/pyproject.toml | 2 +- python/raft-dask/pyproject.toml | 2 +- 14 files changed, 18 insertions(+), 18 deletions(-) diff --git a/build.sh b/build.sh index 37851bf584..289c3922ef 100755 --- a/build.sh +++ b/build.sh @@ -496,18 +496,18 @@ fi # Build and (optionally) install the pylibraft Python package if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ - python -m pip install --no-build-isolation --no-deps --config-setting disable-cuda=true ${REPODIR}/python/pylibraft + python -m pip install --no-build-isolation --no-deps --config-setting rapidsai.disable-cuda=true ${REPODIR}/python/pylibraft fi # Build and (optionally) install the raft-dask Python package if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ - python -m pip install --no-build-isolation --no-deps --config-setting disable-cuda=true ${REPODIR}/python/raft-dask + python -m pip install --no-build-isolation --no-deps --config-setting rapidsai.disable-cuda=true ${REPODIR}/python/raft-dask fi # Build and (optionally) install the raft-ann-bench Python package if (( ${NUMARGS} == 0 )) || hasArg bench-ann; then - python -m pip install --no-build-isolation --no-deps --config-setting disable-cuda=true ${REPODIR}/python/raft-ann-bench -vvv + python -m pip install --no-build-isolation --no-deps --config-setting rapidsai.disable-cuda=true ${REPODIR}/python/raft-ann-bench -vvv fi if hasArg docs; then diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 1448373b07..2fad2e103d 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -46,7 +46,7 @@ dependencies: - pydata-sphinx-theme - pytest-cov - pytest==7.* -- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.8.*,>=0.0.0a0 - recommonmark - rmm==24.8.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index b7a76f9d49..b141ebd7e0 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -46,7 +46,7 @@ dependencies: - pydata-sphinx-theme - pytest-cov - pytest==7.* -- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.8.*,>=0.0.0a0 - recommonmark - rmm==24.8.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-122_arch-aarch64.yaml b/conda/environments/all_cuda-122_arch-aarch64.yaml index 9b98d2f40f..9333930e01 100644 --- a/conda/environments/all_cuda-122_arch-aarch64.yaml +++ b/conda/environments/all_cuda-122_arch-aarch64.yaml @@ -42,7 +42,7 @@ dependencies: - pydata-sphinx-theme - pytest-cov - pytest==7.* -- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.8.*,>=0.0.0a0 - recommonmark - rmm==24.8.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 9c9c1ce37f..e209b5b046 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -42,7 +42,7 @@ dependencies: - pydata-sphinx-theme - pytest-cov - pytest==7.* -- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - rapids-dask-dependency==24.8.*,>=0.0.0a0 - recommonmark - rmm==24.8.*,>=0.0.0a0 diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml index 0169bb8693..ac72b6424c 100644 --- a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml @@ -39,7 +39,7 @@ dependencies: - openblas - pandas - pyyaml -- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-aarch64==2.17 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 0df6e2f321..b1058d6c57 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -39,7 +39,7 @@ dependencies: - openblas - pandas - pyyaml -- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-64==2.17 diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml index ae71eda509..b6d5b6c6f2 100644 --- a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml @@ -35,7 +35,7 @@ dependencies: - openblas - pandas - pyyaml -- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-aarch64==2.17 diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml index 7686329abe..9e93941732 100644 --- a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml @@ -35,7 +35,7 @@ dependencies: - openblas - pandas - pyyaml -- rapids-build-backend>=0.2.0,<0.3.0.dev0 +- rapids-build-backend>=0.3.0,<0.4.0.dev0 - rmm==24.8.*,>=0.0.0a0 - scikit-build-core>=0.7.0 - sysroot_linux-64==2.17 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 739fcd609c..16fecadd83 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -56,7 +56,7 @@ requirements: - rmm ={{ minor_version }} - scikit-build-core >=0.7.0 - setuptools - - rapids-build-backend>=0.2.0,<0.3.0.dev0 + - rapids-build-backend>=0.3.0,<0.4.0.dev0 run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} diff --git a/dependencies.yaml b/dependencies.yaml index 7d8abb9101..363d2588e1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -151,11 +151,11 @@ dependencies: common: - output_types: [conda] packages: - - rapids-build-backend>=0.2.0,<0.3.0.dev0 + - rapids-build-backend>=0.3.0,<0.4.0.dev0 - scikit-build-core>=0.7.0 - output_types: [requirements, pyproject] packages: - - rapids-build-backend>=0.2.0,<0.3.0.dev0 + - rapids-build-backend>=0.3.0,<0.4.0.dev0 - scikit-build-core[pyproject]>=0.7.0 rapids_build: common: @@ -409,7 +409,7 @@ dependencies: packages: - wheel - setuptools - - rapids-build-backend>=0.2.0,<0.3.0.dev0 + - rapids-build-backend>=0.3.0,<0.4.0.dev0 py_version: specific: - output_types: conda diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 33525e0242..3c1b0ef118 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -15,7 +15,7 @@ [build-system] requires = [ - "rapids-build-backend>=0.2.0,<0.3.0.dev0", + "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "rapids_build_backend.build" diff --git a/python/raft-ann-bench/pyproject.toml b/python/raft-ann-bench/pyproject.toml index 55d5afc399..f393cfc513 100644 --- a/python/raft-ann-bench/pyproject.toml +++ b/python/raft-ann-bench/pyproject.toml @@ -3,7 +3,7 @@ [build-system] build-backend = "rapids_build_backend.build" requires = [ - "rapids-build-backend>=0.2.0,<0.3.0.dev0", + "rapids-build-backend>=0.3.0,<0.4.0.dev0", "setuptools", "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index ef7aaeadd4..f3d5acacac 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -16,7 +16,7 @@ build-backend = "rapids_build_backend.build" requires = [ - "rapids-build-backend>=0.2.0,<0.3.0.dev0", + "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From be0c62bea410c340ab06c03395fb8bc0acd4dd06 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 23 May 2024 15:34:17 -0400 Subject: [PATCH 07/10] Pipe rapids-generate-version directly to VERSION --- ci/build_wheel.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 0930e98c85..62d93a668e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -14,12 +14,9 @@ rm -rf /usr/lib64/libuc* source rapids-configure-sccache source rapids-date-string -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -echo "${version}" > VERSION +rapids-generate-version > VERSION cd "${package_dir}" From ae67cd0d7b23d4a3ab6c4499eced2b3ed0fde561 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 23 May 2024 15:52:35 -0400 Subject: [PATCH 08/10] Remove sed from build_python --- ci/build_python.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index a8b76269ae..80d37b5ae3 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -22,12 +22,6 @@ git_commit=$(git rev-parse HEAD) export RAPIDS_PACKAGE_VERSION=${version} echo "${version}" > VERSION -package_dir="python" -for package_name in pylibraft raft-dask; do - underscore_package_name=$(echo "${package_name}" | tr "-" "_") - sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py" -done - # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly rapids-conda-retry mambabuild \ From 0bec763068e46053ecc82f35a0730054f12c0fc6 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 23 May 2024 16:51:42 -0400 Subject: [PATCH 09/10] Add RBB dependency to Conda recipes --- conda/recipes/raft-ann-bench-cpu/meta.yaml | 1 + conda/recipes/raft-ann-bench/meta.yaml | 1 + conda/recipes/raft-dask/meta.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/conda/recipes/raft-ann-bench-cpu/meta.yaml b/conda/recipes/raft-ann-bench-cpu/meta.yaml index d0748fdb16..94f7102726 100644 --- a/conda/recipes/raft-ann-bench-cpu/meta.yaml +++ b/conda/recipes/raft-ann-bench-cpu/meta.yaml @@ -53,6 +53,7 @@ requirements: - python - pyyaml - pandas + - rapids-build-backend>=0.3.0,<0.4.0.dev0 run: - glog {{ glog_version }} diff --git a/conda/recipes/raft-ann-bench/meta.yaml b/conda/recipes/raft-ann-bench/meta.yaml index 8a6a3d033d..d6aeb5f860 100644 --- a/conda/recipes/raft-ann-bench/meta.yaml +++ b/conda/recipes/raft-ann-bench/meta.yaml @@ -82,6 +82,7 @@ requirements: - pyyaml # rmm is needed to determine if package is gpu-enabled - rmm ={{ minor_version }} + - rapids-build-backend>=0.3.0,<0.4.0.dev0 run: - python diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index af22c8853e..83ec1922e1 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -58,6 +58,7 @@ requirements: - setuptools - ucx-py {{ ucx_py_version }} - ucxx {{ ucxx_version }} + - rapids-build-backend>=0.3.0,<0.4.0.dev0 run: {% if cuda_major == "11" %} - cudatoolkit From d891c3c65413c484ce3199fc9ac769379d8acb38 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 23 May 2024 18:35:58 -0400 Subject: [PATCH 10/10] Fix location of GIT_COMMIT file --- python/raft-ann-bench/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/raft-ann-bench/pyproject.toml b/python/raft-ann-bench/pyproject.toml index 0d5e417110..226dc41e40 100644 --- a/python/raft-ann-bench/pyproject.toml +++ b/python/raft-ann-bench/pyproject.toml @@ -67,4 +67,4 @@ version = { file = "raft_ann_bench/VERSION" } build-backend = "setuptools.build_meta" requires = [] dependencies-file = "../../dependencies.yaml" -commit-files = ["src/raft-ann-bench/GIT_COMMIT"] +commit-files = ["src/raft_ann_bench/GIT_COMMIT"]