Switch to scikit-build-core (#146)

Contributes to rapidsai/build-planning#2

Authors:
  - Vyas Ramasubramani (https://github.com/vyasr)
  - Peter Andreas Entschev (https://github.com/pentschev)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - AJ Schmidt (https://github.com/ajschmidt8)
  - Robert Maynard (https://github.com/robertmaynard)
  - Peter Andreas Entschev (https://github.com/pentschev)

URL: #146

build.sh

@@ -215,24 +215,20 @@ if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_UCXX_CPP"* ]]; then
     EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DFIND_UCXX_CPP=ON"
 fi
 
+# Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS
+SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${EXTRA_CMAKE_ARGS} | sed 's/ /;/g')
+
 # Build and install the UCXX Python package
 if buildAll || hasArg ucxx; then
 
     cd ${REPODIR}/python/
-    SKBUILD_CONFIGURE_OPTIONS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} ${EXTRA_CMAKE_ARGS}" \
-        SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL:-1}" \
-        python setup.py build_ext --inplace
-    if [[ ${INSTALL_TARGET} != "" ]]; then
-            python setup.py install --single-version-externally-managed --record=record.txt
-    fi
+    SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_BUILD_TYPE=${BUILD_TYPE};${SKBUILD_EXTRA_CMAKE_ARGS}" \
+        python -m pip install --no-build-isolation --no-deps .
 fi
 
 # Build and install the distributed_ucxx Python package
 if buildAll || hasArg distributed_ucxx; then
 
     cd ${REPODIR}/python/distributed-ucxx/
-    python setup.py build_ext --inplace
-    if [[ ${INSTALL_TARGET} != "" ]]; then
-        python setup.py install --single-version-externally-managed --record=record.txt
-    fi
+    python -m pip install --no-build-isolation --no-deps .
 fi

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -39,10 +39,7 @@ dependencies:
 - pytest-rerunfailures
 - python>=3.9,<3.11
 - rmm==24.2.*
-- scikit-build>=0.13.1
-- setuptools
+- scikit-build-core>=0.7.0
 - spdlog>=1.12.0,<1.13
-- tomli
 - ucx
-- wheel
 name: all_cuda-118_arch-x86_64

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -0,0 +1,45 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- rapidsai
+- rapidsai-nightly
+- dask/label/dev
+- conda-forge
+- nvidia
+dependencies:
+- autoconf
+- automake
+- c-compiler
+- cloudpickle
+- cmake>=3.26.4
+- cuda-cudart-dev
+- cuda-version=12.0
+- cudf==24.2.*
+- cupy
+- cxx-compiler
+- cython>=3.0.0
+- dask
+- dask-cuda==24.2.*
+- dask-cudf==24.2.*
+- distributed
+- fmt>=10.1.1,<11
+- gmock>=1.13.0
+- gtest>=1.13.0
+- librmm==24.2.*
+- libtool
+- ninja
+- numba>=0.57.1
+- numpy>=1.21
+- pip
+- pkg-config
+- pre-commit
+- pynvml>=11.4.1
+- pytest
+- pytest-asyncio
+- pytest-rerunfailures
+- python>=3.9,<3.11
+- rmm==24.2.*
+- scikit-build-core>=0.7.0
+- spdlog>=1.12.0,<1.13
+- ucx
+name: all_cuda-120_arch-x86_64

conda/recipes/ucxx/meta.yaml

@@ -212,8 +212,7 @@ outputs:
       host:
         - python
         - pip
-        - scikit-build>=0.13.1
-        - setuptools
+        - scikit-build-core >=0.7.0
         - cython >=3.0.0
         - numpy 1.21
         - {{ pin_subpackage('libucxx', exact=True) }}
@@ -267,6 +266,8 @@ outputs:
       host:
         - python
         - pip
+        - setuptools
+        - wheel
         - tomli
       run:
         - python * *_cpython

dependencies.yaml

@@ -81,10 +81,12 @@ dependencies:
           - cython>=3.0.0
           - ninja
           - rmm==24.2.*
-          - scikit-build>=0.13.1
-          - setuptools
-          - tomli
-          - wheel
+      - output_types: conda
+        packages:
+          - scikit-build-core>=0.7.0
+      - output_types: [requirements, pyproject]
+        packages:
+          - scikit-build-core[pyproject]>=0.7.0
   checks:
     common:
       - output_types: [conda, requirements]

python/CMakeLists.txt

@@ -12,11 +12,7 @@ include(../fetch_rapids.cmake)
 project(
   ucxx-python
   VERSION ${ucxx_version}
-  LANGUAGES # TODO: Building Python extension modules via the python_extension_module requires the C
-            # language to be enabled here. The test project that is built in scikit-build to verify
-            # various linking options for the python library is hardcoded to build with C, so until
-            # that is fixed we need to keep C.
-            C CXX
+  LANGUAGES CXX
 )
 
 option(FIND_UCXX_CPP "Search for existing UCXX C++ installations before defaulting to local files"
@@ -30,7 +26,7 @@ else()
   set(ucxx_FOUND OFF)
 endif()
 
-include(rapids-cython)
+include(rapids-cython-core)
 
 if(NOT ucxx_FOUND)
   set(BUILD_TESTS OFF)

python/LICENSE

@@ -0,0 +1 @@
+../LICENSE

python/README.md

@@ -0,0 +1 @@
+../README.md

python/distributed-ucxx/pyproject.toml

@@ -29,6 +29,9 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
 ]
 
+[project.entry-points."distributed.comm.backends"]
+ucxx = "distributed_ucxx:UCXXBackend"
+
 [project.optional-dependencies]
 docs = [
     "sphinx",

python/pyproject.toml

@@ -2,16 +2,13 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 [build-system]
-build-backend = "setuptools.build_meta"
+build-backend = "scikit_build_core.build"
 requires = [
     "cmake>=3.26.4",
     "cython>=3.0.0",
     "ninja",
     "rmm==24.2.*",
-    "scikit-build>=0.13.1",
-    "setuptools",
-    "tomli",
-    "wheel",
+    "scikit-build-core[pyproject]>=0.7.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project]
@@ -96,6 +93,10 @@ skip = [
     "__init__.py",
 ]
 
-[tool.setuptools]
-license-files = ["LICENSE"]
-zip-safe = false
+[tool.scikit-build]
+build-dir = "build/{wheel_tag}"
+cmake.build-type = "Release"
+cmake.minimum-version = "3.26.4"
+ninja.make-fallback = true
+sdist.reproducible = true
+wheel.packages = ["ucxx"]

python/ucxx/_lib/CMakeLists.txt

@@ -12,27 +12,10 @@ rapids_cython_create_modules(
   LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS ucxx
 )
 
-# TODO: Finding NumPy currently requires finding Development due to a bug in CMake. This bug was
-# fixed in https://gitlab.kitware.com/cmake/cmake/-/merge_requests/7410 and will be available in
-# CMake 3.24, so we can remove the Development component once we upgrade to CMake 3.24.
-# find_package(Python REQUIRED COMPONENTS Development NumPy)
+find_package(Python REQUIRED COMPONENTS Development NumPy)
 
-# Note: The bug noted above prevents us from finding NumPy successfully using FindPython.cmake
-# inside the manylinux images used to build wheels because manylinux images do not contain
-# libpython.so and therefore Development cannot be found. Until we upgrade to CMake 3.24, we should
-# use FindNumpy.cmake instead (provided by scikit-build). When we switch to 3.24 we can try
-# switching back, but it may not work if that implicitly still requires Python libraries. In that
-# case we'll need to follow up with the CMake team to remove that dependency.  The stopgap solution
-# is to unpack the static lib tarballs in the wheel building jobs so that there are at least static
-# libs to be found, but that should be a last resort since it implies a dependency that isn't really
-# necessary. The relevant command is tar -xf /opt/_internal/static-libs-for-embedding-only.tar.xz -C
-# /opt/_internal"
-find_package(NumPy REQUIRED)
 set(targets_using_numpy libucxx)
 foreach(target IN LISTS targets_using_numpy)
-  target_include_directories(${target} PRIVATE "${NumPy_INCLUDE_DIRS}")
-  # Switch to the line below when we switch back to FindPython.cmake in CMake 3.24.
-  # target_include_directories(${target} PRIVATE "${Python_NumPy_INCLUDE_DIRS}")
-
+  target_include_directories(${target} PRIVATE "${Python_NumPy_INCLUDE_DIRS}")
   target_compile_definitions(${target} PRIVATE NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION)
 endforeach()