diff --git a/.github/workflows/dependency-files.yml b/.github/workflows/dependency-files.yml new file mode 100644 index 00000000000..2ae939292d7 --- /dev/null +++ b/.github/workflows/dependency-files.yml @@ -0,0 +1,12 @@ +name: pr + +on: + pull_request: + +jobs: + checks: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main + with: + enable_check_size: false + enable_check_style: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6eb621abcc3..e69ad63c282 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,13 +99,13 @@ cd $CUDF_HOME **Note:** Using a conda environment is the easiest way to satisfy the library's dependencies. Instructions for a minimal build environment without conda are included below. -- Create the conda development environment `cudf_dev`: +- Create the conda development environment: ```bash # create the conda environment (assuming in base `cudf` directory) # note: RAPIDS currently doesn't support `channel_priority: strict`; # use `channel_priority: flexible` instead -conda env create --name cudf_dev --file conda/environments/cudf_dev_cuda11.5.yml +conda env create --name cudf_dev --file conda/environments/all_cuda-115_arch-x86_64.yaml # activate the environment conda activate cudf_dev ``` @@ -114,9 +114,6 @@ conda activate cudf_dev development environment may also need to be updated if dependency versions or pinnings are changed. -- For other CUDA versions, check the corresponding `cudf_dev_cuda*.yml` file in - `conda/environments/`. - #### Building without a conda environment - libcudf has the following minimal dependencies (in addition to those listed in the [General diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 52dc22b6c49..9dcfe093643 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -63,9 +63,10 @@ sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/cudf/source sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/cudf/source/conf.py # bump rmm & dask-cuda -for FILE in conda/environments/*.yml; do - sed_runner "s/rmm=${CURRENT_SHORT_TAG}/rmm=${NEXT_SHORT_TAG}/g" ${FILE}; +for FILE in conda/environments/*.yaml dependencies.yaml; do sed_runner "s/dask-cuda=${CURRENT_SHORT_TAG}/dask-cuda=${NEXT_SHORT_TAG}/g" ${FILE}; + sed_runner "s/rmm=${CURRENT_SHORT_TAG}/rmm=${NEXT_SHORT_TAG}/g" ${FILE}; + sed_runner "s/rmm-cu11=${CURRENT_SHORT_TAG}/rmm-cu11=${NEXT_SHORT_TAG}/g" ${FILE}; done # Doxyfile update diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml new file mode 100644 index 00000000000..a7e5f1a04a6 --- /dev/null +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -0,0 +1,78 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- dask/label/dev +- conda-forge +- nvidia +dependencies: +- aiobotocore>=2.2.0 +- arrow-cpp=9 +- boto3>=1.21.21 +- botocore>=1.24.21 +- c-compiler +- cachetools +- cmake>=3.23.1 +- cubinlinker +- cuda-python>=11.7.1,<12.0 +- cudatoolkit=11.5 +- cupy>=9.5.0,<12.0.0a0 +- cxx-compiler +- cython>=0.29,<0.30 +- dask-cuda=22.12.* +- dask>=2022.9.2 +- distributed>=2022.9.2 +- dlpack>=0.5,<0.6.0a0 +- doxygen=1.8.20 +- fastavro>=0.22.9 +- fsspec>=0.6.0 +- gcc_linux-64=9.* +- hypothesis +- ipython +- librdkafka=1.7.0 +- mimesis>=4.1.0 +- moto>=4.0.8 +- myst-nb +- nbsphinx +- notebook>=0.5.0 +- numba>=0.56.2 +- numpy +- numpydoc +- nvcc_linux-64=11.5 +- nvtx>=0.2.1 +- packaging +- pandas>=1.0,<1.6.0dev0 +- pandoc<=2.0.0 +- pip +- pre-commit +- protobuf>=3.20.1,<3.21.0a0 +- ptxcompiler +- pyarrow=9.0.0 +- pydata-sphinx-theme +- pytest +- pytest-benchmark +- pytest-cases +- pytest-cov +- pytest-xdist +- python-confluent-kafka=1.7.0 +- python-snappy>=0.6.0 +- python>=3.8,<3.10 +- pytorch<1.12.0 +- rmm=22.12.* +- s3fs>=2022.3.0 +- scikit-build>=0.13.1 +- scipy +- sphinx +- sphinx-autobuild +- sphinx-copybutton +- sphinx-markdown-tables +- sphinxcontrib-websupport +- streamz +- sysroot_linux-64==2.17 +- transformers +- typing_extensions +- pip: + - git+https://github.com/python-streamz/streamz.git@master + - pyorc +name: all_cuda-115_arch-x86_64 diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml deleted file mode 100644 index 2cad2002456..00000000000 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. - -name: cudf_dev -channels: - - rapidsai - - rapidsai-nightly - - dask/label/dev - - conda-forge - - nvidia -dependencies: - - c-compiler - - cxx-compiler - - clang=11.1.0 - - clang-tools=11.1.0 - - cupy>=9.5.0,<12.0.0a0 - - rmm=22.12.* - - cmake>=3.23.1 - - cmake_setuptools>=0.1.3 - - scikit-build>=0.13.1 - - python>=3.8,<3.10 - - numba>=0.56.2 - - numpy - - pandas>=1.0,<1.6.0dev0 - - pyarrow=9 - - fastavro>=0.22.9 - - python-snappy>=0.6.0 - - notebook>=0.5.0 - - cython>=0.29,<0.30 - - fsspec>=0.6.0 - - pytest - - pytest-benchmark - - pytest-cases - - pytest-xdist - - sphinx - - sphinxcontrib-websupport - - nbsphinx - - numpydoc - - ipython - - pandoc<=2.0.0 - - cudatoolkit=11.5 - - cuda-python>=11.7.1,<12.0 - - pip - - doxygen=1.8.20 - - typing_extensions - - pre-commit - - dask>=2022.9.2 - - distributed>=2022.9.2 - - streamz - - arrow-cpp=9 - - dlpack>=0.5,<0.6.0a0 - - double-conversion - - rapidjson - - hypothesis - - sphinx-markdown-tables - - sphinx-copybutton - - sphinx-autobuild - - myst-nb - - scipy - - dask-cuda=22.12.* - - mimesis>=4.1.0 - - packaging - - protobuf>=3.20.1,<3.21.0a0 - - nvtx>=0.2.1 - - cachetools - - transformers<=4.10.3 - - pydata-sphinx-theme - - pyorc - - librdkafka=1.7.0 - - python-confluent-kafka=1.7.0 - - moto>=3.1.6 - - boto3>=1.21.21 - - botocore>=1.24.21 - - aiobotocore>=2.2.0 - - s3fs>=2022.3.0 - - werkzeug<2.2.0 # Temporary transient dependency pinning to avoid URL-LIB3 + moto timeouts - - pytorch<1.12.0 - - cubinlinker # [linux64] - - gcc_linux-64=9.* # [linux64] - - sysroot_linux-64==2.17 # [linux64] - - nvcc_linux-64=11.5 - # Un-comment following lines for ARM specific packages. - # - gcc_linux-aarch64=9.* # [aarch64] - # - sysroot_linux-aarch64==2.17 # [aarch64] - # - nvcc_linux-aarch64=11.5 # [aarch64] - - pip: - - git+https://github.com/python-streamz/streamz.git@master diff --git a/dependencies.yaml b/dependencies.yaml new file mode 100644 index 00000000000..b8470f02f86 --- /dev/null +++ b/dependencies.yaml @@ -0,0 +1,207 @@ +# Dependency list for https://github.com/rapidsai/dependency-file-generator +files: + all: + output: conda + matrix: + cuda: ["11.5"] + arch: [x86_64] + includes: + - build + - cudatoolkit + - develop + - doc + - notebook + - run + - test_python + test_cpp: + output: none + includes: + - cudatoolkit + test_python: + output: none + includes: + - cudatoolkit + - py_version + - test_python + checks: + output: none + includes: + - build + - develop + - py_version +channels: + - rapidsai + - rapidsai-nightly + - dask/label/dev + - conda-forge + - nvidia +dependencies: + build: + common: + - output_types: [conda, requirements] + packages: + - cmake>=3.23.1 + - cuda-python>=11.7.1,<12.0 + - cython>=0.29,<0.30 + - dlpack>=0.5,<0.6.0a0 + - pyarrow=9.0.0 + - rmm=22.12.* + - scikit-build>=0.13.1 + - output_types: conda + packages: + - arrow-cpp=9 + - c-compiler + - cxx-compiler + - librdkafka=1.7.0 + - protobuf>=3.20.1,<3.21.0a0 + - python>=3.8,<3.10 + specific: + - output_types: conda + matrices: + - matrix: + arch: x86_64 + packages: + - gcc_linux-64=9.* + - sysroot_linux-64==2.17 + - matrix: + arch: aarch64 + packages: + - gcc_linux-aarch64=9.* + - sysroot_linux-aarch64==2.17 + - output_types: conda + matrices: + - matrix: + arch: x86_64 + cuda: "11.5" + packages: + - nvcc_linux-64=11.5 + - matrix: + arch: aarch64 + cuda: "11.5" + packages: + - nvcc_linux-aarch64=11.5 + cudatoolkit: + specific: + - output_types: conda + matrices: + - matrix: + cuda: "11.2" + packages: + - cudatoolkit=11.2 + - matrix: + cuda: "11.4" + packages: + - cudatoolkit=11.4 + - matrix: + cuda: "11.5" + packages: + - cudatoolkit=11.5 + develop: + common: + - output_types: [conda, requirements] + packages: + - pre-commit + - output_types: conda + packages: + - doxygen=1.8.20 # pre-commit hook needs a specific version. + doc: + common: + - output_types: [conda, requirements] + packages: + - myst-nb + - nbsphinx + - numpydoc + - pandoc<=2.0.0 # We should check and fix all "<=" pinnings + - pydata-sphinx-theme + - sphinx + - sphinx-autobuild + - sphinx-copybutton + - sphinx-markdown-tables + - sphinxcontrib-websupport + notebook: + common: + - output_types: [conda, requirements] + packages: + - ipython + - notebook>=0.5.0 + py_version: + specific: + - output_types: conda + matrices: + - matrix: + py: "3.8" + packages: + - python=3.8 + - matrix: + py: "3.9" + packages: + - python=3.9 + run: + common: + - output_types: [conda, requirements] + packages: + - cachetools + - dask>=2022.9.2 + - distributed>=2022.9.2 + - fsspec>=0.6.0 + - numba>=0.56.2 + - numpy + - nvtx>=0.2.1 + - packaging + - pandas>=1.0,<1.6.0dev0 + - python-confluent-kafka=1.7.0 + - streamz + - typing_extensions + - output_types: conda + packages: + - cubinlinker + - cupy>=9.5.0,<12.0.0a0 + - pip + - pip: + - git+https://github.com/python-streamz/streamz.git@master + - pyorc + - ptxcompiler + - rmm=22.12.* + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm, cubinlinker, ptxcompiler. + - --extra-index-url=https://pypi.ngc.nvidia.com + - cubinlinker-cu11 + - git+https://github.com/python-streamz/streamz.git@master + - ptxcompiler-cu11 + - pyorc + - rmm-cu11=22.12.* + specific: + - output_types: requirements + matrices: + - matrix: + arch: x86_64 + packages: + - cupy-cuda115>=9.5.0,<12.0.0a0 # TODO: This might change to cupy-cuda11x? + - matrix: + arch: aarch64 + packages: + - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. + test_python: + common: + - output_types: [conda, requirements] + packages: + - aiobotocore>=2.2.0 + - boto3>=1.21.21 + - botocore>=1.24.21 + - dask-cuda=22.12.* + - fastavro>=0.22.9 + - hypothesis + - mimesis>=4.1.0 + - moto>=4.0.8 + - pytest + - pytest-benchmark + - pytest-cases + - pytest-cov + - pytest-xdist + - python-snappy>=0.6.0 + - pytorch<1.12.0 # We should check and fix all "<=" pinnings + - s3fs>=2022.3.0 + - scipy + - transformers