From d7323a11ed72baf06b8bf0fdacc0a6bc1e53af63 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 7 Jul 2023 16:51:26 -0500 Subject: [PATCH] Update to CUDF 23.06 (#1020) Closes #995. Depends on https://github.com/nv-morpheus/MRC/pull/346. Authors: - Christopher Harris (https://github.com/cwharris) - Michael Demoret (https://github.com/mdemoret-nv) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: https://github.com/nv-morpheus/Morpheus/pull/1020 --- CMakeLists.txt | 2 +- ci/conda/recipes/morpheus/conda_build_config.yaml | 2 +- ci/conda/recipes/morpheus/meta.yaml | 2 +- docker/Dockerfile | 2 +- docker/build_container.sh | 2 +- docker/conda/environments/cuda11.8_dev.yml | 8 ++++---- docker/conda/environments/cuda11.8_examples.yml | 6 +++--- docs/source/developer_guide/contributing.md | 8 ++++---- examples/digital_fingerprinting/production/conda_env.yml | 2 +- .../production/morpheus/benchmarks/README.md | 2 +- examples/gnn_fraud_detection_pipeline/requirements.yml | 6 +++--- examples/ransomware_detection/README.md | 2 +- morpheus/_lib/src/io/serializers.cpp | 2 +- 13 files changed, 23 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc35c3595b..2d0b7716c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,7 +38,7 @@ option(MORPHEUS_USE_IWYU "Enable running include-what-you-use as part of the bui set(MORPHEUS_PY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/wheel" CACHE STRING "Location to install the python directory") -set(MORPHEUS_RAPIDS_VERSION "23.02" CACHE STRING "Sets default versions for RAPIDS libraries.") +set(MORPHEUS_RAPIDS_VERSION "23.06" CACHE STRING "Sets default versions for RAPIDS libraries.") set(MORPHEUS_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data") mark_as_advanced(MORPHEUS_CACHE_DIR) diff --git a/ci/conda/recipes/morpheus/conda_build_config.yaml b/ci/conda/recipes/morpheus/conda_build_config.yaml index 586a875dca..b4d6fe33d0 100644 --- a/ci/conda/recipes/morpheus/conda_build_config.yaml +++ b/ci/conda/recipes/morpheus/conda_build_config.yaml @@ -32,4 +32,4 @@ boost: - 1.74 rapids_version: - - 23.02 + - 23.06 diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index c0cc88fc47..ebd0aae4b0 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -18,7 +18,7 @@ {% set py_version=environ.get('CONDA_PY', '3.10') %} {% set cuda_version='.'.join(environ.get('CUDA', '11.8').split('.')[:2]) %} {% set cuda_major=cuda_version.split('.')[0] %} -{% set rapids_version = "23.02" %} +{% set rapids_version = "23.06" %} package: name: morpheus-split diff --git a/docker/Dockerfile b/docker/Dockerfile index 8085f57c44..bec22db63f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -54,7 +54,7 @@ FROM ${FROM_IMAGE}:${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}.${CUDA_REV_VER}-base-${LI ARG PYTHON_VER ARG IMAGE_TYPE=base ARG RAPIDS_CHANNEL=rapidsai-nightly -ARG RAPIDS_VER=23.02 +ARG RAPIDS_VER=23.06 ARG CONDA_CHANNEL=rapidsai ARG CUDA_MAJOR_VER ARG CUDA_MINOR_VER diff --git a/docker/build_container.sh b/docker/build_container.sh index 9da9903d7c..02aaf6c22a 100755 --- a/docker/build_container.sh +++ b/docker/build_container.sh @@ -32,7 +32,7 @@ LINUX_DISTRO=${LINUX_DISTRO:-ubuntu} LINUX_VER=${LINUX_VER:-22.04} MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-"OFF"} PYTHON_VER=${PYTHON_VER:-3.10} -RAPIDS_VER=${RAPIDS_VER:-23.02} +RAPIDS_VER=${RAPIDS_VER:-23.06} TENSORRT_VERSION=${TENSORRT_VERSION:-8.2.1.3} DOCKER_ARGS="-t ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" diff --git a/docker/conda/environments/cuda11.8_dev.yml b/docker/conda/environments/cuda11.8_dev.yml index f476e79b97..df9594e29f 100644 --- a/docker/conda/environments/cuda11.8_dev.yml +++ b/docker/conda/environments/cuda11.8_dev.yml @@ -35,8 +35,8 @@ dependencies: - cuda-compiler=11.8 - cuda-nvml-dev=11.8 - cuda-toolkit=11.8 - - cudf=23.02 - - cupy=11.6.0 + - cudf=23.06 + - cupy>=12.0.0 - cxx-compiler - cython=0.29.24 - datacompy=0.8 @@ -51,11 +51,11 @@ dependencies: - git-lfs=3.2 - git>=2.35.3 # Needed for wildcards on safe.directory - glog=0.6 - - gmock=1.10 + - gmock>=1.13.0 - gputil - grpc-cpp>=1.43 - grpcio - - gtest=1.10 + - gtest>=1.13.0 - gxx_linux-64=11.2 - include-what-you-use=0.18 - isort diff --git a/docker/conda/environments/cuda11.8_examples.yml b/docker/conda/environments/cuda11.8_examples.yml index d932b4689a..4b018b08d4 100644 --- a/docker/conda/environments/cuda11.8_examples.yml +++ b/docker/conda/environments/cuda11.8_examples.yml @@ -26,10 +26,10 @@ channels: dependencies: - boto3=1.24.59 - chardet=5.0.0 - - cuml=23.02 - - dask==2023.1.1 + - cuml=23.06 + - dask>=2023.1.1 - dill=0.3.6 - - distributed==2023.1.1 + - distributed>=2023.1.1 - mlflow>=2.2.1,<3 - papermill=2.3.4 - s3fs==2022.8.2 diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 53635be46f..97c604b3dc 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -66,7 +66,7 @@ The following instructions are for developers who are getting started with the M All of the following instructions assume several variables have been set: - `MORPHEUS_ROOT`: The Morpheus repository has been checked out at a location specified by this variable. Any non-absolute paths are relative to `MORPHEUS_ROOT`. - `PYTHON_VER`: The desired Python version. Minimum required is `3.10` - - `RAPIDS_VER`: The desired RAPIDS version for all RAPIDS libraries including cuDF and RMM. If in doubt use `23.02` + - `RAPIDS_VER`: The desired RAPIDS version for all RAPIDS libraries including cuDF and RMM. If in doubt use `23.06` - `TRITONCLIENT_VERSION`: The desired Triton client. If in doubt use `22.10` - `CUDA_VER`: The desired CUDA version to use. If in doubt use `11.8` @@ -75,7 +75,7 @@ All of the following instructions assume several variables have been set: ```bash export PYTHON_VER=3.10 -export RAPIDS_VER=23.02 +export RAPIDS_VER=23.06 export TRITONCLIENT_VERSION=22.10 export CUDA_VER=11.8 export MORPHEUS_ROOT=$(pwd)/morpheus @@ -190,7 +190,7 @@ Note: These instructions assume the user is using `mamba` instead of `conda` sin 1. Set up env variables and clone the repo: ```bash export PYTHON_VER=3.10 - export RAPIDS_VER=23.02 + export RAPIDS_VER=23.06 export CUDA_VER=11.8 export MORPHEUS_ROOT=$(pwd)/morpheus git clone https://github.com/nv-morpheus/Morpheus.git $MORPHEUS_ROOT @@ -246,7 +246,7 @@ git submodule update --init --recursive 1. Optional: Install cuML - Many users may wish to install cuML. Due to the complex dependency structure and versioning requirements, we need to specify exact versions of each package. The command to accomplish this is: ```bash - mamba install -c rapidsai -c nvidia -c conda-forge cuml=23.02 + mamba install -c rapidsai -c nvidia -c conda-forge cuml=23.06 ``` 1. Run Morpheus ```bash diff --git a/examples/digital_fingerprinting/production/conda_env.yml b/examples/digital_fingerprinting/production/conda_env.yml index 70b72d08a5..9104972ec1 100644 --- a/examples/digital_fingerprinting/production/conda_env.yml +++ b/examples/digital_fingerprinting/production/conda_env.yml @@ -27,7 +27,7 @@ dependencies: - distributed - kfp - librdkafka - - mlflow>1.29.0,<2 + - mlflow>=2.2.1,<3 - nodejs=17.4.0 - papermill - s3fs==2022.8.2 diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md index eeb78530cf..8ec8aa2c3f 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md @@ -24,7 +24,7 @@ To set up and run the benchmarks on production DFP pipeline, follow the instruct > **Note**: Make sure `gputil`, `dask` and `distributed` are installed in your Conda environment before running the benchmarks. Run the installation command specified below if not. ```bash -conda install gputil dask==2022.7.0 distributed==2022.7.0 +conda install gputil 'dask>=2023.1.1' 'distributed>=2023.1.1' ``` ### Run E2E Benchmarks diff --git a/examples/gnn_fraud_detection_pipeline/requirements.yml b/examples/gnn_fraud_detection_pipeline/requirements.yml index fa3f867ee9..66fa035b27 100644 --- a/examples/gnn_fraud_detection_pipeline/requirements.yml +++ b/examples/gnn_fraud_detection_pipeline/requirements.yml @@ -19,9 +19,9 @@ channels: - conda-forge dependencies: - chardet=5.0.0 - - cuml=23.02 - - dask==2023.1.1 - - distributed==2023.1.1 + - cuml=23.06 + - dask>=2023.1.1 + - distributed>=2023.1.1 - pip - pip: # tensorflow exists in conda-forge but is tied to CUDA-11.3 diff --git a/examples/ransomware_detection/README.md b/examples/ransomware_detection/README.md index c2bc3a092a..7588853a3b 100644 --- a/examples/ransomware_detection/README.md +++ b/examples/ransomware_detection/README.md @@ -63,7 +63,7 @@ Once Triton server finishes starting up, it will display the status of all loade > **Note**: Make sure `dask` and `distributed` are installed in your Conda environment before running the ransomware detection pipeline. Run the installation command specified below if not. ```bash -mamba install dask==2023.1.1 distributed==2023.1.1 +mamba install 'dask>=2023.1.1' 'distributed>=2023.1.1' ``` ## Run Ransomware Detection Pipeline diff --git a/morpheus/_lib/src/io/serializers.cpp b/morpheus/_lib/src/io/serializers.cpp index 6a882150dd..cf7f1d78cd 100644 --- a/morpheus/_lib/src/io/serializers.cpp +++ b/morpheus/_lib/src/io/serializers.cpp @@ -205,7 +205,7 @@ void table_to_parquet( auto destination = cudf::io::sink_info(&sink); auto options_builder = cudf::io::parquet_writer_options_builder(destination, tbl_view); - cudf::io::write_parquet(options_builder.build(), rmm::mr::get_current_device_resource()); + cudf::io::write_parquet(options_builder.build()); if (flush) {