From 6ee17408dbffc21c4bd76ba7b50d4efa8749ad2d Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Wed, 19 Apr 2023 22:19:53 +0000 Subject: [PATCH 01/12] Updating dependencies to use `cuda-toolkit` instead of `cudatoolkit` --- .devcontainer/init.sh | 0 ci/conda/environments/dev_env.yml | 28 +++++++++---------- .../recipes/libmrc/conda_build_config.yaml | 2 +- ci/conda/recipes/libmrc/meta.yaml | 7 +++-- docs/quickstart/CMakeLists.txt | 3 +- external/utilities | 2 +- python/CMakeLists.txt | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) mode change 100644 => 100755 .devcontainer/init.sh diff --git a/.devcontainer/init.sh b/.devcontainer/init.sh old mode 100644 new mode 100755 diff --git a/ci/conda/environments/dev_env.yml b/ci/conda/environments/dev_env.yml index c639ed62e..49c4685a8 100644 --- a/ci/conda/environments/dev_env.yml +++ b/ci/conda/environments/dev_env.yml @@ -1,8 +1,9 @@ channels: - - conda-forge - rapidsai + - nvidia/label/cuda-11.8.0 - nvidia - rapidsai-nightly + - conda-forge dependencies: - autoconf>=2.69 - bash-completion @@ -10,12 +11,11 @@ dependencies: - boost-cpp=1.74 - ccache - cmake=3.24 - - cuda-nvml-dev=11.8 - - cudatoolkit=11.8 - - cython>=0.29,<0.30 + - cuda-toolkit=11.8 + - cxx-compiler # Sets up the distro versions of our compilers - doxygen=1.9.2 + - flake8 - flatbuffers=2.0 - - gcc_linux-64=11.2 - gcovr=5.0 - gdb - gflags=2.2 @@ -25,7 +25,7 @@ dependencies: - graphviz=3.0 - grpc-cpp=1.48 - gtest=1.10 - - gxx_linux-64=11.2 + - gxx=11.2 # Specifies which versions of GXX and GCC to use - isort - jinja2=3.0 - lcov=1.15 @@ -37,19 +37,17 @@ dependencies: - nlohmann_json=3.9 - numactl-libs-cos7-x86_64 - numpy>=1.21 - - nvcc_linux-64=11.8 - pip - pkg-config=0.29 - pybind11-stubgen=0.10 + - pytest + - pytest-timeout - python=3.10 - - scikit-build>=0.12 + - scikit-build>=0.17 - spdlog=1.8.5 - sysroot_linux-64=2.17 - ucx=1.13 - - pip: - - cython - - flake8 - - numpy==1.21.2 - - pytest - - pytest-timeout - - yapf + - yapf + + # Remove once `mamba repoquery whoneeds cudatoolkit` is empty. For now, we need to specify a version + - cudatoolkit=11.8 diff --git a/ci/conda/recipes/libmrc/conda_build_config.yaml b/ci/conda/recipes/libmrc/conda_build_config.yaml index 73369ef27..5ba3ff7f5 100644 --- a/ci/conda/recipes/libmrc/conda_build_config.yaml +++ b/ci/conda/recipes/libmrc/conda_build_config.yaml @@ -20,7 +20,7 @@ cxx_compiler_version: - 11.2 cuda_compiler: - - nvcc + - cuda-nvcc cuda_compiler_version: - 11.8 diff --git a/ci/conda/recipes/libmrc/meta.yaml b/ci/conda/recipes/libmrc/meta.yaml index 818419539..35c1afc6b 100644 --- a/ci/conda/recipes/libmrc/meta.yaml +++ b/ci/conda/recipes/libmrc/meta.yaml @@ -50,8 +50,8 @@ requirements: - sysroot_linux-64 >=2.17 host: # Libraries necessary to build. Keep sorted! - - libabseil - boost-cpp + - cuda-cudart-dev {{ cuda_version }}.* - cuda-nvml-dev {{ cuda_version }}.* - cudatoolkit {{ cuda_version }}.* - cython 0.29.* @@ -62,6 +62,7 @@ requirements: - gmock 1.10.* - grpc-cpp - gtest 1.10.* + - libabseil - libhwloc 2.5.* - libprotobuf - librmm {{ rapids_version }} @@ -69,7 +70,7 @@ requirements: - pybind11-abi # See: https://conda-forge.org/docs/maintainer/knowledge_base.html#pybind11-abi-constraints - pybind11-stubgen 0.10 - python {{ python }} - - scikit-build >=0.12 + - scikit-build >=0.17 - spdlog 1.8.5 - ucx @@ -92,12 +93,12 @@ outputs: - sysroot_linux-64 2.17 host: # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted! - - libabseil # Needed for transitive run_exports from grpc-cpp. Does not need a version - boost-cpp - flatbuffers 2.0.* - gflags - glog - grpc-cpp + - libabseil # Needed for transitive run_exports from grpc-cpp. Does not need a version - libhwloc 2.5.* - libprotobuf # Needed for transitive run_exports from grpc-cpp. Does not need a version - librmm {{ rapids_version }} diff --git a/docs/quickstart/CMakeLists.txt b/docs/quickstart/CMakeLists.txt index bc25f2422..1b87c4b87 100644 --- a/docs/quickstart/CMakeLists.txt +++ b/docs/quickstart/CMakeLists.txt @@ -39,7 +39,8 @@ rapids_cpm_init() # Set the option prefix to match the outer project before including. Must be before find_package(mrc) set(OPTION_PREFIX "MRC") -morpheus_utils_python_ensure_loaded() + +morpheus_utils_python_configure() rapids_find_package(mrc REQUIRED) rapids_find_package(CUDAToolkit REQUIRED) diff --git a/external/utilities b/external/utilities index b5a53cf77..429ae1840 160000 --- a/external/utilities +++ b/external/utilities @@ -1 +1 @@ -Subproject commit b5a53cf77c9c9e9406939f1e6bbc163ba16b0f0f +Subproject commit 429ae1840d02cf4cd2d4f1bbbac85a61f7ee8907 diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 03aa35d73..b9efadd17 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -19,7 +19,7 @@ find_package(CUDAToolkit REQUIRED) # Ensure python is configured -morpheus_utils_python_ensure_loaded() +morpheus_utils_python_configure() morpheus_utils_print_python_info() From ec061e16f31603c5c1709bcdc9f8b7a40f4c2352 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 00:12:01 +0000 Subject: [PATCH 02/12] Updating meta.yaml --- ci/conda/recipes/libmrc/meta.yaml | 17 +++++++++++------ ci/conda/recipes/run_conda_build.sh | 9 +-------- python/setup.py | 6 ++++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ci/conda/recipes/libmrc/meta.yaml b/ci/conda/recipes/libmrc/meta.yaml index 35c1afc6b..b28e128a5 100644 --- a/ci/conda/recipes/libmrc/meta.yaml +++ b/ci/conda/recipes/libmrc/meta.yaml @@ -43,6 +43,7 @@ requirements: - autoconf >=2.69 - ccache - cmake >=3.24 + - cuda-cudart-dev # Needed by CMake to compile a test application - libtool - ninja - numactl-libs-cos7-x86_64 @@ -51,10 +52,8 @@ requirements: host: # Libraries necessary to build. Keep sorted! - boost-cpp - - cuda-cudart-dev {{ cuda_version }}.* - - cuda-nvml-dev {{ cuda_version }}.* - - cudatoolkit {{ cuda_version }}.* - - cython 0.29.* + - cuda-cudart-dev + - cuda-nvml-dev - doxygen 1.9.2.* - flatbuffers 2.0.* - gflags @@ -68,11 +67,13 @@ requirements: - librmm {{ rapids_version }} - nlohmann_json 3.9.1 - pybind11-abi # See: https://conda-forge.org/docs/maintainer/knowledge_base.html#pybind11-abi-constraints - - pybind11-stubgen 0.10 + - pybind11-stubgen 0.10.5 - python {{ python }} - scikit-build >=0.17 - spdlog 1.8.5 - ucx + # Need to specify cudatoolkit to get correct version. Remove once all libraries migrate to cuda-toolkit + - cudatoolkit {{ cuda_version }}.* outputs: - name: libmrc @@ -89,6 +90,7 @@ outputs: - {{ compiler("cuda") }} - {{ compiler("cxx") }} - cmake >=3.24 + - cuda-cudart-dev - numactl-libs-cos7-x86_64 - sysroot_linux-64 2.17 host: @@ -106,10 +108,14 @@ outputs: - ucx run: # Manually add any packages necessary for run that do not have run_exports. Keep sorted! + - {{ pin_compatible('cuda-cudart', min_pin='x.x', max_pin='x') }} - {{ pin_compatible('flatbuffers', max_pin='x.x')}} - {{ pin_compatible('nlohmann_json', max_pin='x.x')}} - {{ pin_compatible('ucx', max_pin='x.x')}} - boost-cpp # Needed to use pin_run_as_build + run_constrained: + # Since we dont explicitly require this but other packages might, constrain the versions + - {{ pin_compatible('cudatoolkit', min_pin='x.x', max_pin='x') }} test: script: test_libmrc.sh files: @@ -128,7 +134,6 @@ outputs: host: # Only should need libmrc and python. Keep sorted! - {{ pin_subpackage('libmrc', exact=True) }} - - libabseil # mrc does not require abseil at build time. See https://github.com/conda-forge/arrow-cpp-feedstock/issues/814 - python {{ python }} run: - {{ pin_subpackage('libmrc', exact=True) }} diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh index 7170497e2..203f4aa62 100755 --- a/ci/conda/recipes/run_conda_build.sh +++ b/ci/conda/recipes/run_conda_build.sh @@ -40,13 +40,6 @@ export CONDA_COMMAND=${CONDA_COMMAND:-"mambabuild"} # Get the path to the morpheus git folder export MRC_ROOT=${MRC_ROOT:-$(git rev-parse --show-toplevel)} -export CUDA="$(conda list | grep cudatoolkit | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+")" -export PYTHON_VER="$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")" -export CUDA=11.8.0 -echo "CUDA : ${CUDA}" -echo "PYTHON_VER : ${PYTHON_VER}" -echo "" - export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)} # Export variables for the cache @@ -103,7 +96,7 @@ if hasArg quick; then fi # And default channels -CONDA_ARGS_ARRAY+=("-c" "rapidsai" "-c" "nvidia" "-c" "conda-forge" "-c" "main") +CONDA_ARGS_ARRAY+=("-c" "rapidsai" "-c" "nvidia/label/cuda-11.8.0" "-c" "nvidia" "-c" "conda-forge" "-c" "main") # Set GIT_VERSION to set the project version inside of meta.yaml export GIT_VERSION="$(get_version)" diff --git a/python/setup.py b/python/setup.py index ae7a60876..d6d17dfc4 100644 --- a/python/setup.py +++ b/python/setup.py @@ -32,9 +32,11 @@ author="NVIDIA Corporation", setup_requires=[], include_package_data=True, - packages=find_packages(include=['mrc', 'mrc.*'], exclude=['tests']), + packages=find_packages(include=['mrc*'], exclude=['tests']), package_data={ "mrc": ["_pymrc/*.so"] # Add the pymrc library for the root package }, license="Apache", - cmdclass=versioneer.get_cmdclass()) + cmdclass=versioneer.get_cmdclass(), + zip_safe=False +) From 8fae721e9c0ee76e60b56b9254ca2deed823b677 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 00:39:11 +0000 Subject: [PATCH 03/12] More meta work --- ci/conda/recipes/libmrc/meta.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ci/conda/recipes/libmrc/meta.yaml b/ci/conda/recipes/libmrc/meta.yaml index b28e128a5..1c21ecb05 100644 --- a/ci/conda/recipes/libmrc/meta.yaml +++ b/ci/conda/recipes/libmrc/meta.yaml @@ -90,14 +90,13 @@ outputs: - {{ compiler("cuda") }} - {{ compiler("cxx") }} - cmake >=3.24 - - cuda-cudart-dev + - cuda-cudart-dev {{ cuda_version }}.* - numactl-libs-cos7-x86_64 - sysroot_linux-64 2.17 host: # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted! - boost-cpp - flatbuffers 2.0.* - - gflags - glog - grpc-cpp - libabseil # Needed for transitive run_exports from grpc-cpp. Does not need a version @@ -106,10 +105,11 @@ outputs: - librmm {{ rapids_version }} - nlohmann_json 3.9.* - ucx + # Need to specify cudatoolkit to get correct version. Remove once all libraries migrate to cuda-toolkit + - cudatoolkit {{ cuda_version }}.* run: # Manually add any packages necessary for run that do not have run_exports. Keep sorted! - {{ pin_compatible('cuda-cudart', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('flatbuffers', max_pin='x.x')}} - {{ pin_compatible('nlohmann_json', max_pin='x.x')}} - {{ pin_compatible('ucx', max_pin='x.x')}} - boost-cpp # Needed to use pin_run_as_build @@ -128,6 +128,10 @@ outputs: - {{ pin_subpackage("mrc", max_pin="x.x") }} ignore_run_exports: - pypy + missing_dso_whitelist: + - $RPATH/ld-linux-x86-64.so.2 # From python + - $RPATH/libc.so.6 # From python + - $RPATH/libpthread.so.0 # Sample modules include_recipe: False script: move_mrc_files.sh requirements: @@ -155,8 +159,8 @@ outputs: - cuml {{ rapids_version }}.* # Ensure we can install cuml. This can cause issues solving libabseil about: - home: https://www.nvidia.com/ - summary: A GPU accelerated streaming data library with python bindings - license: Apache-2.0 + home: https://github.com/nv-morpheus/MRC license_family: Apache license_file: LICENSE + license: Apache-2.0 + summary: A GPU accelerated streaming data library with python bindings From 133c8870d0ea67c27cdf366ce4ee554a5df5128d Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 01:19:55 +0000 Subject: [PATCH 04/12] Adding cuda-cudart to host --- ci/conda/recipes/libmrc/meta.yaml | 5 +++-- python/setup.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/conda/recipes/libmrc/meta.yaml b/ci/conda/recipes/libmrc/meta.yaml index 1c21ecb05..58cc56539 100644 --- a/ci/conda/recipes/libmrc/meta.yaml +++ b/ci/conda/recipes/libmrc/meta.yaml @@ -43,7 +43,7 @@ requirements: - autoconf >=2.69 - ccache - cmake >=3.24 - - cuda-cudart-dev # Needed by CMake to compile a test application + - cuda-cudart-dev {{ cuda_version }}.* # Needed by CMake to compile a test application - libtool - ninja - numactl-libs-cos7-x86_64 @@ -52,7 +52,7 @@ requirements: host: # Libraries necessary to build. Keep sorted! - boost-cpp - - cuda-cudart-dev + - cuda-cudart-dev {{ cuda_version }}.* - cuda-nvml-dev - doxygen 1.9.2.* - flatbuffers 2.0.* @@ -96,6 +96,7 @@ outputs: host: # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted! - boost-cpp + - cuda-cudart - flatbuffers 2.0.* - glog - grpc-cpp diff --git a/python/setup.py b/python/setup.py index d6d17dfc4..6cc0713b1 100644 --- a/python/setup.py +++ b/python/setup.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages +from setuptools import find_namespace_packages from setuptools import setup import versioneer @@ -23,7 +23,7 @@ # - Python package generation ------------------------------------------------ setup( - name='mrc', + name="mrc", description="mrc", version=versioneer.get_version(), classifiers=[ @@ -32,11 +32,11 @@ author="NVIDIA Corporation", setup_requires=[], include_package_data=True, - packages=find_packages(include=['mrc*'], exclude=['tests']), + packages=find_namespace_packages(include=["mrc*"], + exclude=["tests", "mrc._pymrc*", "mrc.core.segment.module_definitions"]), package_data={ "mrc": ["_pymrc/*.so"] # Add the pymrc library for the root package }, license="Apache", cmdclass=versioneer.get_cmdclass(), - zip_safe=False -) + zip_safe=False) From 9aa617d51e7ac1ff684a19d7b843c11cae3b91e2 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 01:46:26 +0000 Subject: [PATCH 05/12] Trying to use install RPATH --- ci/conda/recipes/libmrc/build.sh | 15 ++++++++------- ci/conda/recipes/libmrc/meta.yaml | 1 - 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/conda/recipes/libmrc/build.sh b/ci/conda/recipes/libmrc/build.sh index 34246be6f..beadd121c 100644 --- a/ci/conda/recipes/libmrc/build.sh +++ b/ci/conda/recipes/libmrc/build.sh @@ -51,17 +51,18 @@ export CC=${GCC} export CXX=${GXX} # Common CMake args -CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}" -CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}" -CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}" CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${MRC_BUILD_TYPE} ${CMAKE_ARGS}" -CMAKE_ARGS="-DMRC_USE_CONDA=ON ${CMAKE_ARGS}" -CMAKE_ARGS="-DMRC_USE_CCACHE=OFF ${CMAKE_ARGS}" -CMAKE_ARGS="-DMRC_BUILD_PYTHON=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}" -CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMRC_BUILD_PYTHON=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DMRC_RAPIDS_VERSION=${rapids_version} ${CMAKE_ARGS}" +CMAKE_ARGS="-DMRC_USE_CCACHE=OFF ${CMAKE_ARGS}" +CMAKE_ARGS="-DMRC_USE_CONDA=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" CMAKE_ARGS="-DUCX_VERSION=${ucx} ${CMAKE_ARGS}" echo "CC : ${CC}" diff --git a/ci/conda/recipes/libmrc/meta.yaml b/ci/conda/recipes/libmrc/meta.yaml index 58cc56539..c3262b0be 100644 --- a/ci/conda/recipes/libmrc/meta.yaml +++ b/ci/conda/recipes/libmrc/meta.yaml @@ -97,7 +97,6 @@ outputs: # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted! - boost-cpp - cuda-cudart - - flatbuffers 2.0.* - glog - grpc-cpp - libabseil # Needed for transitive run_exports from grpc-cpp. Does not need a version From 1da1932d6a2701d6782546092a0b00747423e289 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 01:58:05 +0000 Subject: [PATCH 06/12] Removing unnecessary RPATH change --- ci/conda/recipes/libmrc/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/conda/recipes/libmrc/build.sh b/ci/conda/recipes/libmrc/build.sh index beadd121c..cad5dfb64 100644 --- a/ci/conda/recipes/libmrc/build.sh +++ b/ci/conda/recipes/libmrc/build.sh @@ -53,7 +53,6 @@ export CXX=${GXX} # Common CMake args CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${MRC_BUILD_TYPE} ${CMAKE_ARGS}" -CMAKE_ARGS="-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}" From 19d7a3d79c0636c0f811fbd0f8e84e4d73871822 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 15:51:48 +0000 Subject: [PATCH 07/12] Added comments and cleaning up cudart versions --- ci/conda/environments/dev_env.yml | 2 +- ci/conda/recipes/libmrc/meta.yaml | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/conda/environments/dev_env.yml b/ci/conda/environments/dev_env.yml index 49c4685a8..074ada6a0 100644 --- a/ci/conda/environments/dev_env.yml +++ b/ci/conda/environments/dev_env.yml @@ -11,7 +11,7 @@ dependencies: - boost-cpp=1.74 - ccache - cmake=3.24 - - cuda-toolkit=11.8 + - cuda-toolkit # Version comes from the channel above - cxx-compiler # Sets up the distro versions of our compilers - doxygen=1.9.2 - flake8 diff --git a/ci/conda/recipes/libmrc/meta.yaml b/ci/conda/recipes/libmrc/meta.yaml index c3262b0be..e8771cb10 100644 --- a/ci/conda/recipes/libmrc/meta.yaml +++ b/ci/conda/recipes/libmrc/meta.yaml @@ -43,7 +43,7 @@ requirements: - autoconf >=2.69 - ccache - cmake >=3.24 - - cuda-cudart-dev {{ cuda_version }}.* # Needed by CMake to compile a test application + - cuda-cudart-dev # Needed by CMake to compile a test application - libtool - ninja - numactl-libs-cos7-x86_64 @@ -52,7 +52,7 @@ requirements: host: # Libraries necessary to build. Keep sorted! - boost-cpp - - cuda-cudart-dev {{ cuda_version }}.* + - cuda-cudart-dev - cuda-nvml-dev - doxygen 1.9.2.* - flatbuffers 2.0.* @@ -90,13 +90,12 @@ outputs: - {{ compiler("cuda") }} - {{ compiler("cxx") }} - cmake >=3.24 - - cuda-cudart-dev {{ cuda_version }}.* - numactl-libs-cos7-x86_64 - sysroot_linux-64 2.17 host: # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted! - boost-cpp - - cuda-cudart + - cuda-cudart # Needed to allow pin_compatible to work - glog - grpc-cpp - libabseil # Needed for transitive run_exports from grpc-cpp. Does not need a version From 1faa2cf90f66492696d51eb9ad9e3fcc8d52582c Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 15:53:37 +0000 Subject: [PATCH 08/12] Temporarily enabling CI to build the conda package --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1a6cfd061..83d2640d7 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -47,7 +47,7 @@ jobs: uses: ./.github/workflows/ci_pipe.yml with: run_check: ${{ startsWith(github.ref_name, 'pull-request/') }} - run_package_conda: ${{ !startsWith(github.ref_name, 'pull-request/') }} + run_package_conda: true # ${{ !startsWith(github.ref_name, 'pull-request/') }} container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230412 test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230412 secrets: From f7a4821055b3a6f68f3d1ad04e3d8cf7e8747ede Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 17:06:47 +0000 Subject: [PATCH 09/12] Trying out using conda-merge in CI to combine environment files --- ci/conda/environments/ci_env.yml | 1 + ci/conda/environments/dev_env.yml | 17 +++++++++++++++++ ci/scripts/github/common.sh | 25 +++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/ci/conda/environments/ci_env.yml b/ci/conda/environments/ci_env.yml index 103ffe8c4..ad05425dd 100644 --- a/ci/conda/environments/ci_env.yml +++ b/ci/conda/environments/ci_env.yml @@ -19,3 +19,4 @@ channels: - conda-forge dependencies: - codecov=2.1 + - conda-merge>=0.2 diff --git a/ci/conda/environments/dev_env.yml b/ci/conda/environments/dev_env.yml index 074ada6a0..4d3e066f9 100644 --- a/ci/conda/environments/dev_env.yml +++ b/ci/conda/environments/dev_env.yml @@ -1,3 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dependencies needed for development environment. Runtime deps are in meta.yml +name: mrc channels: - rapidsai - nvidia/label/cuda-11.8.0 diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index 8765d51e6..e402db6a1 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -37,6 +37,7 @@ export BUILD_CC=${BUILD_CC:-"gcc"} export CONDA_ENV_YML="${MRC_ROOT}/ci/conda/environments/dev_env.yml" export CONDA_CLANG_ENV_YML="${MRC_ROOT}/ci/conda/environments/clang_env.yml" +export CONDA_CI_ENV_YML="${MRC_ROOT}/ci/conda/environments/ci_env.yml" export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMRC_BUILD_BENCHMARKS=ON -DMRC_BUILD_EXAMPLES=ON -DMRC_BUILD_PYTHON=ON -DMRC_BUILD_TESTS=ON -DMRC_USE_CONDA=ON -DMRC_PYTHON_BUILD_STUBS=ON" export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DMRC_ENABLE_CODECOV=ON -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON -DMRC_PYTHON_INPLACE_BUILD:BOOL=ON" @@ -73,9 +74,29 @@ function print_env_vars() { function update_conda_env() { rapids-logger "Checking for updates to conda env" - rapids-mamba-retry env update -n mrc -q --file ${CONDA_ENV_YML} - rapids-mamba-retry env update -n mrc -q --file ${CONDA_CLANG_ENV_YML} + + # Deactivate the environment first before updating conda deactivate + + # Make sure we have the conda-merge package installed + if [[ -z "$(conda list | grep conda-merge)" ]]; then + rapids-mamba-retry install -n mrc -c conda-forge "conda-merge>=0.2" + fi + + # Create a temp directory which we store the combined environment file in + condatmpdir=$(mktemp -d) + + # Merge the environments together so we can use --prune. Otherwise --prune + # will clobber the last env update + conda run -n mrc --live-stream conda-merge ${CONDA_ENV_YML} ${CONDA_CLANG_ENV_YML} ${CONDA_CI_ENV_YML} > ${condatmpdir}/merged_env.yml + + # Update the conda env with prune remove excess packages (in case one was removed from the env) + rapids-mamba-retry env update -n mrc -q --prune --file ${condatmpdir}/merged_env.yml + + # Delete the temp directory + rm -rf ${condatmpdir} + + # Finally, reactivate conda activate mrc } From f8914d9441b79868f23eb8c073b480a9d931c3f9 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 18:45:46 +0000 Subject: [PATCH 10/12] Reverting the package change in CI --- .github/workflows/pull_request.yml | 2 +- python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 83d2640d7..1a6cfd061 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -47,7 +47,7 @@ jobs: uses: ./.github/workflows/ci_pipe.yml with: run_check: ${{ startsWith(github.ref_name, 'pull-request/') }} - run_package_conda: true # ${{ !startsWith(github.ref_name, 'pull-request/') }} + run_package_conda: ${{ !startsWith(github.ref_name, 'pull-request/') }} container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230412 test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230412 secrets: diff --git a/python/setup.py b/python/setup.py index 6cc0713b1..e39571021 100644 --- a/python/setup.py +++ b/python/setup.py @@ -33,7 +33,7 @@ setup_requires=[], include_package_data=True, packages=find_namespace_packages(include=["mrc*"], - exclude=["tests", "mrc._pymrc*", "mrc.core.segment.module_definitions"]), + exclude=["tests", "mrc.core.segment.module_definitions"]), package_data={ "mrc": ["_pymrc/*.so"] # Add the pymrc library for the root package }, From 23dcbef2ec24da8b40a832582fbfa32c181fba9d Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 18:49:35 +0000 Subject: [PATCH 11/12] Adding final env listing --- ci/scripts/github/common.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index e402db6a1..eab8ac4d5 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -98,6 +98,9 @@ function update_conda_env() { # Finally, reactivate conda activate mrc + + rapids-logger "Final Conda Environment" + conda list } print_env_vars From 8bb8b63c883d1524ab62132c5cdb0c2125ab22de Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 20 Apr 2023 21:06:38 +0000 Subject: [PATCH 12/12] Fixing formatting issue --- python/setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index e39571021..cc37c7077 100644 --- a/python/setup.py +++ b/python/setup.py @@ -32,8 +32,7 @@ author="NVIDIA Corporation", setup_requires=[], include_package_data=True, - packages=find_namespace_packages(include=["mrc*"], - exclude=["tests", "mrc.core.segment.module_definitions"]), + packages=find_namespace_packages(include=["mrc*"], exclude=["tests", "mrc.core.segment.module_definitions"]), package_data={ "mrc": ["_pymrc/*.so"] # Add the pymrc library for the root package },