diff --git a/ci/conda/environments/ci_env.yml b/ci/conda/environments/ci_env.yml index 103ffe8c4..ad05425dd 100644 --- a/ci/conda/environments/ci_env.yml +++ b/ci/conda/environments/ci_env.yml @@ -19,3 +19,4 @@ channels: - conda-forge dependencies: - codecov=2.1 + - conda-merge>=0.2 diff --git a/ci/conda/environments/dev_env.yml b/ci/conda/environments/dev_env.yml index c639ed62e..4d3e066f9 100644 --- a/ci/conda/environments/dev_env.yml +++ b/ci/conda/environments/dev_env.yml @@ -1,8 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dependencies needed for development environment. Runtime deps are in meta.yml +name: mrc channels: - - conda-forge - rapidsai + - nvidia/label/cuda-11.8.0 - nvidia - rapidsai-nightly + - conda-forge dependencies: - autoconf>=2.69 - bash-completion @@ -10,12 +28,11 @@ dependencies: - boost-cpp=1.74 - ccache - cmake=3.24 - - cuda-nvml-dev=11.8 - - cudatoolkit=11.8 - - cython>=0.29,<0.30 + - cuda-toolkit # Version comes from the channel above + - cxx-compiler # Sets up the distro versions of our compilers - doxygen=1.9.2 + - flake8 - flatbuffers=2.0 - - gcc_linux-64=11.2 - gcovr=5.0 - gdb - gflags=2.2 @@ -25,7 +42,7 @@ dependencies: - graphviz=3.0 - grpc-cpp=1.48 - gtest=1.10 - - gxx_linux-64=11.2 + - gxx=11.2 # Specifies which versions of GXX and GCC to use - isort - jinja2=3.0 - lcov=1.15 @@ -37,19 +54,17 @@ dependencies: - nlohmann_json=3.9 - numactl-libs-cos7-x86_64 - numpy>=1.21 - - nvcc_linux-64=11.8 - pip - pkg-config=0.29 - pybind11-stubgen=0.10 + - pytest + - pytest-timeout - python=3.10 - - scikit-build>=0.12 + - scikit-build>=0.17 - spdlog=1.8.5 - sysroot_linux-64=2.17 - ucx=1.13 - - pip: - - cython - - flake8 - - numpy==1.21.2 - - pytest - - pytest-timeout - - yapf + - yapf + + # Remove once `mamba repoquery whoneeds cudatoolkit` is empty. For now, we need to specify a version + - cudatoolkit=11.8 diff --git a/ci/conda/recipes/libmrc/build.sh b/ci/conda/recipes/libmrc/build.sh index 34246be6f..cad5dfb64 100644 --- a/ci/conda/recipes/libmrc/build.sh +++ b/ci/conda/recipes/libmrc/build.sh @@ -51,17 +51,17 @@ export CC=${GCC} export CXX=${GXX} # Common CMake args -CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}" -CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}" -CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}" CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${MRC_BUILD_TYPE} ${CMAKE_ARGS}" -CMAKE_ARGS="-DMRC_USE_CONDA=ON ${CMAKE_ARGS}" -CMAKE_ARGS="-DMRC_USE_CCACHE=OFF ${CMAKE_ARGS}" -CMAKE_ARGS="-DMRC_BUILD_PYTHON=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}" -CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}" +CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DMRC_BUILD_PYTHON=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DMRC_RAPIDS_VERSION=${rapids_version} ${CMAKE_ARGS}" +CMAKE_ARGS="-DMRC_USE_CCACHE=OFF ${CMAKE_ARGS}" +CMAKE_ARGS="-DMRC_USE_CONDA=ON ${CMAKE_ARGS}" +CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" CMAKE_ARGS="-DUCX_VERSION=${ucx} ${CMAKE_ARGS}" echo "CC : ${CC}" diff --git a/ci/conda/recipes/libmrc/conda_build_config.yaml b/ci/conda/recipes/libmrc/conda_build_config.yaml index 73369ef27..5ba3ff7f5 100644 --- a/ci/conda/recipes/libmrc/conda_build_config.yaml +++ b/ci/conda/recipes/libmrc/conda_build_config.yaml @@ -20,7 +20,7 @@ cxx_compiler_version: - 11.2 cuda_compiler: - - nvcc + - cuda-nvcc cuda_compiler_version: - 11.8 diff --git a/ci/conda/recipes/libmrc/meta.yaml b/ci/conda/recipes/libmrc/meta.yaml index 818419539..e8771cb10 100644 --- a/ci/conda/recipes/libmrc/meta.yaml +++ b/ci/conda/recipes/libmrc/meta.yaml @@ -43,6 +43,7 @@ requirements: - autoconf >=2.69 - ccache - cmake >=3.24 + - cuda-cudart-dev # Needed by CMake to compile a test application - libtool - ninja - numactl-libs-cos7-x86_64 @@ -50,11 +51,9 @@ requirements: - sysroot_linux-64 >=2.17 host: # Libraries necessary to build. Keep sorted! - - libabseil - boost-cpp - - cuda-nvml-dev {{ cuda_version }}.* - - cudatoolkit {{ cuda_version }}.* - - cython 0.29.* + - cuda-cudart-dev + - cuda-nvml-dev - doxygen 1.9.2.* - flatbuffers 2.0.* - gflags @@ -62,16 +61,19 @@ requirements: - gmock 1.10.* - grpc-cpp - gtest 1.10.* + - libabseil - libhwloc 2.5.* - libprotobuf - librmm {{ rapids_version }} - nlohmann_json 3.9.1 - pybind11-abi # See: https://conda-forge.org/docs/maintainer/knowledge_base.html#pybind11-abi-constraints - - pybind11-stubgen 0.10 + - pybind11-stubgen 0.10.5 - python {{ python }} - - scikit-build >=0.12 + - scikit-build >=0.17 - spdlog 1.8.5 - ucx + # Need to specify cudatoolkit to get correct version. Remove once all libraries migrate to cuda-toolkit + - cudatoolkit {{ cuda_version }}.* outputs: - name: libmrc @@ -92,23 +94,27 @@ outputs: - sysroot_linux-64 2.17 host: # Any libraries with weak run_exports need to go here to be added to the run. Keep sorted! - - libabseil # Needed for transitive run_exports from grpc-cpp. Does not need a version - boost-cpp - - flatbuffers 2.0.* - - gflags + - cuda-cudart # Needed to allow pin_compatible to work - glog - grpc-cpp + - libabseil # Needed for transitive run_exports from grpc-cpp. Does not need a version - libhwloc 2.5.* - libprotobuf # Needed for transitive run_exports from grpc-cpp. Does not need a version - librmm {{ rapids_version }} - nlohmann_json 3.9.* - ucx + # Need to specify cudatoolkit to get correct version. Remove once all libraries migrate to cuda-toolkit + - cudatoolkit {{ cuda_version }}.* run: # Manually add any packages necessary for run that do not have run_exports. Keep sorted! - - {{ pin_compatible('flatbuffers', max_pin='x.x')}} + - {{ pin_compatible('cuda-cudart', min_pin='x.x', max_pin='x') }} - {{ pin_compatible('nlohmann_json', max_pin='x.x')}} - {{ pin_compatible('ucx', max_pin='x.x')}} - boost-cpp # Needed to use pin_run_as_build + run_constrained: + # Since we dont explicitly require this but other packages might, constrain the versions + - {{ pin_compatible('cudatoolkit', min_pin='x.x', max_pin='x') }} test: script: test_libmrc.sh files: @@ -121,13 +127,16 @@ outputs: - {{ pin_subpackage("mrc", max_pin="x.x") }} ignore_run_exports: - pypy + missing_dso_whitelist: + - $RPATH/ld-linux-x86-64.so.2 # From python + - $RPATH/libc.so.6 # From python + - $RPATH/libpthread.so.0 # Sample modules include_recipe: False script: move_mrc_files.sh requirements: host: # Only should need libmrc and python. Keep sorted! - {{ pin_subpackage('libmrc', exact=True) }} - - libabseil # mrc does not require abseil at build time. See https://github.com/conda-forge/arrow-cpp-feedstock/issues/814 - python {{ python }} run: - {{ pin_subpackage('libmrc', exact=True) }} @@ -149,8 +158,8 @@ outputs: - cuml {{ rapids_version }}.* # Ensure we can install cuml. This can cause issues solving libabseil about: - home: https://www.nvidia.com/ - summary: A GPU accelerated streaming data library with python bindings - license: Apache-2.0 + home: https://github.com/nv-morpheus/MRC license_family: Apache license_file: LICENSE + license: Apache-2.0 + summary: A GPU accelerated streaming data library with python bindings diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh index 7170497e2..203f4aa62 100755 --- a/ci/conda/recipes/run_conda_build.sh +++ b/ci/conda/recipes/run_conda_build.sh @@ -40,13 +40,6 @@ export CONDA_COMMAND=${CONDA_COMMAND:-"mambabuild"} # Get the path to the morpheus git folder export MRC_ROOT=${MRC_ROOT:-$(git rev-parse --show-toplevel)} -export CUDA="$(conda list | grep cudatoolkit | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+")" -export PYTHON_VER="$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")" -export CUDA=11.8.0 -echo "CUDA : ${CUDA}" -echo "PYTHON_VER : ${PYTHON_VER}" -echo "" - export PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)} # Export variables for the cache @@ -103,7 +96,7 @@ if hasArg quick; then fi # And default channels -CONDA_ARGS_ARRAY+=("-c" "rapidsai" "-c" "nvidia" "-c" "conda-forge" "-c" "main") +CONDA_ARGS_ARRAY+=("-c" "rapidsai" "-c" "nvidia/label/cuda-11.8.0" "-c" "nvidia" "-c" "conda-forge" "-c" "main") # Set GIT_VERSION to set the project version inside of meta.yaml export GIT_VERSION="$(get_version)" diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index 8765d51e6..eab8ac4d5 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -37,6 +37,7 @@ export BUILD_CC=${BUILD_CC:-"gcc"} export CONDA_ENV_YML="${MRC_ROOT}/ci/conda/environments/dev_env.yml" export CONDA_CLANG_ENV_YML="${MRC_ROOT}/ci/conda/environments/clang_env.yml" +export CONDA_CI_ENV_YML="${MRC_ROOT}/ci/conda/environments/ci_env.yml" export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMRC_BUILD_BENCHMARKS=ON -DMRC_BUILD_EXAMPLES=ON -DMRC_BUILD_PYTHON=ON -DMRC_BUILD_TESTS=ON -DMRC_USE_CONDA=ON -DMRC_PYTHON_BUILD_STUBS=ON" export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DMRC_ENABLE_CODECOV=ON -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON -DMRC_PYTHON_INPLACE_BUILD:BOOL=ON" @@ -73,10 +74,33 @@ function print_env_vars() { function update_conda_env() { rapids-logger "Checking for updates to conda env" - rapids-mamba-retry env update -n mrc -q --file ${CONDA_ENV_YML} - rapids-mamba-retry env update -n mrc -q --file ${CONDA_CLANG_ENV_YML} + + # Deactivate the environment first before updating conda deactivate + + # Make sure we have the conda-merge package installed + if [[ -z "$(conda list | grep conda-merge)" ]]; then + rapids-mamba-retry install -n mrc -c conda-forge "conda-merge>=0.2" + fi + + # Create a temp directory which we store the combined environment file in + condatmpdir=$(mktemp -d) + + # Merge the environments together so we can use --prune. Otherwise --prune + # will clobber the last env update + conda run -n mrc --live-stream conda-merge ${CONDA_ENV_YML} ${CONDA_CLANG_ENV_YML} ${CONDA_CI_ENV_YML} > ${condatmpdir}/merged_env.yml + + # Update the conda env with prune remove excess packages (in case one was removed from the env) + rapids-mamba-retry env update -n mrc -q --prune --file ${condatmpdir}/merged_env.yml + + # Delete the temp directory + rm -rf ${condatmpdir} + + # Finally, reactivate conda activate mrc + + rapids-logger "Final Conda Environment" + conda list } print_env_vars diff --git a/docs/quickstart/CMakeLists.txt b/docs/quickstart/CMakeLists.txt index bc25f2422..1b87c4b87 100644 --- a/docs/quickstart/CMakeLists.txt +++ b/docs/quickstart/CMakeLists.txt @@ -39,7 +39,8 @@ rapids_cpm_init() # Set the option prefix to match the outer project before including. Must be before find_package(mrc) set(OPTION_PREFIX "MRC") -morpheus_utils_python_ensure_loaded() + +morpheus_utils_python_configure() rapids_find_package(mrc REQUIRED) rapids_find_package(CUDAToolkit REQUIRED) diff --git a/external/utilities b/external/utilities index b5a53cf77..429ae1840 160000 --- a/external/utilities +++ b/external/utilities @@ -1 +1 @@ -Subproject commit b5a53cf77c9c9e9406939f1e6bbc163ba16b0f0f +Subproject commit 429ae1840d02cf4cd2d4f1bbbac85a61f7ee8907 diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 03aa35d73..b9efadd17 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -19,7 +19,7 @@ find_package(CUDAToolkit REQUIRED) # Ensure python is configured -morpheus_utils_python_ensure_loaded() +morpheus_utils_python_configure() morpheus_utils_print_python_info() diff --git a/python/setup.py b/python/setup.py index ae7a60876..cc37c7077 100644 --- a/python/setup.py +++ b/python/setup.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import find_packages +from setuptools import find_namespace_packages from setuptools import setup import versioneer @@ -23,7 +23,7 @@ # - Python package generation ------------------------------------------------ setup( - name='mrc', + name="mrc", description="mrc", version=versioneer.get_version(), classifiers=[ @@ -32,9 +32,10 @@ author="NVIDIA Corporation", setup_requires=[], include_package_data=True, - packages=find_packages(include=['mrc', 'mrc.*'], exclude=['tests']), + packages=find_namespace_packages(include=["mrc*"], exclude=["tests", "mrc.core.segment.module_definitions"]), package_data={ "mrc": ["_pymrc/*.so"] # Add the pymrc library for the root package }, license="Apache", - cmdclass=versioneer.get_cmdclass()) + cmdclass=versioneer.get_cmdclass(), + zip_safe=False)