diff --git a/.devcontainer/initialize-command.sh b/.devcontainer/initialize-command.sh index a9dd50e320..10110a3ee2 100755 --- a/.devcontainer/initialize-command.sh +++ b/.devcontainer/initialize-command.sh @@ -18,7 +18,8 @@ docker network inspect morpheus >/dev/null 2>&1 || docker network create morpheus # create the parent conda folder so it's found when mounting -mkdir -p ../.conda +mkdir -p .cache/conda/envs +mkdir -p ../.conda/pkgs # create a config directory if it does not exist so it's found when mounting mkdir -p ../.config diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index e35eeb40de..1669029d57 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -69,7 +69,7 @@ outputs: - scikit-build >=0.12 - versioneer-518 run: - # Runtime only requirements. This + setup.y is the definitive runtime requirement list + # Runtime only requirements. This + setup.py is the definitive runtime requirement list - click >=8 - configargparse 1.5.* - cuda-python <=11.7.0 # Needed to ensure cudatoolkit and not cuda-toolkit is installed @@ -91,7 +91,11 @@ outputs: - tqdm 4.* - typing_utils 0.1.* - watchdog 2.1.* - + run_constrained: + # If cuml is installed, we need to constrain these two libraries. See: + # https://docs.rapids.ai/notices/rsn0026/ for libcusolver & libcusparse + - libcusolver<=11.4.1.48 + - libcusparse<12 test: requires: - cudatoolkit {{ cuda_version }}.* @@ -99,22 +103,17 @@ outputs: - pytest - pytest-cov - pytest-benchmark + # test that cuml can be installed in the env + - cuml {{ rapids_version }} source_files: - # - .git/* - docker/conda/environments/* - pyproject.toml - scripts/fetch_data.py - tests/* imports: - morpheus - - morpheus._lib commands: - # Install dependencies that are pip-only - - mamba env update --prefix ${CONDA_PREFIX} --file docker/conda/environments/cuda{{ cuda_version }}_runtime.yml - # Ensure the test data is downloaded - - ./scripts/fetch_data.py fetch tests - # Run pytest - - pytest + - echo # make sure test requirements get installed about: home: https://github.com/nv-morpheus/Morpheus diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index c41c48c0e6..513802ffbb 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -21,21 +21,26 @@ if (VERBOSE) morpheus_utils_print_config() endif() -# Load direct physical package dependencies first, so we fail early. -find_package(Protobuf REQUIRED) -find_package(CUDAToolkit REQUIRED) # Required by Morpheus. Fail early if we don't have it. +# Load direct physical package dependencies first, so we fail early. Add all dependencies to our export set +rapids_find_package(Protobuf + REQUIRED + BUILD_EXPORT_SET ${PROJECT_NAME}-core-exports + INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports +) + +find_package(CUDAToolkit REQUIRED) if(MORPHEUS_BUILD_BENCHMARKS) # google benchmark # - Expects package to pre-exist in the build environment # ================ rapids_find_package(benchmark REQUIRED - GLOBAL_TARGETS benchmark::benchmark - BUILD_EXPORT_SET ${PROJECT_NAME}-exports - INSTALL_EXPORT_SET ${PROJECT_NAME}-exports - FIND_ARGS - CONFIG - ) + GLOBAL_TARGETS benchmark::benchmark + BUILD_EXPORT_SET ${PROJECT_NAME}-exports + INSTALL_EXPORT_SET ${PROJECT_NAME}-exports + FIND_ARGS + CONFIG + ) endif() if(MORPHEUS_BUILD_TESTS) @@ -43,12 +48,12 @@ if(MORPHEUS_BUILD_TESTS) # - Expects package to pre-exist in the build environment # =========== rapids_find_package(GTest REQUIRED - GLOBAL_TARGETS GTest::gtest GTest::gmock GTest::gtest_main GTest::gmock_main - BUILD_EXPORT_SET ${PROJECT_NAME}-exports - INSTALL_EXPORT_SET ${PROJECT_NAME}-exports - FIND_ARGS - CONFIG - ) + GLOBAL_TARGETS GTest::gtest GTest::gmock GTest::gtest_main GTest::gmock_main + BUILD_EXPORT_SET ${PROJECT_NAME}-exports + INSTALL_EXPORT_SET ${PROJECT_NAME}-exports + FIND_ARGS + CONFIG + ) endif() # libcudacxx -- get an explicit lubcudacxx build, matx tries to pull a tag that doesn't exist. diff --git a/docker/Dockerfile b/docker/Dockerfile index 3ff5191e5e..645cd704a2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -80,9 +80,8 @@ WORKDIR ${MORPHEUS_ROOT} # Install mamba to speed the solve up RUN conda config --set ssl_verify false &&\ conda config --add pkgs_dirs /opt/conda/pkgs &&\ - conda config --env --add channels conda-forge &&\ # Install mamba, boa and git here. Conda build breaks with other git installs - /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.10" "git >=2.35.3" "git-lfs" "python=${PYTHON_VER}" "tini=0.19" &&\ + /opt/conda/bin/mamba install -y -n base -c conda-forge "boa" "git >=2.35.3" "git-lfs" "python=${PYTHON_VER}" "tini=0.19" &&\ source activate base &&\ git lfs install # conda clean -afy @@ -148,10 +147,10 @@ RUN apt update && \ cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvml-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \ + libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \ apt clean && \ rm -rf /var/lib/apt/lists/* @@ -163,7 +162,7 @@ RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locke source activate base &&\ # Need to get around recent versions of git locking paths until they are deemed safe git config --global --add safe.directory "*" &&\ - MORPHEUS_PYTHON_BUILD_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh morpheus + MORPHEUS_PYTHON_BUILD_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld ./ci/conda/recipes/run_conda_build.sh morpheus # sid_visualization is a submodule we need to init RUN git submodule update --init --recursive @@ -212,13 +211,14 @@ ARG CUDA_MINOR_VER RUN apt update && \ DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \ apt install --no-install-recommends -y \ + cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-cudart-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ nodejs \ npm && \ apt clean && \ @@ -247,7 +247,6 @@ ARG MORPHEUS_USER="root" RUN source activate morpheus \ && MORPHEUS_ROOT=/workspace \ CONDA_BLD_PATH=/opt/conda/conda-bld \ - CONDA_ARGS="--no-test" \ ./ci/conda/recipes/run_conda_build.sh pydebug \ && ./ci/conda/recipes/python_dbg_install.sh \ -s $( ls /opt/conda/conda-bld/src_cache/Python-${PYTHON_VER}*.tar.xz ) \