From afb72e8a0a9360e1b8ac48bf7094fa61208e8b1b Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Thu, 16 Feb 2023 20:02:49 +0000 Subject: [PATCH 01/26] reorg --- .devcontainer/Dockerfile | 19 ++----- .devcontainer/devcontainer.json | 45 ++++++++++----- .../gdb/enable_conda_libstd_pretty_print.py | 55 +++++++++++++++++++ .devcontainer/home/.gdbinit | 3 + .../{init.sh => initialize-command.sh} | 6 +- .../morpheus/bin/post-attach-command.sh} | 4 +- .../morpheus/bin/update-content-command.sh | 0 7 files changed, 96 insertions(+), 36 deletions(-) create mode 100644 .devcontainer/home/.config/gdb/enable_conda_libstd_pretty_print.py create mode 100644 .devcontainer/home/.gdbinit rename .devcontainer/{init.sh => initialize-command.sh} (86%) rename .devcontainer/{bin/setup-morpheus-env => opt/morpheus/bin/post-attach-command.sh} (92%) create mode 100755 .devcontainer/opt/morpheus/bin/update-content-command.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b2c61963e3..de7a90b87a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -13,20 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM nvidia/cuda:11.5.1-devel-ubuntu20.04 AS base +FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base -RUN apt-get update && \ - apt-get install --no-install-recommends --yes \ - wget git-lfs gdb iputils-ping - -ENV CONDA_DIR=/opt/conda - -RUN wget "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" && \ - bash Mambaforge-$(uname)-$(uname -m).sh -b -p ${CONDA_DIR} && \ - echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ - echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc - -ENV PATH="${CONDA_DIR}/bin:${PATH}" - -ENV MAMBA_NO_BANNER=1 ENV PATH="${PATH}:/workspaces/morpheus/.devcontainer/bin" + +COPY home/.gdbinit /home/coder/.gdbinit +COPY home/.config/gdb/enable_conda_libstd_pretty_print.py /home/coder/.config/gdb/enable_conda_libstd_pretty_print.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e6dbf8c9fe..c608db42f4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -38,29 +38,40 @@ "containerEnv": { "HOST_MORPHEUS_ROOT": "${localWorkspaceFolder}", - "MORPHEUS_ROOT": "${containerWorkspaceFolder}" + "MORPHEUS_ROOT": "${containerWorkspaceFolder}", + "DEFAULT_CONDA_ENV": "morpheus", + "MAMBA_NO_BANNER": "1" }, - "postStartCommand": "${containerWorkspaceFolder}/.devcontainer/bin/setup-morpheus-env", - - "initializeCommand": [ "./.devcontainer/init.sh" ], + "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ], "remoteUser": "coder", - "mounts": [ - {"type":"bind", "source": "/var/run/docker.sock", "target": "/var/run/docker.sock"}, - {"type":"bind", "source": "${localWorkspaceFolder}/.cache/conda", "target": "/home/coder/.conda"}, - {"type":"bind", "source": "${localEnv:HOME}/.config/gh", "target": "/home/coder/.config/gh"} + { + "type":"bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock" + }, + { + "type":"bind", + "source": "${localWorkspaceFolder}/.cache/conda", + "target": "/home/coder/.conda" + }, + { + "type":"bind", + "source": "${localWorkspaceFolder}/../.config", // parent folder because sister repos are sibling dirs + "target": "/home/coder/.config" + }, + { + "type": "bind", + "source": "${localWorkspaceFolder}/.devcontainer/opt/morpheus", + "target": "/opt/morpheus" + }, ], "features": { - "ghcr.io/devcontainers/features/common-utils:1": { - "uid": "1000", - "gid": "1000", - "username": "coder" - }, "ghcr.io/devcontainers/features/docker-from-docker": {}, - "ghcr.io/devcontainers/features/github-cli": {} + "ghcr.io/devcontainers/features/dotnet:1": {} }, "customizations": { @@ -74,7 +85,11 @@ "nvidia.nsight-vscode-edition", "twxs.cmake", "xaver.clang-format" - ] + ], + "settings": { + "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake", + "cmake.languageSupport.dotnetPath": "/usr/bin/dotnet" + } } }, "settings": { diff --git a/.devcontainer/home/.config/gdb/enable_conda_libstd_pretty_print.py b/.devcontainer/home/.config/gdb/enable_conda_libstd_pretty_print.py new file mode 100644 index 0000000000..f27afa2139 --- /dev/null +++ b/.devcontainer/home/.config/gdb/enable_conda_libstd_pretty_print.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -*- python -*- +import os +import subprocess +import sys + +import gdb + +conda_env_path = os.environ.get("CONDA_PREFIX", None) + +if (conda_env_path is not None): + + gcc_path = os.environ.get("GCC", None) + + if (gcc_path is None): + print( + "Could not find gcc from $GCC: '{}'. Ensure gxx_linux-64, gcc_linux-64, sysroot_linux-64, and gdb have been installed into the current conda environment" + .format(gcc_path)) + else: + # Get the GCC version + result = subprocess.run([gcc_path, '-dumpversion'], stdout=subprocess.PIPE) + gcc_version = result.stdout.decode("utf-8").strip() + + # Build the gcc python path + gcc_python_path = os.path.join(conda_env_path, "share", "gcc-{}".format(gcc_version), "python") + + if (os.path.exists(gcc_python_path)): + + # Add to the path for the pretty printer + sys.path.insert(0, gcc_python_path) + + # Now register the pretty printers + from libstdcxx.v6 import register_libstdcxx_printers + register_libstdcxx_printers(gdb.current_objfile()) + + print("Loaded stdlibc++ pretty printers") + else: + print("Could not find gcc python files at: {}".format(gcc_python_path)) + print( + "Ensure gxx_linux-64, gcc_linux-64, sysroot_linux-64, and gdb have been installed into the current conda environment" + ) diff --git a/.devcontainer/home/.gdbinit b/.devcontainer/home/.gdbinit new file mode 100644 index 0000000000..43e108353b --- /dev/null +++ b/.devcontainer/home/.gdbinit @@ -0,0 +1,3 @@ +add-auto-load-safe-path /opt/conda + +source ~/.config/gdb/enable_conda_libstd_pretty_print.py diff --git a/.devcontainer/init.sh b/.devcontainer/initialize-command.sh similarity index 86% rename from .devcontainer/init.sh rename to .devcontainer/initialize-command.sh index 8f6be9dfc3..7f2918a118 100755 --- a/.devcontainer/init.sh +++ b/.devcontainer/initialize-command.sh @@ -18,7 +18,7 @@ docker network inspect morpheus >/dev/null 2>&1 || docker network create morpheus # create the parent conda folder so it's found when mounting -mkdir -p ./.cache/conda +mkdir -p .cache/conda -# create a github cli config directory if it does not exist so it's found when mounting -mkdir -p ~/.config/gh +# create a config directory if it does not exist so it's found when mounting +mkdir -p ../.config diff --git a/.devcontainer/bin/setup-morpheus-env b/.devcontainer/opt/morpheus/bin/post-attach-command.sh similarity index 92% rename from .devcontainer/bin/setup-morpheus-env rename to .devcontainer/opt/morpheus/bin/post-attach-command.sh index 843479f023..504a27d625 100755 --- a/.devcontainer/bin/setup-morpheus-env +++ b/.devcontainer/opt/morpheus/bin/post-attach-command.sh @@ -17,8 +17,6 @@ # Ensure our ~/.config directory has the correct permissions. If ~/.config did # not exist, and you mount ~/.config/gh from the host, then ~/.config will be # created with root permissions which can break things -sudo mkdir -p ~/.config -sudo chown ${USER}:${USER} ~/.config conda_env_find(){ conda env list | grep "${@}" >/dev/null 2>/dev/null @@ -29,6 +27,6 @@ ENV_NAME=${ENV_NAME:-morpheus} sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc; if conda_env_find "${ENV_NAME}" ; \ -then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml; \ +then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml --prune; \ else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml; \ fi diff --git a/.devcontainer/opt/morpheus/bin/update-content-command.sh b/.devcontainer/opt/morpheus/bin/update-content-command.sh new file mode 100755 index 0000000000..e69de29bb2 From 66993e624d1468dc61b52c290c8dba748c8fad61 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Thu, 16 Feb 2023 22:09:52 +0000 Subject: [PATCH 02/26] add pytest-kafka installation script and VAULT_HOST env variable --- .devcontainer/bin/dev-kafka-logs | 2 ++ .devcontainer/bin/dev-kafka-produce | 2 ++ .devcontainer/bin/dev-kafka-pytest-install | 33 ++++++++++++++++++++++ .devcontainer/bin/dev-kafka-start | 2 ++ .devcontainer/bin/dev-kafka-stop | 2 ++ .devcontainer/bin/dev-triton-start | 2 ++ .devcontainer/bin/dev-triton-stop | 2 ++ .devcontainer/devcontainer.json | 3 +- 8 files changed, 47 insertions(+), 1 deletion(-) create mode 100755 .devcontainer/bin/dev-kafka-pytest-install diff --git a/.devcontainer/bin/dev-kafka-logs b/.devcontainer/bin/dev-kafka-logs index b189222333..b520367198 100755 --- a/.devcontainer/bin/dev-kafka-logs +++ b/.devcontainer/bin/dev-kafka-logs @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + COMPOSE_FILE="${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml" docker compose -f $COMPOSE_FILE logs zookeeper kafka diff --git a/.devcontainer/bin/dev-kafka-produce b/.devcontainer/bin/dev-kafka-produce index 5417e50164..941390ba2f 100755 --- a/.devcontainer/bin/dev-kafka-produce +++ b/.devcontainer/bin/dev-kafka-produce @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + COMPOSE_FILE="${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml" if [[ $# -eq 2 ]]; then diff --git a/.devcontainer/bin/dev-kafka-pytest-install b/.devcontainer/bin/dev-kafka-pytest-install new file mode 100755 index 0000000000..69eef525b4 --- /dev/null +++ b/.devcontainer/bin/dev-kafka-pytest-install @@ -0,0 +1,33 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Replicates the install instructions from the CI runner image +sudo apt update + +# Install openjdk +sudo apt install -y openjdk-11-jdk + +# Clone the repo into home and run setup.py develop + +if [[ ! -d ~/pytest-kafka ]]; then + git clone https://gitlab.com/karolinepauls/pytest-kafka.git ~/pytest-kafka +fi; + +cd ~/pytest-kafka + +python setup.py develop diff --git a/.devcontainer/bin/dev-kafka-start b/.devcontainer/bin/dev-kafka-start index 13126b438e..401d45f25a 100755 --- a/.devcontainer/bin/dev-kafka-start +++ b/.devcontainer/bin/dev-kafka-start @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + COMPOSE_FILE="${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml" docker compose -f $COMPOSE_FILE up -d zookeeper kafka diff --git a/.devcontainer/bin/dev-kafka-stop b/.devcontainer/bin/dev-kafka-stop index 3af2cd7845..a158ae1d8f 100755 --- a/.devcontainer/bin/dev-kafka-stop +++ b/.devcontainer/bin/dev-kafka-stop @@ -14,4 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + docker compose -f ${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml stop kafka zookeeper diff --git a/.devcontainer/bin/dev-triton-start b/.devcontainer/bin/dev-triton-start index 91973afdef..6cb4772e5a 100755 --- a/.devcontainer/bin/dev-triton-start +++ b/.devcontainer/bin/dev-triton-start @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + args="" if [ $# -gt 0 ] ; then diff --git a/.devcontainer/bin/dev-triton-stop b/.devcontainer/bin/dev-triton-stop index 276ea40b4c..7824eefed7 100755 --- a/.devcontainer/bin/dev-triton-stop +++ b/.devcontainer/bin/dev-triton-stop @@ -14,4 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e + docker compose -f ${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml stop triton diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c608db42f4..09f589de28 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -40,7 +40,8 @@ "HOST_MORPHEUS_ROOT": "${localWorkspaceFolder}", "MORPHEUS_ROOT": "${containerWorkspaceFolder}", "DEFAULT_CONDA_ENV": "morpheus", - "MAMBA_NO_BANNER": "1" + "MAMBA_NO_BANNER": "1", + "VAULT_HOST": "https://vault.ops.k8s.rapids.ai" }, "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ], From 1cb3f841f0fc72f8b66323b9d6d47cf07725d1bf Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Thu, 16 Feb 2023 22:12:18 +0000 Subject: [PATCH 03/26] rename pytest-kafka installation script --- .../bin/{dev-kafka-pytest-install => dev-pytest-kafka-install} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .devcontainer/bin/{dev-kafka-pytest-install => dev-pytest-kafka-install} (100%) diff --git a/.devcontainer/bin/dev-kafka-pytest-install b/.devcontainer/bin/dev-pytest-kafka-install similarity index 100% rename from .devcontainer/bin/dev-kafka-pytest-install rename to .devcontainer/bin/dev-pytest-kafka-install From af6d829dfbf01cc85b1a47791928b51445254bd6 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Fri, 17 Feb 2023 18:26:11 +0000 Subject: [PATCH 04/26] move .conda folder to parent dir --- .devcontainer/devcontainer.json | 2 +- .devcontainer/initialize-command.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 09f589de28..6fa6963dc5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -55,7 +55,7 @@ }, { "type":"bind", - "source": "${localWorkspaceFolder}/.cache/conda", + "source": "${localWorkspaceFolder}/../.conda", "target": "/home/coder/.conda" }, { diff --git a/.devcontainer/initialize-command.sh b/.devcontainer/initialize-command.sh index 7f2918a118..a9dd50e320 100755 --- a/.devcontainer/initialize-command.sh +++ b/.devcontainer/initialize-command.sh @@ -18,7 +18,7 @@ docker network inspect morpheus >/dev/null 2>&1 || docker network create morpheus # create the parent conda folder so it's found when mounting -mkdir -p .cache/conda +mkdir -p ../.conda # create a config directory if it does not exist so it's found when mounting mkdir -p ../.config From 4c9cfcfe13ddb0865ab989212f18508d29b35cea Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Thu, 23 Feb 2023 17:04:53 +0000 Subject: [PATCH 05/26] move gdb pretty print files --- .devcontainer/Dockerfile | 3 --- .devcontainer/home/.gdbinit | 3 --- .../opt/morpheus/bin/post-attach-command.sh | 2 +- .../morpheus/bin/update-content-command.sh | 23 +++++++++++++++++++ .devcontainer/opt/morpheus/etc/.gdbinit | 3 +++ .../etc}/enable_conda_libstd_pretty_print.py | 0 6 files changed, 27 insertions(+), 7 deletions(-) delete mode 100644 .devcontainer/home/.gdbinit create mode 100644 .devcontainer/opt/morpheus/etc/.gdbinit rename .devcontainer/{home/.config/gdb => opt/morpheus/etc}/enable_conda_libstd_pretty_print.py (100%) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index de7a90b87a..b8d6dce26a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -16,6 +16,3 @@ FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base ENV PATH="${PATH}:/workspaces/morpheus/.devcontainer/bin" - -COPY home/.gdbinit /home/coder/.gdbinit -COPY home/.config/gdb/enable_conda_libstd_pretty_print.py /home/coder/.config/gdb/enable_conda_libstd_pretty_print.py diff --git a/.devcontainer/home/.gdbinit b/.devcontainer/home/.gdbinit deleted file mode 100644 index 43e108353b..0000000000 --- a/.devcontainer/home/.gdbinit +++ /dev/null @@ -1,3 +0,0 @@ -add-auto-load-safe-path /opt/conda - -source ~/.config/gdb/enable_conda_libstd_pretty_print.py diff --git a/.devcontainer/opt/morpheus/bin/post-attach-command.sh b/.devcontainer/opt/morpheus/bin/post-attach-command.sh index 504a27d625..dbe21ca4ba 100755 --- a/.devcontainer/opt/morpheus/bin/post-attach-command.sh +++ b/.devcontainer/opt/morpheus/bin/post-attach-command.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/.devcontainer/opt/morpheus/bin/update-content-command.sh b/.devcontainer/opt/morpheus/bin/update-content-command.sh index e69de29bb2..8f2893658b 100755 --- a/.devcontainer/opt/morpheus/bin/update-content-command.sh +++ b/.devcontainer/opt/morpheus/bin/update-content-command.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ensure our ~/.config directory has the correct permissions. If ~/.config did +# not exist, and you mount ~/.config/gh from the host, then ~/.config will be +# created with root permissions which can break things + +if [[ ! -f ~/.config/.gdbinit ]]; then + cp /opt/morpheus/etc/.gdbinit ~/.config/.gdbinit +fi diff --git a/.devcontainer/opt/morpheus/etc/.gdbinit b/.devcontainer/opt/morpheus/etc/.gdbinit new file mode 100644 index 0000000000..115ac5e1f9 --- /dev/null +++ b/.devcontainer/opt/morpheus/etc/.gdbinit @@ -0,0 +1,3 @@ +add-auto-load-safe-path /opt/conda + +source /opt/morpheus/etc/enable_conda_libstd_pretty_print.py diff --git a/.devcontainer/home/.config/gdb/enable_conda_libstd_pretty_print.py b/.devcontainer/opt/morpheus/etc/enable_conda_libstd_pretty_print.py similarity index 100% rename from .devcontainer/home/.config/gdb/enable_conda_libstd_pretty_print.py rename to .devcontainer/opt/morpheus/etc/enable_conda_libstd_pretty_print.py From d798d59a3de9d37d866eb160cf27ecad99d6b3d6 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Thu, 23 Feb 2023 17:46:56 +0000 Subject: [PATCH 06/26] add x11 mount --- .devcontainer/devcontainer.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6fa6963dc5..fe11ec1ec1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -48,6 +48,11 @@ "remoteUser": "coder", "mounts": [ + { + "type": "bind", + "source": "/tmp/.X11-unix", + "target": "/tmp/.X11-unix" + }, { "type":"bind", "source": "/var/run/docker.sock", From 62d96d19ee6b9331b8277f219a49a716a33e6240 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Thu, 23 Feb 2023 20:49:45 +0000 Subject: [PATCH 07/26] add x-server related env variables and binds to devcontainer --- .devcontainer/devcontainer.json | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index fe11ec1ec1..466d6214cd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -41,7 +41,12 @@ "MORPHEUS_ROOT": "${containerWorkspaceFolder}", "DEFAULT_CONDA_ENV": "morpheus", "MAMBA_NO_BANNER": "1", - "VAULT_HOST": "https://vault.ops.k8s.rapids.ai" + "VAULT_HOST": "https://vault.ops.k8s.rapids.ai", + "DISPLAY": "${localEnv:DISPLAY:-}", + "XAUTHORITY": "${localEnv:XAUTHORITY:-}", + "XDG_SESSION_TYPE": "${localEnv:XDG_SESSION_TYPE:-}", + "XDG_RUNTIME_DIR": "${localEnv:XDG_RUNTIME_DIR:-}", + "DBUS_SESSION_BUS_ADDRESS": "${localEnv:DBUS_SESSION_BUS_ADDRESS:-}" }, "initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ], @@ -53,6 +58,16 @@ "source": "/tmp/.X11-unix", "target": "/tmp/.X11-unix" }, + { + "type": "bind", + "source": "${localEnv:XDG_RUNTIME_DIR}", + "target": "${localEnv:XDG_RUNTIME_DIR}" + }, + { + "type": "bind", + "source": "/run/dbus/system_bus_socket", + "target": "/run/dbus/system_bus_socket" + }, { "type":"bind", "source": "/var/run/docker.sock", @@ -73,6 +88,7 @@ "source": "${localWorkspaceFolder}/.devcontainer/opt/morpheus", "target": "/opt/morpheus" }, + ], "features": { From b15e6bad8760024516b23f84e9288e7414212bd8 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Thu, 23 Feb 2023 20:50:31 +0000 Subject: [PATCH 08/26] move devcontainer vscode settings to new location --- .devcontainer/devcontainer.json | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 466d6214cd..6e6190e302 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -110,12 +110,10 @@ ], "settings": { "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake", - "cmake.languageSupport.dotnetPath": "/usr/bin/dotnet" + "cmake.languageSupport.dotnetPath": "/usr/bin/dotnet", + "C_Cpp.intelliSenseEngine": "disabled", + "python.terminal.activateEnvironment": false } } }, - "settings": { - "C_Cpp.intelliSenseEngine": "disabled", - "python.terminal.activateEnvironment": false - } } From 4af3619c3d53d063697524864e4d30fb2ff74ad0 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Tue, 7 Mar 2023 23:14:39 +0000 Subject: [PATCH 09/26] address review feedback --- .devcontainer/devcontainer.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6e6190e302..8afa154d78 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -41,6 +41,7 @@ "MORPHEUS_ROOT": "${containerWorkspaceFolder}", "DEFAULT_CONDA_ENV": "morpheus", "MAMBA_NO_BANNER": "1", + "CONDA_ALWAYS_YES": "true", "VAULT_HOST": "https://vault.ops.k8s.rapids.ai", "DISPLAY": "${localEnv:DISPLAY:-}", "XAUTHORITY": "${localEnv:XAUTHORITY:-}", @@ -74,10 +75,15 @@ "target": "/var/run/docker.sock" }, { - "type":"bind", - "source": "${localWorkspaceFolder}/../.conda", + "type": "bind", + "source": "${localWorkspaceFolder}/.cache/conda", "target": "/home/coder/.conda" }, + { + "type": "bind", + "source": "${localWorkspaceFolder}/../.conda/pkgs", + "target": "/home/coder/.conda/pkgs" + }, { "type":"bind", "source": "${localWorkspaceFolder}/../.config", // parent folder because sister repos are sibling dirs From 6aa526b71987f3e2dd3cfd92e5863ba87305995f Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 8 Mar 2023 15:22:38 +0000 Subject: [PATCH 10/26] make devcontainer conda mounts more resilient --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8afa154d78..86f88d3799 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -76,8 +76,8 @@ }, { "type": "bind", - "source": "${localWorkspaceFolder}/.cache/conda", - "target": "/home/coder/.conda" + "source": "${localWorkspaceFolder}/.cache/conda/envs", + "target": "/home/coder/.conda/envs" }, { "type": "bind", From 67f68364245c744133e75c7a0a8fc1cae8986a9e Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 8 Mar 2023 11:31:27 -0600 Subject: [PATCH 11/26] update to cuda 11.8 --- .devcontainer/opt/morpheus/bin/post-attach-command.sh | 4 ++-- .github/workflows/ci_pipe.yml | 2 +- ci/conda/recipes/morpheus/conda_build_config.yaml | 2 +- ci/conda/recipes/morpheus/meta.yaml | 2 +- ci/conda/recipes/run_conda_build.sh | 3 +-- ci/runner/Dockerfile | 4 ++-- docker/Dockerfile | 2 +- docker/build_container.sh | 2 +- .../environments/{cuda11.5_dev.yml => cuda11.8_dev.yml} | 7 +++---- .../{cuda11.5_runtime.yml => cuda11.8_runtime.yml} | 0 examples/digital_fingerprinting/production/conda_env.yml | 1 - examples/gnn_fraud_detection_pipeline/requirements.yml | 1 - models/mlflow/docker/Dockerfile | 2 +- 13 files changed, 14 insertions(+), 18 deletions(-) rename docker/conda/environments/{cuda11.5_dev.yml => cuda11.8_dev.yml} (93%) rename docker/conda/environments/{cuda11.5_runtime.yml => cuda11.8_runtime.yml} (100%) diff --git a/.devcontainer/opt/morpheus/bin/post-attach-command.sh b/.devcontainer/opt/morpheus/bin/post-attach-command.sh index dbe21ca4ba..8a813eb35f 100755 --- a/.devcontainer/opt/morpheus/bin/post-attach-command.sh +++ b/.devcontainer/opt/morpheus/bin/post-attach-command.sh @@ -27,6 +27,6 @@ ENV_NAME=${ENV_NAME:-morpheus} sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc; if conda_env_find "${ENV_NAME}" ; \ -then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml --prune; \ -else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml; \ +then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml --prune; \ +else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml; \ fi diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml index 81da3c25fc..947f42d5b1 100644 --- a/.github/workflows/ci_pipe.yml +++ b/.github/workflows/ci_pipe.yml @@ -34,7 +34,7 @@ on: env: CHANGE_TARGET: "${{ github.base_ref }}" CUDA_PATH: "/usr/local/cuda/" - CUDA_VER: "11.5" + CUDA_VER: "11.8" GH_TOKEN: "${{ github.token }}" GIT_COMMIT: "${{ github.sha }}" MORPHEUS_ROOT: "${{ github.workspace }}/morpheus" diff --git a/ci/conda/recipes/morpheus/conda_build_config.yaml b/ci/conda/recipes/morpheus/conda_build_config.yaml index 3095539a4f..c336e16247 100644 --- a/ci/conda/recipes/morpheus/conda_build_config.yaml +++ b/ci/conda/recipes/morpheus/conda_build_config.yaml @@ -23,7 +23,7 @@ cuda_compiler: - nvcc cuda_compiler_version: - - 11.5 + - 11.8 python: - 3.8 diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index 4c3679c08e..1183281070 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -16,7 +16,7 @@ {% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set py_version=environ.get('CONDA_PY', '3.8') %} -{% set cuda_version='.'.join(environ.get('CUDA', '11.5').split('.')[:2]) %} +{% set cuda_version='.'.join(environ.get('CUDA', '11.8').split('.')[:2]) %} {% set cuda_major=cuda_version.split('.')[0] %} {% set rapids_version = "22.10" %} diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh index 69714ff0c6..3c4fa33636 100755 --- a/ci/conda/recipes/run_conda_build.sh +++ b/ci/conda/recipes/run_conda_build.sh @@ -55,7 +55,7 @@ export USE_SCCACHE=${USE_SCCACHE:-""} export CUDA="$(conda list | grep cudatoolkit | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+")" export PYTHON_VER="$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")" -export CUDA=11.5 +export CUDA=11.8 echo "CUDA : ${CUDA}" echo "PYTHON_VER : ${PYTHON_VER}" echo "" @@ -100,7 +100,6 @@ CONDA_ARGS_ARRAY+=("--variants" "{python: 3.8}") # And default channels (with optional channel alias) CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}rapidsai") -CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/cuda-11.5.2") # Prefer labeled versions over the latest CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia") CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/dev") CONDA_ARGS_ARRAY+=("-c" "conda-forge") diff --git a/ci/runner/Dockerfile b/ci/runner/Dockerfile index 1d3c75c5dc..7f13db8075 100644 --- a/ci/runner/Dockerfile +++ b/ci/runner/Dockerfile @@ -18,8 +18,8 @@ # Args used in FROM commands must come first ARG FROM_IMAGE="rapidsai/ci" ARG CUDA_PKG_VER=11-5 -ARG CUDA_SHORT_VER=11.5 -ARG CUDA_VER=11.5.1 +ARG CUDA_SHORT_VER=11.8 +ARG CUDA_VER=11.8.0 ARG LINUX_DISTRO=ubuntu ARG LINUX_VER=20.04 ARG PROJ_NAME=morpheus diff --git a/docker/Dockerfile b/docker/Dockerfile index 7ee6282bc9..9dab39fb8f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,7 +18,7 @@ # Args used in FROM commands must come first ARG FROM_IMAGE="gpuci/miniforge-cuda" -ARG CUDA_VER=11.5 +ARG CUDA_VER=11.8 ARG LINUX_DISTRO=ubuntu ARG LINUX_VER=20.04 diff --git a/docker/build_container.sh b/docker/build_container.sh index fb0ef2aee1..6904b08bac 100755 --- a/docker/build_container.sh +++ b/docker/build_container.sh @@ -23,7 +23,7 @@ DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} # Build args FROM_IMAGE=${FROM_IMAGE:-"gpuci/miniforge-cuda"} -CUDA_VER=${CUDA_VER:-11.5} +CUDA_VER=${CUDA_VER:-11.8} LINUX_DISTRO=${LINUX_DISTRO:-ubuntu} LINUX_VER=${LINUX_VER:-20.04} RAPIDS_VER=${RAPIDS_VER:-22.10} diff --git a/docker/conda/environments/cuda11.5_dev.yml b/docker/conda/environments/cuda11.8_dev.yml similarity index 93% rename from docker/conda/environments/cuda11.5_dev.yml rename to docker/conda/environments/cuda11.8_dev.yml index 0984d08d51..49ba38302a 100644 --- a/docker/conda/environments/cuda11.5_dev.yml +++ b/docker/conda/environments/cuda11.8_dev.yml @@ -16,7 +16,6 @@ name: morpheus channels: - rapidsai - - nvidia/label/cuda-11.5.2 # Prefer packages with a specific cuda label over ones without - nvidia - nvidia/label/dev # For pre-releases of MRC. Should still default to full releases if available - conda-forge @@ -31,9 +30,9 @@ dependencies: - click >=8 - cmake=3.24 - configargparse=1.5 - - cuda-nvml-dev=11.5 + - cuda-nvml-dev=11.8 - cuda-python<=11.7.0 # Remove when Issue #251 is closed - - cudatoolkit=11.5 + - cudatoolkit=11.8 - cudf 22.10 - cupy=9.5.0 - cython=0.29.24 @@ -65,7 +64,7 @@ dependencies: - nodejs=17.4.0 - numba>=0.56.2 - numpydoc=1.4 - - nvcc_linux-64=11.5 + - nvcc_linux-64=11.8 - pandas=1.3 - pip - pkg-config # for mrc cmake diff --git a/docker/conda/environments/cuda11.5_runtime.yml b/docker/conda/environments/cuda11.8_runtime.yml similarity index 100% rename from docker/conda/environments/cuda11.5_runtime.yml rename to docker/conda/environments/cuda11.8_runtime.yml diff --git a/examples/digital_fingerprinting/production/conda_env.yml b/examples/digital_fingerprinting/production/conda_env.yml index 5d6dd9ee4d..e1f6130f71 100644 --- a/examples/digital_fingerprinting/production/conda_env.yml +++ b/examples/digital_fingerprinting/production/conda_env.yml @@ -18,7 +18,6 @@ channels: - rapidsai - nvidia - nvidia/label/dev # For pre-releases of MRC. Should still default to full releases if available - - nvidia/label/cuda-11.5.2 # For cuda-nvml-dev=11.5, which is not published under nvidia channel yet. - conda-forge dependencies: ####### Morpheus Dependencies (keep sorted!) ####### diff --git a/examples/gnn_fraud_detection_pipeline/requirements.yml b/examples/gnn_fraud_detection_pipeline/requirements.yml index 7ab0a01725..afcf04e4d5 100644 --- a/examples/gnn_fraud_detection_pipeline/requirements.yml +++ b/examples/gnn_fraud_detection_pipeline/requirements.yml @@ -17,7 +17,6 @@ channels: - rapidsai - nvidia - nvidia/label/dev # For pre-releases of MRC. Should still default to full releases if available - - nvidia/label/cuda-11.5.2 # For cuda-nvml-dev=11.5, which is not published under nvidia channel yet. - stellargraph - conda-forge dependencies: diff --git a/models/mlflow/docker/Dockerfile b/models/mlflow/docker/Dockerfile index 8a8b1f124a..dfe1c72f2a 100644 --- a/models/mlflow/docker/Dockerfile +++ b/models/mlflow/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. ARG FROM_IMAGE=gpuci/miniforge-cuda -ARG CUDA_VER=11.5 +ARG CUDA_VER=11.8 ARG LINUX_VER=ubuntu20.04 ARG PYTHON_VER=3.8 FROM ${FROM_IMAGE}:${CUDA_VER}-runtime-${LINUX_VER} AS base From 564c5939bda7e454b4df5f6f1a732d7d0042b069 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Tue, 14 Mar 2023 22:23:07 +0000 Subject: [PATCH 12/26] conda build fixes for devcontainer --- ci/conda/recipes/morpheus/meta.yaml | 20 +++++++++++++------- ci/conda/recipes/morpheus/morpheus_build.sh | 1 + 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index 1183281070..bed1b5fe40 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -69,7 +69,7 @@ outputs: - scikit-build >=0.12 - versioneer-518 run: - # Runtime only requirements. This + setup.y is the definitive runtime requirement list + # Runtime only requirements. This + setup.py is the definitive runtime requirement list - click >=8 - configargparse 1.5.* - cuda-python <=11.7.0 # Needed to ensure cudatoolkit and not cuda-toolkit is installed @@ -99,6 +99,11 @@ outputs: - pytest - pytest-cov - pytest-benchmark + # test that cuml can be installed in the env + # https://docs.rapids.ai/notices/rsn0026/ for libcusolver & libcusparse + - cuml {{ rapids_version }} + - libcusolver<=11.4.1.48 + - libcusparse<12 source_files: # - .git/* - docker/conda/environments/* @@ -109,12 +114,13 @@ outputs: - morpheus - morpheus._lib commands: - # Install dependencies that are pip-only - - mamba env update --prefix ${CONDA_PREFIX} --file docker/conda/environments/cuda{{ cuda_version }}_runtime.yml - # Ensure the test data is downloaded - - ./scripts/fetch_data.py fetch tests - # Run pytest - - pytest + - "echo pwd: ${PWD}" + # # Install dependencies that are pip-only + # - mamba env update --prefix ${CONDA_PREFIX} --file docker/conda/environments/cuda{{ cuda_version }}_runtime.yml + # # Ensure the test data is downloaded + # - ./scripts/fetch_data.py fetch tests + # # Run pytest + # - pytest about: home: https://github.com/nv-morpheus/Morpheus diff --git a/ci/conda/recipes/morpheus/morpheus_build.sh b/ci/conda/recipes/morpheus/morpheus_build.sh index e11022b69d..be548308a3 100644 --- a/ci/conda/recipes/morpheus/morpheus_build.sh +++ b/ci/conda/recipes/morpheus/morpheus_build.sh @@ -44,6 +44,7 @@ CMAKE_ARGS="-DMORPHEUS_BUILD_PYTHON_WHEEL=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"all"} ${CMAKE_ARGS}" CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" +CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" if [[ "${USE_SCCACHE}" == "1" ]]; then CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}" From c69762bc1756bc80b9b00ba0c257ae40cf746e48 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 15 Mar 2023 14:06:31 +0000 Subject: [PATCH 13/26] add comments --- ci/conda/recipes/morpheus/meta.yaml | 8 +------- ci/conda/recipes/morpheus/morpheus_build.sh | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index bed1b5fe40..43ad9419fa 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -114,13 +114,7 @@ outputs: - morpheus - morpheus._lib commands: - - "echo pwd: ${PWD}" - # # Install dependencies that are pip-only - # - mamba env update --prefix ${CONDA_PREFIX} --file docker/conda/environments/cuda{{ cuda_version }}_runtime.yml - # # Ensure the test data is downloaded - # - ./scripts/fetch_data.py fetch tests - # # Run pytest - # - pytest + - echo # make sure test requirements get installed about: home: https://github.com/nv-morpheus/Morpheus diff --git a/ci/conda/recipes/morpheus/morpheus_build.sh b/ci/conda/recipes/morpheus/morpheus_build.sh index be548308a3..fe9e504156 100644 --- a/ci/conda/recipes/morpheus/morpheus_build.sh +++ b/ci/conda/recipes/morpheus/morpheus_build.sh @@ -44,7 +44,7 @@ CMAKE_ARGS="-DMORPHEUS_BUILD_PYTHON_WHEEL=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"all"} ${CMAKE_ARGS}" CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" -CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" +CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11 if [[ "${USE_SCCACHE}" == "1" ]]; then CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}" From 9871d1281618ba5bd40516ee01f0fa2d44ac4515 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 15 Mar 2023 17:32:32 +0000 Subject: [PATCH 14/26] add cuda-11.8.0 label back --- .devcontainer/initialize-command.sh | 3 ++- ci/conda/recipes/run_conda_build.sh | 1 + docker/Dockerfile | 5 ++--- docker/conda/environments/cuda11.8_dev.yml | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.devcontainer/initialize-command.sh b/.devcontainer/initialize-command.sh index a9dd50e320..10110a3ee2 100755 --- a/.devcontainer/initialize-command.sh +++ b/.devcontainer/initialize-command.sh @@ -18,7 +18,8 @@ docker network inspect morpheus >/dev/null 2>&1 || docker network create morpheus # create the parent conda folder so it's found when mounting -mkdir -p ../.conda +mkdir -p .cache/conda/envs +mkdir -p ../.conda/pkgs # create a config directory if it does not exist so it's found when mounting mkdir -p ../.config diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh index 3c4fa33636..9b91592933 100755 --- a/ci/conda/recipes/run_conda_build.sh +++ b/ci/conda/recipes/run_conda_build.sh @@ -101,6 +101,7 @@ CONDA_ARGS_ARRAY+=("--variants" "{python: 3.8}") # And default channels (with optional channel alias) CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}rapidsai") CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia") +CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/cuda-11.8.0") CONDA_ARGS_ARRAY+=("-c" "${CONDA_CHANNEL_ALIAS:+"${CONDA_CHANNEL_ALIAS%/}/"}nvidia/label/dev") CONDA_ARGS_ARRAY+=("-c" "conda-forge") diff --git a/docker/Dockerfile b/docker/Dockerfile index 6ced8395cb..35769f5680 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -161,7 +161,7 @@ RUN --mount=type=cache,id=workspace_cache,target=/workspace/.cache,sharing=locke source activate base &&\ # Need to get around recent versions of git locking paths until they are deemed safe git config --global --add safe.directory "*" &&\ - MORPHEUS_BUILD_PYTHON_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh morpheus + MORPHEUS_BUILD_PYTHON_STUBS=OFF CONDA_BLD_PATH=/opt/conda/conda-bld ./ci/conda/recipes/run_conda_build.sh morpheus # sid_visualization is a submodule we need to init RUN git submodule update --init --recursive @@ -178,7 +178,7 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ source activate morpheus &&\ # Install morpheus - CONDA_ALWAYS_YES=true /opt/conda/bin/mamba install -n morpheus -c local -c rapidsai -c nvidia -c nvidia/label/dev -c conda-forge morpheus &&\ + CONDA_ALWAYS_YES=true /opt/conda/bin/mamba install -n morpheus -c local -c rapidsai -c nvidia -c nvidia/label/cuda-11.8.0 -c nvidia/label/dev -c conda-forge morpheus &&\ # Install runtime dependencies that are pip-only /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml &&\ # Clean and activate @@ -245,7 +245,6 @@ ARG MORPHEUS_USER="root" RUN source activate morpheus \ && MORPHEUS_ROOT=/workspace \ CONDA_BLD_PATH=/opt/conda/conda-bld \ - CONDA_ARGS="--no-test" \ ./ci/conda/recipes/run_conda_build.sh pydebug \ && ./ci/conda/recipes/python_dbg_install.sh \ -s $( ls /opt/conda/conda-bld/src_cache/Python-${PYTHON_VER}*.tar.xz ) \ diff --git a/docker/conda/environments/cuda11.8_dev.yml b/docker/conda/environments/cuda11.8_dev.yml index 8042fa06ef..d806c8ee4d 100644 --- a/docker/conda/environments/cuda11.8_dev.yml +++ b/docker/conda/environments/cuda11.8_dev.yml @@ -17,6 +17,7 @@ name: morpheus channels: - rapidsai - nvidia + - nvidia/label/cuda-11.8.0 - nvidia/label/dev # For pre-releases of MRC. Should still default to full releases if available - conda-forge dependencies: From 1146fb03bd9d6421ada2fa2b40425910147afb8c Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 15 Mar 2023 19:33:24 +0000 Subject: [PATCH 15/26] install cuda-compiler for dev and release container builds --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 35769f5680..9b2902d4cb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -146,6 +146,7 @@ RUN apt update && \ cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvml-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ @@ -213,6 +214,7 @@ RUN apt update && \ cuda-cudart-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ From ca92c7fb81e9b2ccd2c4387a3ac3bcdd98961fc8 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Wed, 22 Mar 2023 11:37:31 -0600 Subject: [PATCH 16/26] Removing `morpheus._lib` from the conda build tests since that requires the driver --- ci/conda/recipes/morpheus/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index 1466b1b1fd..bb866aa7b2 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -105,14 +105,14 @@ outputs: - libcusolver<=11.4.1.48 - libcusparse<12 source_files: - # - .git/* - docker/conda/environments/* - pyproject.toml - scripts/fetch_data.py - tests/* imports: - morpheus - - morpheus._lib + - cudf + - cuml commands: - echo # make sure test requirements get installed From 470f3ac438bfe7fab6536b23e3e5d0085a382b30 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Wed, 22 Mar 2023 13:53:11 -0600 Subject: [PATCH 17/26] Setting RAPIDS_NO_INITIALIZE=1 to allow import tests to work --- ci/conda/recipes/morpheus/meta.yaml | 7 +------ ci/conda/recipes/morpheus/run_test.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 ci/conda/recipes/morpheus/run_test.py diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index bb866aa7b2..c63250f662 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -109,12 +109,7 @@ outputs: - pyproject.toml - scripts/fetch_data.py - tests/* - imports: - - morpheus - - cudf - - cuml - commands: - - echo # make sure test requirements get installed + script: run_test.py about: home: https://github.com/nv-morpheus/Morpheus diff --git a/ci/conda/recipes/morpheus/run_test.py b/ci/conda/recipes/morpheus/run_test.py new file mode 100644 index 0000000000..931eab2a9c --- /dev/null +++ b/ci/conda/recipes/morpheus/run_test.py @@ -0,0 +1,11 @@ +import os + +# Set RAPIDS_NO_INITIALIZE to prevent cuda from being initialized +# See issue https://github.com/rapidsai/cudf/issues/4827 +os.environ["RAPIDS_NO_INITIALIZE"] = "1" + +# Now run the imports +import cudf +import cuml + +import morpheus From e0a3463ce8e4e0ba93717139390e94ca784d2efb Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Wed, 22 Mar 2023 16:31:01 -0600 Subject: [PATCH 18/26] Removing GPU libraries from import --- ci/conda/recipes/morpheus/meta.yaml | 6 +++++- ci/conda/recipes/morpheus/run_test.py | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index c63250f662..437199670d 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -109,7 +109,11 @@ outputs: - pyproject.toml - scripts/fetch_data.py - tests/* - script: run_test.py + # script: run_test.py + imports: + - morpheus + commands: + - echo # make sure test requirements get installed about: home: https://github.com/nv-morpheus/Morpheus diff --git a/ci/conda/recipes/morpheus/run_test.py b/ci/conda/recipes/morpheus/run_test.py index 931eab2a9c..2b98d5d602 100644 --- a/ci/conda/recipes/morpheus/run_test.py +++ b/ci/conda/recipes/morpheus/run_test.py @@ -4,8 +4,9 @@ # See issue https://github.com/rapidsai/cudf/issues/4827 os.environ["RAPIDS_NO_INITIALIZE"] = "1" -# Now run the imports -import cudf -import cuml +# # Now run the imports +# import cudf +# import cuml +# import cupy import morpheus From 5a656ceafb5295ab7750b1decf6865d07aebfe38 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Wed, 22 Mar 2023 16:43:59 -0600 Subject: [PATCH 19/26] Test removing cusolver and cusparse --- ci/conda/recipes/morpheus/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index 437199670d..46a014e062 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -102,8 +102,8 @@ outputs: # test that cuml can be installed in the env # https://docs.rapids.ai/notices/rsn0026/ for libcusolver & libcusparse - cuml {{ rapids_version }} - - libcusolver<=11.4.1.48 - - libcusparse<12 + # - libcusolver<=11.4.1.48 + # - libcusparse<12 source_files: - docker/conda/environments/* - pyproject.toml From c1a30a23e7ee6eda27dbdec77c8c49f10f3663da Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Wed, 22 Mar 2023 16:58:19 -0600 Subject: [PATCH 20/26] Switching to use run_constrained --- ci/conda/recipes/morpheus/meta.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index 46a014e062..871cfea01d 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -91,7 +91,11 @@ outputs: - tqdm 4.* - typing_utils 0.1.* - watchdog 2.1.* - + run_constrained: + # If cuml is installed, we need to constrain these two libraries. See: + # https://docs.rapids.ai/notices/rsn0026/ for libcusolver & libcusparse + - libcusolver<=11.4.1.48 + - libcusparse<12 test: requires: - cudatoolkit {{ cuda_version }}.* @@ -100,10 +104,7 @@ outputs: - pytest-cov - pytest-benchmark # test that cuml can be installed in the env - # https://docs.rapids.ai/notices/rsn0026/ for libcusolver & libcusparse - cuml {{ rapids_version }} - # - libcusolver<=11.4.1.48 - # - libcusparse<12 source_files: - docker/conda/environments/* - pyproject.toml From 9add0cd3dd095da135e8045806e714f7086cb2ea Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Thu, 23 Mar 2023 10:49:33 -0600 Subject: [PATCH 21/26] Minor tweaks to dockerfile and add dependencies to export set --- cmake/dependencies.cmake | 39 ++++++++++++++++++++++++--------------- docker/Dockerfile | 3 +-- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index c41c48c0e6..e96058a20a 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -21,21 +21,30 @@ if (VERBOSE) morpheus_utils_print_config() endif() -# Load direct physical package dependencies first, so we fail early. -find_package(Protobuf REQUIRED) -find_package(CUDAToolkit REQUIRED) # Required by Morpheus. Fail early if we don't have it. +# Load direct physical package dependencies first, so we fail early. Add all dependencies to our export set +rapids_find_package(Protobuf + REQUIRED + BUILD_EXPORT_SET ${PROJECT_NAME}-core-exports + INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports +) + +rapids_find_package(CUDAToolkit + REQUIRED + BUILD_EXPORT_SET ${PROJECT_NAME}-core-exports + INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports +) if(MORPHEUS_BUILD_BENCHMARKS) # google benchmark # - Expects package to pre-exist in the build environment # ================ rapids_find_package(benchmark REQUIRED - GLOBAL_TARGETS benchmark::benchmark - BUILD_EXPORT_SET ${PROJECT_NAME}-exports - INSTALL_EXPORT_SET ${PROJECT_NAME}-exports - FIND_ARGS - CONFIG - ) + GLOBAL_TARGETS benchmark::benchmark + BUILD_EXPORT_SET ${PROJECT_NAME}-exports + INSTALL_EXPORT_SET ${PROJECT_NAME}-exports + FIND_ARGS + CONFIG + ) endif() if(MORPHEUS_BUILD_TESTS) @@ -43,12 +52,12 @@ if(MORPHEUS_BUILD_TESTS) # - Expects package to pre-exist in the build environment # =========== rapids_find_package(GTest REQUIRED - GLOBAL_TARGETS GTest::gtest GTest::gmock GTest::gtest_main GTest::gmock_main - BUILD_EXPORT_SET ${PROJECT_NAME}-exports - INSTALL_EXPORT_SET ${PROJECT_NAME}-exports - FIND_ARGS - CONFIG - ) + GLOBAL_TARGETS GTest::gtest GTest::gmock GTest::gtest_main GTest::gmock_main + BUILD_EXPORT_SET ${PROJECT_NAME}-exports + INSTALL_EXPORT_SET ${PROJECT_NAME}-exports + FIND_ARGS + CONFIG + ) endif() # libcudacxx -- get an explicit lubcudacxx build, matx tries to pull a tag that doesn't exist. diff --git a/docker/Dockerfile b/docker/Dockerfile index 17b919dd91..cb11c4d889 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -79,9 +79,8 @@ WORKDIR ${MORPHEUS_ROOT} # Install mamba to speed the solve up RUN conda config --set ssl_verify false &&\ conda config --add pkgs_dirs /opt/conda/pkgs &&\ - conda config --env --add channels conda-forge &&\ # Install mamba, boa and git here. Conda build breaks with other git installs - /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.10" "git >=2.35.3" "git-lfs" "python=${PYTHON_VER}" "tini=0.19" &&\ + /opt/conda/bin/mamba install -y -n base -c conda-forge "boa" "git >=2.35.3" "git-lfs" "python=${PYTHON_VER}" "tini=0.19" &&\ source activate base &&\ git lfs install # conda clean -afy From 83999926d127cf3277be29c05d2cbbb57cd902e0 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 27 Mar 2023 10:47:09 -0500 Subject: [PATCH 22/26] reorder apt installs --- docker/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0c53da7d32..e18c63a07d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -147,11 +147,10 @@ RUN apt update && \ cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvml-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \ libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \ apt clean && \ rm -rf /var/lib/apt/lists/* @@ -212,14 +211,14 @@ ARG CUDA_MINOR_VER RUN apt update && \ DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \ apt install --no-install-recommends -y \ + cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-cudart-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - cuda-compiler-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ nodejs \ npm && \ apt clean && \ From f3c9e3fb96d70238f8643a4c55cbcdd0c87779fc Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 27 Mar 2023 15:58:59 +0000 Subject: [PATCH 23/26] add copyright header --- ci/conda/recipes/morpheus/run_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ci/conda/recipes/morpheus/run_test.py b/ci/conda/recipes/morpheus/run_test.py index 2b98d5d602..f38e3413bf 100644 --- a/ci/conda/recipes/morpheus/run_test.py +++ b/ci/conda/recipes/morpheus/run_test.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os # Set RAPIDS_NO_INITIALIZE to prevent cuda from being initialized From 6c5a65a650df6228bfda410d6461213915c356ed Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 27 Mar 2023 16:03:55 +0000 Subject: [PATCH 24/26] remove run_test.py and fix typo --- ci/conda/recipes/morpheus/meta.yaml | 1 - ci/conda/recipes/morpheus/run_test.py | 27 --------------------------- cmake/dependencies.cmake | 6 +----- docker/Dockerfile | 2 +- 4 files changed, 2 insertions(+), 34 deletions(-) delete mode 100644 ci/conda/recipes/morpheus/run_test.py diff --git a/ci/conda/recipes/morpheus/meta.yaml b/ci/conda/recipes/morpheus/meta.yaml index 871cfea01d..1669029d57 100644 --- a/ci/conda/recipes/morpheus/meta.yaml +++ b/ci/conda/recipes/morpheus/meta.yaml @@ -110,7 +110,6 @@ outputs: - pyproject.toml - scripts/fetch_data.py - tests/* - # script: run_test.py imports: - morpheus commands: diff --git a/ci/conda/recipes/morpheus/run_test.py b/ci/conda/recipes/morpheus/run_test.py deleted file mode 100644 index f38e3413bf..0000000000 --- a/ci/conda/recipes/morpheus/run_test.py +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -# Set RAPIDS_NO_INITIALIZE to prevent cuda from being initialized -# See issue https://github.com/rapidsai/cudf/issues/4827 -os.environ["RAPIDS_NO_INITIALIZE"] = "1" - -# # Now run the imports -# import cudf -# import cuml -# import cupy - -import morpheus diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index e96058a20a..a72ecc128f 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -28,11 +28,7 @@ rapids_find_package(Protobuf INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports ) -rapids_find_package(CUDAToolkit - REQUIRED - BUILD_EXPORT_SET ${PROJECT_NAME}-core-exports - INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports -) +find_package(CUDAToolkit) if(MORPHEUS_BUILD_BENCHMARKS) # google benchmark diff --git a/docker/Dockerfile b/docker/Dockerfile index e18c63a07d..d8dd9f6881 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -147,7 +147,7 @@ RUN apt update && \ cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvml-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \ + libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ From daa9a13eebf676f7edd853a5bd294a57b01b50b6 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Mon, 27 Mar 2023 16:05:28 +0000 Subject: [PATCH 25/26] make CUDAToolkit find_package REQUIRED --- cmake/dependencies.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index a72ecc128f..513802ffbb 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -28,7 +28,7 @@ rapids_find_package(Protobuf INSTALL_EXPORT_SET ${PROJECT_NAME}-core-exports ) -find_package(CUDAToolkit) +find_package(CUDAToolkit REQUIRED) if(MORPHEUS_BUILD_BENCHMARKS) # google benchmark From 77db009fa951cac04f291a46bfeb1d1b37ec53df Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 27 Mar 2023 09:08:09 -0700 Subject: [PATCH 26/26] Fix placement of the end of the apt command --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e18c63a07d..645cd704a2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -147,10 +147,10 @@ RUN apt update && \ cuda-cupti-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvml-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ cuda-nvtx-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \ + libcublas-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcufft-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ libcurand-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ - libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} \ + libcusolver-dev-${CUDA_MAJOR_VER}-${CUDA_MINOR_VER} && \ apt clean && \ rm -rf /var/lib/apt/lists/*