diff --git a/.gitmodules b/.gitmodules index 0c48d0dc37..2951889362 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,8 @@ [submodule "external/morpheus-visualizations"] path = external/morpheus-visualizations url = https://github.com/nv-morpheus/morpheus-visualizations.git - branch = branch-23.01 + branch = branch-23.07 [submodule "morpheus_utils"] path = external/utilities url = https://github.com/nv-morpheus/utilities.git - branch = branch-23.01 + branch = branch-23.07 diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh new file mode 100755 index 0000000000..295058d426 --- /dev/null +++ b/ci/release/update-version.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Usage +# Either supply full versions: +# `bash update-version.sh ` +# Format is YY.MM.PP - no leading 'v' or trailing 'a' +# Or no versions: +# `bash update-version.sh` + +set -e + +# If the user has not supplied the versions, determine them from the git tags +if [[ "$#" -ne 2 ]]; then + echo "No versions were provided. Using last 2 git tags to determined current and next version" + + # Current version comes from the previous alpha tag + CURRENT_FULL_VERSION=$(git tag --merged HEAD --list 'v*' | sort --version-sort | tail -n 2 | head -n 1 | tr -d 'va') + # Next version comes from the latest alpha tag + NEXT_FULL_VERSION=$(git tag --merged HEAD --list 'v*' | sort --version-sort | tail -n 1 | tr -d 'va') +else + # User has supplied current and next arguments + CURRENT_FULL_VERSION=$1 + NEXT_FULL_VERSION=$2 +fi + +CURRENT_MAJOR=$(echo ${CURRENT_FULL_VERSION} | awk '{split($0, a, "."); print a[1]}') +CURRENT_MINOR=$(echo ${CURRENT_FULL_VERSION} | awk '{split($0, a, "."); print a[2]}') +CURRENT_PATCH=$(echo ${CURRENT_FULL_VERSION} | awk '{split($0, a, "."); print a[3]}') +CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} + +NEXT_MAJOR=$(echo ${NEXT_FULL_VERSION} | awk '{split($0, a, "."); print a[1]}') +NEXT_MINOR=$(echo ${NEXT_FULL_VERSION} | awk '{split($0, a, "."); print a[2]}') +NEXT_PATCH=$(echo ${NEXT_FULL_VERSION} | awk '{split($0, a, "."); print a[3]}') +NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} + +# Need to distutils-normalize the versions for some use cases +CURRENT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${CURRENT_SHORT_TAG}'))") +NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") + +echo "Preparing release $CURRENT_FULL_VERSION (PEP ${CURRENT_SHORT_TAG_PEP440}) => $NEXT_FULL_VERSION (PEP ${NEXT_SHORT_TAG_PEP440})" + +# Inplace sed replace; workaround for Linux and Mac +function sed_runner() { + sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak +} + +# .gitmodules +git submodule set-branch -b branch-${NEXT_SHORT_TAG} external/morpheus-visualizations +git submodule set-branch -b branch-${NEXT_SHORT_TAG} morpheus_utils + +# Root CMakeLists.txt +sed_runner 's/'"VERSION ${CURRENT_FULL_VERSION}.*"'/'"VERSION ${NEXT_FULL_VERSION}"'/g' CMakeLists.txt + +# Conda environment file +sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" docker/conda/environments/cuda11.8_dev.yml + +# examples/digital_fingerprinting +sed_runner "s/v${CURRENT_FULL_VERSION}-runtime/v${NEXT_FULL_VERSION}-runtime/g" examples/digital_fingerprinting/production/docker-compose.yml +sed_runner "s/v${CURRENT_FULL_VERSION}-runtime/v${NEXT_FULL_VERSION}-runtime/g" examples/digital_fingerprinting/production/Dockerfile +sed_runner "s|blob/branch-${CURRENT_SHORT_TAG}|blob/branch-${NEXT_SHORT_TAG}|g" examples/digital_fingerprinting/starter/README.md + +# docs/source/cloud_deployment_guide.md +sed_runner "s|${CURRENT_SHORT_TAG}.tgz|${NEXT_SHORT_TAG}.tgz|g" docs/source/cloud_deployment_guide.md +sed_runner "s|blob/branch-${CURRENT_SHORT_TAG}|blob/branch-${NEXT_SHORT_TAG}|g" docs/source/cloud_deployment_guide.md +sed_runner "s|tree/branch-${CURRENT_SHORT_TAG}|tree/branch-${NEXT_SHORT_TAG}|g" docs/source/cloud_deployment_guide.md + +# docs/source/getting_started.md +# Only do the minor version here since the full version can mess up the examples +sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" docs/source/getting_started.md diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index 81ef529b26..c87fa0b449 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -39,5 +39,8 @@ cmake --build build --target morpheus_style_checks --parallel ${PARALLEL_LEVEL} rapids-logger "sccache usage for source build:" sccache --show-stats +rapids-logger "Checking versions" +${MORPHEUS_ROOT}/ci/scripts/version_checks.sh + rapids-logger "Runing C++ style checks" ${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh diff --git a/ci/scripts/version_checks.sh b/ci/scripts/version_checks.sh new file mode 100755 index 0000000000..0c76aed1be --- /dev/null +++ b/ci/scripts/version_checks.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# set -x + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +source ${SCRIPT_DIR}/common.sh + +# Run the update version script +UPDATE_VERSION_OUTPUT=`${MORPHEUS_ROOT}/ci/release/update-version.sh` + +# If any diffs were found, the versions were out of date +VERSIONS_OUT_OF_DATE=$(git diff --name-only) + +if [[ "${VERSIONS_OUT_OF_DATE}" != "" ]]; then + echo -e "\n\n>>>> FAILED: version check; begin output\n\n" + echo -e "${UPDATE_VERSION_OUTPUT}" + echo -e "\n\nThe following files have out of date versions:" + echo -e "${VERSIONS_OUT_OF_DATE}" + echo -e "\n\n>>>> FAILED: version check; end output\n\n" \ + "To update the versions, run the following and commit the results:\n" \ + " ./ci/release/update-version.sh\n\n" + exit 1 +fi diff --git a/docs/source/cloud_deployment_guide.md b/docs/source/cloud_deployment_guide.md index 481c303b74..08539be36e 100644 --- a/docs/source/cloud_deployment_guide.md +++ b/docs/source/cloud_deployment_guide.md @@ -105,7 +105,7 @@ The Helm chart (`morpheus-ai-engine`) that offers the auxiliary components requi Follow the below steps to install Morpheus AI Engine: ```bash -helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-ai-engine-23.03.tgz --username='$oauthtoken' --password=$API_KEY --untar +helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-ai-engine-23.07.tgz --username='$oauthtoken' --password=$API_KEY --untar ``` ```bash helm install --set ngc.apiKey="$API_KEY" \ @@ -147,7 +147,7 @@ replicaset.apps/zookeeper-87f9f4dd 1 1 1 54s Run the following command to pull the Morpheus SDK Client (referred to as Helm chart `morpheus-sdk-client`) on to your instance: ```bash -helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-sdk-client-23.03.tgz --username='$oauthtoken' --password=$API_KEY --untar +helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-sdk-client-23.07.tgz --username='$oauthtoken' --password=$API_KEY --untar ``` #### Morpheus SDK Client in Sleep Mode @@ -185,7 +185,7 @@ kubectl -n $NAMESPACE exec sdk-cli-helper -- cp -RL /workspace/models /common The Morpheus MLflow Helm chart offers MLFlow server with Triton plugin to deploy, update, and remove models from the Morpheus AI Engine. The MLflow server UI can be accessed using NodePort `30500`. Follow the below steps to install the Morpheus MLflow: ```bash -helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-mlflow-23.03.tgz --username='$oauthtoken' --password=$API_KEY --untar +helm fetch https://helm.ngc.nvidia.com/nvidia/morpheus/charts/morpheus-mlflow-23.07.tgz --username='$oauthtoken' --password=$API_KEY --untar ``` ```bash helm install --set ngc.apiKey="$API_KEY" \ @@ -403,7 +403,7 @@ To publish messages to a Kafka topic, we need to copy datasets to locations wher kubectl -n $NAMESPACE exec sdk-cli-helper -- cp -R /workspace/examples/data /common ``` -Refer to the [Morpheus CLI Overview](https://github.com/nv-morpheus/Morpheus/blob/branch-23.03/docs/source/basics/overview.rst) and [Building a Pipeline](https://github.com/nv-morpheus/Morpheus/blob/branch-23.03/docs/source/basics/building_a_pipeline.rst) documentation for more information regarding the commands. +Refer to the [Morpheus CLI Overview](https://github.com/nv-morpheus/Morpheus/blob/branch-23.07/docs/source/basics/overview.rst) and [Building a Pipeline](https://github.com/nv-morpheus/Morpheus/blob/branch-23.07/docs/source/basics/building_a_pipeline.rst) documentation for more information regarding the commands. > **Note**: Before running the example pipelines, ensure that the criteria below are met: - Ensure that models specific to the pipeline are deployed. @@ -782,8 +782,8 @@ kubectl -n $NAMESPACE exec deploy/broker -c broker -- kafka-topics.sh \ ## Additional Documentation For more information on how to use the Morpheus Python API to customize and run your own optimized AI pipelines, Refer to below documentation. -- [Morpheus Developer Guide](https://github.com/nv-morpheus/Morpheus/tree/branch-23.03/docs/source/developer_guide) -- [Morpheus Pipeline Examples](https://github.com/nv-morpheus/Morpheus/tree/branch-23.03/examples) +- [Morpheus Developer Guide](https://github.com/nv-morpheus/Morpheus/tree/branch-23.07/docs/source/developer_guide) +- [Morpheus Pipeline Examples](https://github.com/nv-morpheus/Morpheus/tree/branch-23.07/examples) ## Troubleshooting diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md index 171aeda60a..c1545b1bad 100644 --- a/docs/source/getting_started.md +++ b/docs/source/getting_started.md @@ -41,14 +41,14 @@ More advanced users, or those who are interested in using the latest pre-release ### Pull the Morpheus Image 1. Go to [https://catalog.ngc.nvidia.com/orgs/nvidia/teams/morpheus/containers/morpheus/tags](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/morpheus/containers/morpheus/tags) 1. Choose a version -1. Download the selected version, for example for `23.03`: +1. Download the selected version, for example for `23.07`: ```bash -docker pull nvcr.io/nvidia/morpheus/morpheus:23.03-runtime +docker pull nvcr.io/nvidia/morpheus/morpheus:23.07-runtime ``` > **Note about Morpheus versions:** > -> Morpheus uses Calendar Versioning ([CalVer](https://calver.org/)). For each Morpheus release there will be an image tagged in the form of `YY.MM-runtime` this tag will always refer to the latest point release for that version. In addition to this there will also be at least one point release version tagged in the form of `vYY.MM.00-runtime` this will be the initial point release for that version (ex. `v23.03.00-runtime`). In the event of a major bug, we may release additional point releases (ex. `v23.03.01-runtime`, `v23.03.02-runtime` etc...), and the `YY.MM-runtime` tag will be updated to reference that point release. +> Morpheus uses Calendar Versioning ([CalVer](https://calver.org/)). For each Morpheus release there will be an image tagged in the form of `YY.MM-runtime` this tag will always refer to the latest point release for that version. In addition to this there will also be at least one point release version tagged in the form of `vYY.MM.00-runtime` this will be the initial point release for that version (ex. `v23.07.00-runtime`). In the event of a major bug, we may release additional point releases (ex. `v23.07.01-runtime`, `v23.07.02-runtime` etc...), and the `YY.MM-runtime` tag will be updated to reference that point release. > > Users who want to ensure they are running with the latest bug fixes should use a release image tag (`YY.MM-runtime`). Users who need to deploy a specific version into production should use a point release image tag (`vYY.MM.00-runtime`). @@ -56,7 +56,7 @@ docker pull nvcr.io/nvidia/morpheus/morpheus:23.03-runtime 1. Ensure that [The NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker) is installed. 1. Start the container downloaded from the previous section: ```bash -docker run --rm -ti --runtime=nvidia --gpus=all --net=host -v /var/run/docker.sock:/var/run/docker.sock nvcr.io/nvidia/morpheus/morpheus:23.03-runtime bash +docker run --rm -ti --runtime=nvidia --gpus=all --net=host -v /var/run/docker.sock:/var/run/docker.sock nvcr.io/nvidia/morpheus/morpheus:23.07-runtime bash ``` Note about some of the flags above: @@ -133,10 +133,10 @@ To run the built "release" container, use the following: ./docker/run_container_release.sh ``` -The `./docker/run_container_release.sh` script accepts the same `DOCKER_IMAGE_NAME`, and `DOCKER_IMAGE_TAG` environment variables that the `./docker/build_container_release.sh` script does. For example, to run version `v23.03.00` use the following: +The `./docker/run_container_release.sh` script accepts the same `DOCKER_IMAGE_NAME`, and `DOCKER_IMAGE_TAG` environment variables that the `./docker/build_container_release.sh` script does. For example, to run version `v23.07.00` use the following: ```bash -DOCKER_IMAGE_TAG="v23.03.00-runtime" ./docker/run_container_release.sh +DOCKER_IMAGE_TAG="v23.07.00-runtime" ./docker/run_container_release.sh ``` ## Launching Triton Server diff --git a/external/morpheus-visualizations b/external/morpheus-visualizations index 59018edb16..32e387ac41 160000 --- a/external/morpheus-visualizations +++ b/external/morpheus-visualizations @@ -1 +1 @@ -Subproject commit 59018edb16482193f966ea743ffb18f7c205bccc +Subproject commit 32e387ac419a1308f916aa1450cf1374e0a75a82 diff --git a/external/utilities b/external/utilities index 67a1cca120..b5a53cf77c 160000 --- a/external/utilities +++ b/external/utilities @@ -1 +1 @@ -Subproject commit 67a1cca120498a3eb3765ad8cc6e1d89656176a3 +Subproject commit b5a53cf77c9c9e9406939f1e6bbc163ba16b0f0f