Add DFP Production Example Workflow

ci/conda/recipes/morpheus/meta.yaml

@@ -78,7 +78,6 @@ outputs:
         - mlflow >=1.23
         - networkx
         - pandas 1.3
-        - pluggy
         - python
         - tqdm
         - typing_utils

examples/digital_fingerprinting/production/.env.sample

@@ -0,0 +1,11 @@
+# NOTE: This file should be copied to `.env` in the same folder and updated for each user
+MYSQL_DATABASE="db"
+MYSQL_USER="mlflow"
+MYSQL_PASSWORD="good_password"
+MYSQL_ROOT_PASSWORD="even_better_password"
+MYSQL_ROOT_HOST="172.*.*.*"
+MYSQL_LOG_CONSOLE=1
+
+# Update these with your own credentials UID=$(id -u) GID=$(id -g)
+UID=1000
+GID=1000

examples/digital_fingerprinting/production/.gitignore

@@ -0,0 +1,2 @@
+*.s3_cache
+artifacts/

examples/digital_fingerprinting/production/README.md

@@ -0,0 +1,17 @@
+# "Production" Digital Fingerprinting Pipeline
+
+### Build the Morpheus container
+
+This is necessary to get the latest changes needed for DFP
+
+```bash
+./docker/build_container_release.sh
+```
+
+### Running locally via `docker-compose`
+
+```bash
+docker-compose build
+
+docker-compose up
+```

examples/digital_fingerprinting/production/docker-compose.yml

@@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+version: '3.3'
+
+services:
+  db:
+    restart: always
+    image: mysql/mysql-server
+    container_name: mlflow_db
+    expose:
+      - "3306"
+    networks:
+      - backend
+    environment:
+      - MYSQL_DATABASE=${MYSQL_DATABASE}
+      - MYSQL_USER=${MYSQL_USER}
+      - MYSQL_PASSWORD=${MYSQL_PASSWORD}
+      - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}
+      - MYSQL_ROOT_HOST=${MYSQL_ROOT_HOST}
+    volumes:
+      - db_data:/var/lib/mysql
+
+  mlflow:
+    restart: always
+    build: ./mlflow
+    image: mlflow_server
+    container_name: mlflow_server
+    ports:
+      - "5000:5000"
+    networks:
+      - frontend
+      - backend
+    # command: mlflow server --backend-store-uri mysql+pymysql://${MYSQL_USER}:${MYSQL_PASSWORD}@db:3306/${MYSQL_DATABASE} --serve-artifacts --artifacts-destination /opt/mlflow/artifacts --host 0.0.0.0
+    command: mlflow server --gunicorn-opts "--log-level debug" --backend-store-uri sqlite:////opt/mlflow/dbdata/mlflow.db --serve-artifacts --artifacts-destination /opt/mlflow/artifacts --host 0.0.0.0
+    volumes:
+      - db_data:/opt/mlflow/dbdata
+      - mlflow_data:/opt/mlflow/artifacts
+    # depends_on:
+    #   - db
+
+  jupyter:
+    restart: always
+    build:
+      context: ./jupyter
+      args:
+        - MORPHEUS_CONTAINER_VERSION=v22.08.00a-runtime
+    image: dfp_morpheus_jupyter
+    container_name: jupyter
+    ports:
+      - "8888:8888"
+    networks:
+      - frontend
+      - backend
+    environment:
+      - VAULT_ROLE_ID=${VAULT_ROLE_ID}
+      - VAULT_SECRET_ID=${VAULT_SECRET_ID}
+    command: jupyter-lab --no-browser --allow-root --ip='*'
+    volumes:
+      - ../..:/work
+    working_dir: /work/examples/dfp_workflow
+    depends_on:
+      - mlflow
+    profiles:
+      - dev
+
+  morpheus_training:
+    # restart: always
+    build:
+      context: ./morpheus
+      args:
+        - MORPHEUS_CONTAINER_VERSION=v22.08.00a-runtime
+    image: dfp_morpheus
+    container_name: morpheus_training
+    networks:
+      - frontend
+      - backend
+    environment:
+      # Colorize the terminal in the container if possible
+      TERM: "${TERM:-}"
+      # PS1: "$$(whoami):$$(pwd) $$ "
+      VAULT_ROLE_ID: "${VAULT_ROLE_ID}"
+      VAULT_SECRET_ID: "${VAULT_SECRET_ID}"
+      DFP_CACHE_DIR: "/work/.cache/dfp"
+      DFP_TRACKING_URI: "http://mlflow:5000"
+    command: ./launch.sh --train_users=generic --duration=1d
+    volumes:
+      # - ./.s3_cache:/work/.s3_cache
+      - ../..:/work
+      # - /etc/passwd:/etc/passwd:ro
+      # - /etc/group:/etc/group:ro
+    working_dir: /work/examples/dfp_workflow/morpheus
+    depends_on:
+      - mlflow
+    profiles:
+      - training
+    cap_add:
+      - sys_nice
+    user: "${UID}:${GID}"
+
+  # nginx:
+  #   restart: always
+  #   build: ./nginx
+  #   image: mlflow_nginx
+  #   container_name: mlflow_nginx
+  #   ports:
+  #     - "80:80"
+  #   networks:
+  #     - frontend
+  #   depends_on:
+  #     - web
+
+networks:
+  frontend:
+    driver: bridge
+  backend:
+    driver: bridge
+
+volumes:
+  db_data:
+  mlflow_data:

examples/digital_fingerprinting/production/jupyter/Dockerfile

@@ -0,0 +1,74 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG MORPHEUS_CONTAINER=nvcr.io/nvidia/morpheus/morpheus
+ARG MORPHEUS_CONTAINER_VERSION=v22.08.00-runtime
+
+FROM ${MORPHEUS_CONTAINER}:${MORPHEUS_CONTAINER_VERSION} as base
+
+# Fix the entrypoint to work with different WORKDIR
+ENTRYPOINT [ "/opt/conda/bin/tini", "--", "/workspace/docker/entrypoint.sh" ]
+
+SHELL ["/bin/bash", "-c"]
+
+# Install vault
+RUN apt-get update \
+    && apt-get install -y \
+      apt-utils \
+      jq \
+      lsb-release \
+      software-properties-common \
+    && curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - \
+    && apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \
+    && apt-get update \
+    && apt-get install --reinstall -y  \
+      vault \
+    && apt-get clean all \
+    && rm -rf /var/lib/apt/lists/*
+
+# Fix vault install per: https://github.com/hashicorp/vault/issues/10924#issuecomment-1197259930
+RUN setcap -r /usr/bin/vault
+
+# # Install NGC CLI
+# COPY ${UBA_COMMON}/utils/*.sh ./
+# RUN chmod +x *.sh \
+#   && bash ngc-cli-install.sh
+
+# Install DFP dependencies
+RUN source activate morpheus \
+    && mamba install -y -c conda-forge \
+        boto3 \
+        dill \
+        ipywidgets \
+        jupyterlab \
+        kfp \
+        librdkafka \
+        mlflow \
+        nb_conda_kernels \
+        papermill \
+        s3fs
+
+WORKDIR /work/examples/dfp_workflow/morpheus
+
+# # This will get used by pipelines for the --s3_cache option
+# ENV DFP_S3_CACHE="/work/examples/dfp_workflow/morpheus/.s3_cache"
+
+# Set the tracking URI for mlflow
+ENV MLFLOW_TRACKING_URI="http://mlflow:5000"
+
+# Copy the sources
+COPY . ./
+
+CMD ["jupyter-lab", "--ip=0.0.0.0", "--no-browser", "--allow-root"]

examples/digital_fingerprinting/production/mlflow/Dockerfile

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.8-slim-buster
+
+# Install curl for health check
+RUN apt update && \
+    apt install -y --no-install-recommends \
+        curl libyaml-cpp-dev libyaml-dev && \
+    apt autoremove -y && \
+    apt clean all && \
+    rm -rf /var/cache/apt/* /var/lib/apt/lists/*
+
+# Install python packages
+RUN pip install mlflow boto3 pymysql pyyaml
+
+# We run on port 5000
+EXPOSE 5000
+
+HEALTHCHECK CMD curl -f http://localhost:5000/health || exit 1

examples/digital_fingerprinting/production/morpheus/.dockerignore

@@ -0,0 +1,9 @@
+# Ignore any Dockerfile
+Dockerfile
+
+# ML Flow files
+artifacts/
+mlflow.db
+
+# Ignore any S3 cache folders
+*.s3_cache

examples/digital_fingerprinting/production/morpheus/.gitignore

@@ -0,0 +1 @@
+get_aws_credentials.sh

examples/digital_fingerprinting/production/morpheus/Dockerfile

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG MORPHEUS_CONTAINER=nvcr.io/nvidia/morpheus/morpheus
+ARG MORPHEUS_CONTAINER_VERSION=v22.09.00-runtime
+ARG ADDITIONAL_GROUPS
+ARG FIXUID_VERSION=0.5.1
+
+FROM ${MORPHEUS_CONTAINER}:${MORPHEUS_CONTAINER_VERSION} as common_base
+
+# Install fixuid
+RUN curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.5.1/fixuid-0.5.1-linux-amd64.tar.gz | tar -C /usr/bin -xzf - && \
+    chown root:root /usr/bin/fixuid && chmod 4755 /usr/bin/fixuid && mkdir -p /etc/fixuid && \
+    bash -c 'echo -e "\
+user: morpheus\n\
+group: morpheus\n\
+paths:\n\
+  - /home/morpheus\n\
+  - /opt/conda/envs/morpheus\n\
+" | tee /etc/fixuid/config.yml >/dev/null' && \
+    \
+    # Add a non-root user
+    useradd \
+      --uid 1000 --shell /bin/bash \
+      --user-group ${ADDITIONAL_GROUPS} \
+      --create-home --home-dir /home/morpheus \
+      morpheus
+
+# Fix the entrypoint to work with different WORKDIR
+ENTRYPOINT [ "/opt/conda/bin/tini", "--", "fixuid", "-q", "/workspace/docker/entrypoint.sh" ]
+
+SHELL ["/bin/bash", "-c"]
+
+# Install vault
+RUN apt-get update \
+    && apt-get install -y \
+      apt-utils \
+      jq \
+      lsb-release \
+      software-properties-common \
+    && curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - \
+    && apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \
+    && apt-get update \
+    && apt-get install --reinstall -y  \
+      vault \
+    && apt-get clean all \
+    && rm -rf /var/lib/apt/lists/*
+
+# Fix vault install per: https://github.com/hashicorp/vault/issues/10924#issuecomment-1197259930
+RUN setcap -r /usr/bin/vault
+
+# Copy the conda_env file
+COPY ./conda_env.yml ./
+
+# Install DFP dependencies
+RUN source activate morpheus \
+    && mamba env update -n morpheus -f ./conda_env.yml
+
+USER morpheus
+
+FROM common_base as base
+
+WORKDIR /work/examples/dfp_workflow/morpheus
+
+# This will get used by pipelines for the --s3_cache option
+# ENV DFP_S3_CACHE="/work/examples/dfp_workflow/morpheus/.s3_cache"
+
+# Copy the sources
+COPY . ./
+
+CMD ["bash", "-c", "./launch.sh"]