Various Dockerfile-based updates for 22.11 (#466)

Closes #464

Authors:
  - Pete MacKinnon (https://github.com/pdmack)

Approvers:
  - David Gardner (https://github.com/dagardner-nv)
  - https://github.com/raykallen
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #466

ci/conda/recipes/morpheus/meta.yaml

@@ -79,7 +79,7 @@ outputs:
         - distributed
         - docker-py 5.0
         - grpcio # Version determined from cudf
-        - mlflow >=1.23
+        - mlflow >1.29,<2
         - networkx 2.8
         - numpydoc 1.4
         - pandas 1.3

docker/Dockerfile

@@ -145,9 +145,6 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target
     CONDA_ALWAYS_YES=true /opt/conda/bin/mamba install -n morpheus -c local -c rapidsai -c nvidia -c nvidia/label/dev -c conda-forge morpheus &&\
     # Install runtime dependencies that are pip-only
     /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_VER}_runtime.yml &&\
-    # Install jupyter support (e.g., DFP)
-    # TODO: this might not be the right spot to get these
-    /opt/conda/bin/mamba install -n morpheus ipywidgets jupyterlab nb_conda_kernels &&\
     # Clean and activate
     conda clean -afy
 

docker/conda/environments/cuda11.5_dev.yml

@@ -57,7 +57,7 @@ dependencies:
     - include-what-you-use=0.18
     - isort
     - librdkafka=1.7.0
-    - mlflow>=1.23
+    - mlflow>1.29,<2
     - myst-parser==0.17
     - networkx=2.8
     - ninja=1.10

docker/conda/environments/cuda11.5_runtime.yml

@@ -20,6 +20,7 @@ channels:
     - rapidsai-nightly
     - conda-forge
 dependencies:
+    - nb_conda_kernels
     - pip
     ####### Morpheus Pip Dependencies (keep sorted!) #######
     - pip:

docker/conda/environments/requirements.txt

@@ -5,6 +5,9 @@
 ####### Pip-only runtime dependencies (keep sorted!) #######
 # Packages listed here should also be listed in setup.py
 git+https://github.com/nv-morpheus/dfencoder.git@branch-22.09#egg=dfencoder
+ipywidgets
+jupyter-core>=4.11.2,<5.0
+jupyterlab
 nvidia-pyindex
 torch==1.10.2+cu113
 tritonclient[all]==2.17.*

examples/digital_fingerprinting/production/README.md

@@ -106,6 +106,8 @@ MLflow for this production digital fingerprint use case can be installed from NG
 
 The deployment of the [Morpheus SDK Client](../../../docs/source/morpheus_quickstart_guide.md#install-morpheus-sdk-client) is also done _almost_ the same way as what's specified in the Quick Start Guide. However, you would specify command arguments differently for this production DFP use case.
 
+NOTE: The published SDK image includes a minimal set of packages for launching JupyterLab but you will likely still want to update the conda environment inside the running pod with the `conda_env.yml` file in this same directory to install other use case dependencies such as boto3 and s3fs.
+
 #### Notebooks
 
 ```

examples/digital_fingerprinting/production/mlflow/Dockerfile

@@ -24,7 +24,7 @@ RUN apt update && \
     rm -rf /var/cache/apt/* /var/lib/apt/lists/*
 
 # Install python packages
-RUN pip install "mlflow<1.29.0" boto3 pymysql pyyaml
+RUN pip install "mlflow>1.29.0,<2" boto3 pymysql pyyaml
 
 # We run on port 5000
 EXPOSE 5000

models/mlflow/docker/Dockerfile

@@ -16,6 +16,7 @@
 ARG FROM_IMAGE=gpuci/miniforge-cuda
 ARG CUDA_VER=11.5
 ARG LINUX_VER=ubuntu20.04
+ARG PYTHON_VER=3.8
 FROM ${FROM_IMAGE}:${CUDA_VER}-runtime-${LINUX_VER} AS base
 
 WORKDIR /mlflow
@@ -24,13 +25,15 @@ COPY . ./
 
 # need to do an upgrade in case gpuci is stale for whatever reason
 RUN apt-get update && apt upgrade -y && apt-get install -y procps vim
+# but pin python version to avoid some newer CVE
+RUN source /etc/profile.d/conda.sh && conda deactivate && conda install python=${PYTHON_VER}
 RUN /opt/conda/bin/conda env create -f docker/conda/mlflow-env.yml
 RUN sed -i 's/conda activate base/conda activate mlflow/g' ~/.bashrc
 
 SHELL ["/opt/conda/bin/conda", "run", "-n", "mlflow", "/bin/bash", "-c"]
 
 ARG TRITON_DIR=/mlflow/triton-inference-server
-ARG TRITON_VER=r22.08
+ARG TRITON_VER=r22.10
 
 RUN mkdir ${TRITON_DIR} && \
     cd ${TRITON_DIR} && \
@@ -45,6 +48,9 @@ RUN ln -sf ${TRITON_DIR}/server/deploy/mlflow-triton-plugin/scripts/publish_mode
 # remedy for CVE-2015-20107
 RUN find / -name '*mailcap*.*py*' | xargs rm
 
+# remedy for CVE-2022-42919
+RUN find / -path '*multiprocessing/util.py' -exec sed -i 's/=\s*_platform_supports_abstract_sockets()/= False/g' {} +
+
 EXPOSE 5000
 
 # Set the entrypoint to use the entrypoint.sh script which sets the conda env

models/mlflow/docker/conda/mlflow-env.yml

@@ -21,9 +21,9 @@ dependencies:
   - python=3.8
   - pip
   - onnx
-  - openssl>=1.1.1q
+  - openssl>=1.1.1s,<3
   - psycopg2<3
   - pip:
     - boto3
-    - mlflow<1.29.0
+    - mlflow>1.29.0,<2
     - pymysql