Misc import fixes and reset request_id

.github/workflows/ci_pipe.yml

@@ -0,0 +1,154 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CI Pipeline
+
+on:
+  workflow_call:
+    inputs:
+      aws_region:
+        default: 'us-west-2'
+        type: string
+      run_check:
+        required: true
+        type: boolean
+      container:
+        required: true
+        type: string
+      test_container:
+        required: true
+        type: string
+    secrets:
+      GHA_AWS_ACCESS_KEY_ID:
+        required: true
+      GHA_AWS_SECRET_ACCESS_KEY:
+        required: true
+      NGC_API_KEY:
+        required: true
+
+env:
+  AWS_DEFAULT_REGION:  ${{ inputs.aws_region }}
+  AWS_ACCESS_KEY_ID: "${{ secrets.GHA_AWS_ACCESS_KEY_ID }}"
+  AWS_SECRET_ACCESS_KEY: "${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }}"
+  CHANGE_TARGET: "${{ github.base_ref }}"
+  CUDA_PATH: "/usr/local/cuda/"
+  CUDA_VER: "11.5"
+  GH_TOKEN: "${{ github.token }}"
+  GIT_COMMIT: "${{ github.sha }}"
+  MORPHEUS_ROOT: "${{ github.workspace }}/morpheus"
+  WORKSPACE: "${{ github.workspace }}/morpheus"
+  WORKSPACE_TMP: "${{ github.workspace }}/tmp"
+
+
+jobs:
+  check:
+    if: ${{ inputs.run_check }}
+    name: Check
+    runs-on: [self-hosted, linux, amd64, cpu4]
+    timeout-minutes: 60
+    container:
+      credentials:
+        username: '$oauthtoken'
+        password: ${{ secrets.NGC_API_KEY }}
+      image: ${{ inputs.container }}
+    strategy:
+      fail-fast: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          lfs: false
+          path: 'morpheus'
+          fetch-depth: 0
+
+      - name: Check
+        shell: bash
+        run: ./morpheus/ci/scripts/github/checks.sh
+
+  build:
+    name: Build
+    runs-on: [self-hosted, linux, amd64, cpu16]
+    timeout-minutes: 60
+    container:
+      credentials:
+        username: '$oauthtoken'
+        password: ${{ secrets.NGC_API_KEY }}
+      image: ${{ inputs.container }}
+    strategy:
+      fail-fast: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          lfs: false
+          path: 'morpheus'
+
+      - name: Build:linux:x86_64:gcc
+        shell: bash
+        run: ./morpheus/ci/scripts/github/build.sh
+
+  test:
+    name: Test
+    needs: [build]
+    runs-on: [self-hosted, linux, amd64, gpu-v100-495-1]
+    timeout-minutes: 60
+    container:
+      credentials:
+        username: '$oauthtoken'
+        password: ${{ secrets.NGC_API_KEY }}
+      image: ${{ inputs.test_container }}
+      options: --cap-add=sys_nice
+      env:
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+        PARALLEL_LEVEL: '10'
+    strategy:
+      fail-fast: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          lfs: false
+          path: 'morpheus'
+
+      - name: Test:linux:x86_64:gcc
+        shell: bash
+        run: ./morpheus/ci/scripts/github/test.sh
+
+  documentation:
+    name: Documentation
+    needs: [build]
+    runs-on: [self-hosted, linux, amd64, cpu4]
+    timeout-minutes: 60
+    container:
+      credentials:
+        username: '$oauthtoken'
+        password: ${{ secrets.NGC_API_KEY }}
+      image: ${{ inputs.container }}
+    strategy:
+      fail-fast: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          lfs: false
+          path: 'morpheus'
+
+      - name: build_docs
+        shell: bash
+        run: ./morpheus/ci/scripts/github/docs.sh

.github/workflows/pull_request.yml

@@ -26,105 +26,14 @@ concurrency:
   group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
   cancel-in-progress: true
 
-env:
-  AWS_DEFAULT_REGION: us-west-2
-  AWS_ACCESS_KEY_ID: "${{ secrets.GHA_AWS_ACCESS_KEY_ID }}"
-  AWS_SECRET_ACCESS_KEY: "${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }}"
-  CHANGE_TARGET: "${{ github.base_ref }}"
-  CUDA_PATH: "/usr/local/cuda/"
-  CUDA_VER: "11.5"
-  GH_TOKEN: "${{ github.token }}"
-  GIT_COMMIT: "${{ github.sha }}"
-  MORPHEUS_ROOT: "${{ github.workspace }}/morpheus"
-  WORKSPACE: "${{ github.workspace }}/morpheus"
-  WORKSPACE_TMP: "${{ github.workspace }}/tmp"
-
-
 jobs:
-  check:
-    if: ${{ startsWith(github.ref_name, 'pull-request/') }}
-    name: Check
-    runs-on: [self-hosted, linux, amd64, cpu4]
-    timeout-minutes: 60
-    container:
-      image: rapidsai/ci:cuda11.5.1-ubuntu20.04-py3.8
-    strategy:
-      fail-fast: true
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          lfs: false
-          path: 'morpheus'
-          fetch-depth: 0
-
-      - name: Check
-        shell: bash
-        run: ./morpheus/ci/scripts/github/checks.sh
-
-  build:
-    name: Build
-    runs-on: [self-hosted, linux, amd64, cpu16]
-    timeout-minutes: 60
-    container:
-      image: rapidsai/ci:cuda11.5.1-ubuntu20.04-py3.8
-    strategy:
-      fail-fast: true
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          lfs: false
-          path: 'morpheus'
-
-      - name: Build:linux:x86_64:gcc
-        shell: bash
-        run: ./morpheus/ci/scripts/github/build.sh
-
-  test:
-    name: Test
-    needs: [build]
-    runs-on: [self-hosted, linux, amd64, gpu-v100-495-1]
-    timeout-minutes: 60
-    container:
-      image: rapidsai/ci:cuda11.5.1-ubuntu20.04-py3.8
-      options: --cap-add=sys_nice
-      env:
-        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
-        PARALLEL_LEVEL: '10'
-    strategy:
-      fail-fast: true
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          lfs: false
-          path: 'morpheus'
-
-      - name: Test:linux:x86_64:gcc
-        shell: bash
-        run: ./morpheus/ci/scripts/github/test.sh
-
-  documentation:
-    name: Documentation
-    needs: [build]
-    runs-on: [self-hosted, linux, amd64, cpu4]
-    timeout-minutes: 60
-    container:
-      image: rapidsai/ci:cuda11.5.1-ubuntu20.04-py3.8
-    strategy:
-      fail-fast: true
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          lfs: false
-          path: 'morpheus'
-
-      - name: build_docs
-        shell: bash
-        run: ./morpheus/ci/scripts/github/docs.sh
+  ci_pipe:
+    uses: ./.github/workflows/ci_pipe.yml
+    with:
+      run_check: ${{ startsWith(github.ref_name, 'pull-request/') }}
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-driver-221102
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-221102
+    secrets:
+      GHA_AWS_ACCESS_KEY_ID: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }}
+      GHA_AWS_SECRET_ACCESS_KEY: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }}
+      NGC_API_KEY: ${{ secrets.NGC_API_KEY }}

ci/runner/Dockerfile

@@ -0,0 +1,94 @@
+# syntax=docker/dockerfile:1.3
+
+# SPDX-FileCopyrightText: Copyright (c) 2022,NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Args used in FROM commands must come first
+ARG FROM_IMAGE="rapidsai/ci"
+ARG CUDA_PKG_VER=11-5
+ARG CUDA_SHORT_VER=11.5
+ARG CUDA_VER=11.5.1
+ARG LINUX_DISTRO=ubuntu
+ARG LINUX_VER=20.04
+ARG PROJ_NAME=morpheus
+ARG PYTHON_VER=3.8
+
+# Configure the base docker img
+FROM ${FROM_IMAGE}:cuda${CUDA_VER}-${LINUX_DISTRO}${LINUX_VER}-py${PYTHON_VER} AS base
+
+ARG PROJ_NAME
+ARG CUDA_SHORT_VER
+
+SHELL ["/bin/bash",  "-c"]
+
+# OS deps
+RUN apt update && \
+    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
+    apt install --no-install-recommends -y \
+        libnuma1 && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Create conda environment
+COPY ./docker/conda/environments/* /tmp/conda/
+
+RUN CONDA_ALWAYS_YES=true /opt/conda/bin/mamba env create -n ${PROJ_NAME} -q --file /tmp/conda/cuda${CUDA_SHORT_VER}_dev.yml && \
+    sed -i "s/conda activate base/conda activate ${PROJ_NAME}/g" ~/.bashrc && \
+    conda clean -afy && \
+    rm -rf /tmp/conda
+
+
+# ============ driver ==================
+FROM base as driver
+
+ARG CUDA_PKG_VER
+
+RUN apt update && \
+    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
+    apt install --no-install-recommends -y \
+    libcublas-dev-${CUDA_PKG_VER} \
+    libcufft-dev-${CUDA_PKG_VER} \
+    libcurand-dev-${CUDA_PKG_VER} \
+    libcusolver-dev-${CUDA_PKG_VER} \
+    libnvidia-compute-495 && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+
+
+# ============ test ==================
+FROM base as test
+
+ARG PROJ_NAME
+
+RUN apt update && \
+    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
+    apt install --no-install-recommends -y \
+        nodejs \
+        npm \
+        openjdk-11-jdk && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install camouflage needed for unittests to mock a triton server
+RUN npm install -g camouflage-server@0.9 && \
+    npm cache clean --force
+
+# Install pytest-kafka
+# Installing pytest-kafka from source instead of conda/pip as the setup.py includes helper methods for downloading Kafka
+# https://gitlab.com/karolinepauls/pytest-kafka/-/issues/9
+RUN git clone https://gitlab.com/karolinepauls/pytest-kafka.git /opt/pytest-kafka && \
+    cd /opt/pytest-kafka && \
+    source activate ${PROJ_NAME} && \
+    python setup.py develop