Daily and Pre-merge workflows updates (#2085)

* CI instance start script updated to run with full automation * Updated daily workflow to use reusable workflow and split tests to each task * Revised pre-merge workflow to reuse workflow and matrix * Updated .ci/README.md
openvinotoolkit · May 3, 2023 · e67ce53 · e67ce53
1 parent 63d9d7a
commit e67ce53
Show file tree

Hide file tree

Showing 10 changed files with 168 additions and 171 deletions.
diff --git a/.ci/README.md b/.ci/README.md
@@ -43,16 +43,7 @@
    training_extensions$ .ci/start-runner.sh otx-ci-container <github-token> <codacy-token> -g 0
    ```
 
-   Once it executed, An interactive session for the runner configuration will be started and you should input some information to each prompt.
-
-   - `Enter the name of the runner group to add this runner to:`
-     - Just press Enter key for default
-   - `Enter the name of runner:`
-     - Input the runner name which would be used by Github. If you want to replace an existing runner to this one, input the same runner's name.
-   - [`Optional`] If you input the existing runner's name to replace in the step above, confirmation prompt will be shown.
-     - Input 'y' for the replacing
-     - `Enter name of work folder:`
-     - Just press Enter key for default
+   If there exist the container named as same, it will be stopped before starting a new container.
 
    All configurations were configured and the runner is started successfully, you can see the messages below.
 

diff --git a/.ci/build.sh b/.ci/build.sh
@@ -55,8 +55,8 @@ docker build -f .ci/Dockerfile \
 --build-arg NO_PROXY="${no_proxy:?}" \
 --build-arg ver_cuda="$VER_CUDA" \
 --build-arg action_runner_url="$ACTIONS_RUNNER_URL" \
---tag registry.toolbox.iotg.sclab.intel.com/ote/ci/cu"$VER_CUDA"/runner:"$TAG" \
---tag registry.toolbox.iotg.sclab.intel.com/ote/ci/cu"$VER_CUDA"/runner:latest .; RET=$?
+--tag "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:"$TAG" \
+--tag "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:latest .; RET=$?
 
 
 if [ $RET -ne 0 ]; then
@@ -67,12 +67,12 @@ fi
 echo "Successfully built docker image."
 
 if [ "$PUSH" == "yes" ]; then
-    docker push registry.toolbox.iotg.sclab.intel.com/ote/ci/cu"$VER_CUDA"/runner:"$TAG"; RET=$?
+    docker push "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:"$TAG"; RET=$?
     if [ $RET -ne 0 ]; then
         echo "failed to push a docker image to registry. $RET"
         exit 1
     fi
-    docker push registry.toolbox.iotg.sclab.intel.com/ote/ci/cu"$VER_CUDA"/runner:latest; RET=$?
+    docker push "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:latest; RET=$?
     if [ $RET -ne 0 ]; then
         echo "failed to push a docker image to registry. $RET"
         exit 1

diff --git a/.ci/start-runner.sh b/.ci/start-runner.sh
@@ -1,8 +1,9 @@
 #!/bin/bash
 
 GPU_ID="all"
-VER_CUDA="11.1.1"
+VER_CUDA="11.7.1"
 TAG_RUNNER="latest"
+ADDITIONAL_LABELS=""
 DEBUG_CONTAINER=false
 POSITIONAL=()
 while [[ $# -gt 0 ]]; do
@@ -24,6 +25,11 @@ while [[ $# -gt 0 ]]; do
       shift # past argument
       shift # past value
       ;;
+    -l|=--labels)
+      ADDITIONAL_LABELS="$2"
+      shift # past argument
+      shift # past value
+      ;;
     -d|--debug)
       DEBUG_CONTAINER=true
       shift # past argument
@@ -44,11 +50,16 @@ set -- "${POSITIONAL[@]}" # restore positional parameters
 
 if [[ "$#" -lt 3 ||  "$DEFAULT" == "yes" ]] && [ $DEBUG_CONTAINER = false ]; then
 cat << EndofMessage
-    USAGE: $0 <container-name> <github-token> <codacy-token> [Options]
+    USAGE: $0 <container-name> <github-token> <instance-name> [Options]
+    Positional args
+        <container-name>    Prefix to the ci container
+        <github-token>      Github token string
+        <instance-name>     Prefix to the actions-runner
     Options
         -g|--gpu-ids        GPU ID or IDs (comma separated) for runner or 'all'
         -c|--cuda           Specify CUDA version
         -t|--tag            Specify TAG for the CI container
+        -l|--labels         Additional label string to set the actions-runner
         -d|--debug          Flag to start debugging CI container
         -h|--help           Print this message
 EndofMessage
@@ -57,12 +68,22 @@ fi
 
 CONTAINER_NAME=$1
 GITHUB_TOKEN=$2
-CODACY_TOKEN=$3
+INSTANCE_NAME=$3
+LABELS="self-hosted,Linux,X64"
+
+if [ -z "$ADDITIONAL_LABELS" ]; then
+    LABELS="$LABELS,$ADDITIONAL_LABELS"
+fi
 
 if [ "$DEBUG_CONTAINER" = true ]; then
     CONTAINER_NAME="otx-ci-container-debug"
 fi
 
+CONTAINER_NAME="$CONTAINER_NAME"-${GPU_ID//,/_}
+INSTANCE_NAME="$INSTANCE_NAME"-${GPU_ID//,/_}
+
+echo "container name = $CONTAINER_NAME, instance name = $INSTANCE_NAME"
+
 docker inspect "$CONTAINER_NAME"; RET=$?
 
 if [ $RET -eq 0 ]; then
@@ -76,10 +97,11 @@ if [ "$DEBUG_CONTAINER" = true ]; then
     docker run -itd \
         --env NVIDIA_VISIBLE_DEVICES="$GPU_ID" \
         --runtime=nvidia \
-        --ipc=host \
-        --cpus=40 \
+        --ipc=private \
+        --shm-size=24g \
+        --cpu-shares=1024 \
         --name "$CONTAINER_NAME" \
-        registry.toolbox.iotg.sclab.intel.com/ote/ci/cu"$VER_CUDA"/runner:"$TAG_RUNNER"; RET=$?
+        "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:"$TAG_RUNNER"; RET=$?
 
     if [ $RET -ne 0 ]; then
         echo "failed to start ci container. $RET"
@@ -92,10 +114,11 @@ else
     docker run -itd \
         --env NVIDIA_VISIBLE_DEVICES="$GPU_ID" \
         --runtime=nvidia \
-        --ipc=host \
-        --cpus=40 \
+        --ipc=private \
+        --shm-size=24g \
+        --cpu-shares=1024 \
         --name "$CONTAINER_NAME" \
-        registry.toolbox.iotg.sclab.intel.com/ote/ci/cu"$VER_CUDA"/runner:"$TAG_RUNNER"; RET=$?
+        "$DOCKER_REG_ADDR"/ote/ci/cu"$VER_CUDA"/runner:"$TAG_RUNNER"; RET=$?
 
     if [ $RET -ne 0 ]; then
         echo "failed to start ci container. $RET"
@@ -108,17 +131,20 @@ echo "Successfully started ci container - $CONTAINER_NAME"
 
 docker exec -it "$CONTAINER_NAME" bash -c \
     "./actions-runner/config.sh  \
+    --unattended \
     --url https://github.com/openvinotoolkit/training_extensions \
-    --token $GITHUB_TOKEN" ; RET=$?
+    --token $GITHUB_TOKEN \
+    --name $INSTANCE_NAME \
+    --labels $LABELS \
+    --replace true" ; RET=$?
 
 if [ $RET -ne 0 ]; then
     echo "failed to configure the runner. $RET"
     exit 1
 fi
 
 docker exec -d "$CONTAINER_NAME" bash -c \
-    "export CODACY_PROJECT_TOKEN=$CODACY_TOKEN && \
-    ./actions-runner/run.sh" ; RET=$?
+    "./actions-runner/run.sh" ; RET=$?
 
 if [ $RET -ne 0 ]; then
     echo "failed to start actions runner. $RET"

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
@@ -6,25 +6,34 @@ on:
     # every UTC 7PM from Mon to Fri
     - cron: "0 19 * * 1-5"
 
+# This is what will cancel the workflow concurrency
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  Daily-Tests:
-    runs-on: [self-hosted, linux, x64, dev]
-    timeout-minutes: 1440
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install -r requirements/dev.txt
-      - name: E2E Tests
-        run: tox -e pre-merge-all-py310 -- tests/e2e
-      - name: Upload test results
-        uses: actions/upload-artifact@v3
-        with:
-          name: test-results
-          path: .tox/test-results.xml
-        # Use always() to always run this step to publish test results when there are test failures
-        if: ${{ always() }}
+  E2E-tests:
+    strategy:
+      matrix:
+        include:
+          - task: "act"
+            test_dir: "tests/e2e/cli/action"
+          - task: "ano"
+            test_dir: "tests/e2e/cli/anomaly"
+          - task: "cls"
+            test_dir: "tests/e2e/cli/classification"
+          - task: "det"
+            test_dir: "tests/e2e/cli/detection"
+          - task: "iseg"
+            test_dir: "tests/e2e/cli/instance_segmentation"
+          - task: "seg"
+            test_dir: "tests/e2e/cli/semantic_segmentation"
+    name: E2E-Test-py310-${{ matrix.task }}
+    uses: ./.github/workflows/run_tests_in_tox.yml
+    with:
+      python-version: "3.10"
+      toxenv-pyver: "py310"
+      toxenv-task: ${{ matrix.task }}
+      tests-dir: ${{ matrix.test_dir }}
+      timeout-minutes: 360
+      upload-artifact: true
diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml
@@ -1,4 +1,4 @@
-name: Pre-Merge Checks
+name: PR Checks
 
 on:
   push:
@@ -30,8 +30,8 @@ jobs:
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Code quality checks
-        run: tox -e pre-commit-all-py310
-  Pre-Merge-Unit-Test:
+        run: tox -vv -e pre-commit-all-py310
+  Unit-Test:
     runs-on: [self-hosted, linux, x64, dev]
     needs: Code-Quality-Checks
     timeout-minutes: 120
@@ -55,7 +55,7 @@ jobs:
       - name: Install dependencies
         run: python -m pip install -r requirements/dev.txt
       - name: Run unit test
-        run: tox -e pre-merge-all-${{ matrix.tox-env }} -- tests/unit
+        run: tox -vv -e tests-all-${{ matrix.tox-env }} -- tests/unit
       - name: Upload coverage reports to Codecov
         run: |
           # If the workflow is triggered from PR then it gets the commit id from the PR.
@@ -74,108 +74,31 @@ jobs:
           curl -Os https://uploader.codecov.io/latest/linux/codecov
           chmod +x codecov
           ./codecov -t ${{ secrets.CODECOV_TOKEN }} --sha $COMMIT_ID -U $HTTP_PROXY -f .tox/coverage.xml -F ${{ matrix.tox-env }}
-  Pre-Merge-Integration-Common-Test:
-    runs-on: [self-hosted, linux, x64, dev]
-    needs: Pre-Merge-Unit-Test
-    timeout-minutes: 120
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install -r requirements/dev.txt
-      - name: Run integration test
-        run: tox -e pre-merge-all-py310 -- tests/integration/cli/test_cli.py
-  Pre-Merge-Integration-Cls-Test:
-    runs-on: [self-hosted, linux, x64, dev]
-    needs: Pre-Merge-Unit-Test
-    timeout-minutes: 120
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install -r requirements/dev.txt
-      - name: Run integration test
-        run: tox -e pre-merge-cls-py310
-  Pre-Merge-Integration-Det-Test:
-    runs-on: [self-hosted, linux, x64, dev]
-    needs: Pre-Merge-Unit-Test
-    timeout-minutes: 120
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install -r requirements/dev.txt
-      - name: Run integration test
-        run: tox -e pre-merge-det-py310 -- tests/integration/cli/detection
-  Pre-Merge-Integration-Instance-Seg-Test:
-    runs-on: [self-hosted, linux, x64, dev]
-    needs: Pre-Merge-Unit-Test
-    timeout-minutes: 120
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install -r requirements/dev.txt
-      - name: Run integration test
-        run: tox -e pre-merge-det-py310 -- tests/integration/cli/instance_segmentation
-  Pre-Merge-Integration-Semantic-Seg-Test:
-    runs-on: [self-hosted, linux, x64, dev]
-    needs: Pre-Merge-Unit-Test
-    timeout-minutes: 120
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install -r requirements/dev.txt
-      - name: Run integration test
-        run: tox -e pre-merge-seg-py310 -- tests/integration/cli/semantic_segmentation
-  Pre-Merge-Integration-Action-Test:
-    runs-on: [self-hosted, linux, x64, dev]
-    needs: Pre-Merge-Unit-Test
-    timeout-minutes: 120
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install -r requirements/dev.txt
-      - name: Run integration test
-        run: tox -e pre-merge-act-py310
-  Pre-Merge-Integration-Anomaly-Test:
-    runs-on: [self-hosted, linux, x64, dev]
-    needs: Pre-Merge-Unit-Test
-    timeout-minutes: 120
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install -r requirements/dev.txt
-      - name: Run integration test
-        run: tox -e pre-merge-ano-py310
+  Integration-Test:
+    needs: Unit-Test
+    strategy:
+      matrix:
+        include:
+          - task: "all"
+            test_dir: "tests/integration/cli/test_cli.py"
+          - task: "cls"
+            test_dir: "tests/integration/cli/classification"
+          - task: "det"
+            test_dir: "tests/integration/cli/detection"
+          - task: "iseg"
+            test_dir: "tests/integration/cli/instance_segmentation"
+          - task: "seg"
+            test_dir: "tests/integration/cli/semantic_segmentation"
+          - task: "act"
+            test_dir: "tests/integration/cli/action"
+          - task: "ano"
+            test_dir: "tests/integration/cli/anomaly"
+    name: Integration-Test-py310-${{ matrix.task }}
+    uses: ./.github/workflows/run_tests_in_tox.yml
+    with:
+      python-version: "3.10"
+      toxenv-pyver: "py310"
+      toxenv-task: ${{ matrix.task }}
+      tests-dir: ${{ matrix.test_dir }}
+      timeout-minutes: 120
+      upload-artifact: true