Merge remote-tracking branch 'upstream/master'

.azure-pipelines/get_dut_version.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import argparse
 import logging
 import os
@@ -13,7 +15,7 @@
 if ansible_path not in sys.path:
     sys.path.append(ansible_path)
 
-from devutil.devices import init_localhost, init_testbed_sonichosts  # noqa E402
+from devutil.devices.factory import init_localhost, init_testbed_sonichosts  # noqa E402
 
 logger = logging.getLogger(__name__)
 

.azure-pipelines/pr_test_scripts.yaml

@@ -38,7 +38,7 @@ t0:
   - override_config_table/test_override_config_table.py
   - pc/test_po_cleanup.py
   - pc/test_po_update.py
-#  - platform_tests/test_advanced_reboot.py::test_warm_reboot
+  - platform_tests/test_advanced_reboot.py::test_warm_reboot
   - platform_tests/test_cpu_memory_usage.py
   - process_monitoring/test_critical_process_monitoring.py
   - radv/test_radv_ipv6_ra.py
@@ -63,6 +63,7 @@ t0:
   - tacacs/test_ro_user.py
   - tacacs/test_rw_user.py
   - telemetry/test_telemetry.py
+  - telemetry/test_events.py
   - test_features.py
   - test_interfaces.py
   - test_procdockerstatsd.py

.azure-pipelines/pytest-collect-only.yml

@@ -35,7 +35,7 @@ steps:
     set -x
 
     sudo docker exec -t -w /var/src/sonic-mgmt/tests sonic-mgmt-collect \
-      pytest --inventory ../ansible/veos_vtb --host-pattern all \
+      python3 -m pytest --inventory ../ansible/veos_vtb --host-pattern all \
       --testbed_file vtestbed.yaml --testbed vms-kvm-t0 \
       --ignore saitests --ignore ptftests --ignore acstests \
       --ignore scripts --ignore k8s --ignore sai_qualify --ignore common \

.azure-pipelines/run-test-elastictest-template.yml

@@ -0,0 +1,266 @@
+parameters:
+  - name: TOPOLOGY
+    type: string
+
+  - name: POLL_INTERVAL
+    type: number
+    default: 10
+
+  - name: POLL_TIMEOUT
+    type: number
+    default: 36000
+
+  - name: MIN_WORKER
+    type: string
+    default: 1
+
+  - name: MAX_WORKER
+    type: string
+    default: 1
+
+  - name: NUM_ASIC
+    type: number
+    default: 1
+
+  - name: TEST_SET
+    type: string
+    default: ""
+
+  - name: DEPLOY_MG_EXTRA_PARAMS
+    type: string
+    default: ""
+
+  - name: COMMON_EXTRA_PARAMS
+    type: string
+    default: ""
+
+  - name: VM_TYPE
+    type: string
+    default: "ceos"
+
+  - name: TESTBED_NAME
+    type: string
+    default: ""
+
+  - name: IMAGE_URL
+    type: string
+    default: ""
+
+  - name: HWSKU
+    type: string
+    default: ""
+
+  - name: TEST_PLAN_TYPE
+    type: string
+    default: ""
+
+  - name: PLATFORM
+    type: string
+    default: ""
+
+  - name: SCRIPTS
+    type: string
+    default: ""
+
+  - name: FEATURES
+    type: string
+    default: ""
+
+  - name: SCRIPTS_EXCLUDE
+    type: string
+    default: ""
+
+  - name: FEATURES_EXCLUDE
+    type: string
+    default: ""
+
+  - name: REPO_NAME
+    type: string
+    default: ""
+
+  - name: MGMT_BRANCH
+    type: string
+    default: ""
+
+  - name: MGMT_URL
+    type: string
+    default: "https://raw.githubusercontent.com/sonic-net/sonic-mgmt"
+
+  - name: STOP_ON_FAILURE
+    type: string
+    default: ""
+
+  - name: RETRY_TIMES
+    type: string
+    default: ""
+
+  - name: DUMP_KVM_IF_FAIL
+    type: string
+    default: "True"
+    values:
+      - "True"
+      - "False"
+
+  - name: REQUESTER
+    type: string
+    default: ""
+
+  - name: MAX_RUN_TEST_MINUTES
+    type: number
+    default: 480
+
+  - name: KVM_IMAGE_BRANCH
+    type: string
+    default: ""
+
+  - name: EXPECTED_RESULT
+    type: string
+    default: ""
+
+
+steps:
+  - ${{ if not(contains(variables['BUILD.REPOSITORY.NAME'], 'sonic-mgmt')) }}:
+      - script: |
+          # If not sonic-mgmt/sonic-mgmt-int repo, need to download test_plan.py and pr_test_scripts.yaml
+          set -ex
+
+          curl "https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/test_plan.py" -o ./.azure-pipelines/test_plan.py
+        displayName: "Download test plan script"
+      - script: |
+          # If not sonic-mgmt/sonic-mgmt-int repo, need to download pr_test_scripts.yaml
+          set -ex
+
+          # If public build image repo, download pr test scripts from public sonic-mgmt repo
+          if [[ "$(BUILD.REPOSITORY.NAME)" = "sonic-net/sonic-buildimage" ]]; then
+            curl "${{ parameters.MGMT_URL }}/${{ parameters.MGMT_BRANCH }}/.azure-pipelines/pr_test_scripts.yaml" -o ./.azure-pipelines/pr_test_scripts.yaml
+
+          # Else, internal build image repo, download from internal sonic-mgmt repo
+          else
+            curl -u $(AZP_REPO_ACCESS_TOKEN) "${{ parameters.MGMT_URL }}&commitOrBranch=${{ parameters.MGMT_BRANCH }}&api-version=5.0-preview.1&path=.azure-pipelines%2Fpr_test_scripts.yaml" -o ./.azure-pipelines/pr_test_scripts.yaml
+          fi
+        displayName: "Download pr script"
+  - ${{ else }}:
+      - ${{ if ne(parameters.MGMT_BRANCH, 'master') }}:
+          - script: |
+              # Else, sonic-mgmt repo, if not master branch, need to download test_plan.py
+              set -ex
+              curl "https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/test_plan.py" -o ./.azure-pipelines/test_plan.py
+            displayName: "Download test plan script"
+
+  - script: |
+      set -e
+
+      pip install PyYAML
+
+      rm -f new_test_plan_id.txt
+
+      python ./.azure-pipelines/test_plan.py create \
+      -t ${{ parameters.TOPOLOGY }} \
+      -o new_test_plan_id.txt \
+      --min-worker ${{ parameters.MIN_WORKER }} \
+      --max-worker ${{ parameters.MAX_WORKER }} \
+      --test-set ${{ parameters.TEST_SET }} \
+      --kvm-build-id $(KVM_BUILD_ID) \
+      --kvm-image-branch "${{ parameters.KVM_IMAGE_BRANCH }}" \
+      --deploy-mg-extra-params "${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" \
+      --common-extra-params "${{ parameters.COMMON_EXTRA_PARAMS }}" \
+      --vm-type ${{ parameters.VM_TYPE }} --num-asic ${{ parameters.NUM_ASIC }} \
+      --image_url ${{ parameters.IMAGE_URL }} \
+      --hwsku ${{ parameters.HWSKU }} \
+      --test-plan-type ${{ parameters.TEST_PLAN_TYPE }} \
+      --platform ${{ parameters.PLATFORM }} \
+      --testbed-name "${{ parameters.TESTBED_NAME }}" \
+      --scripts "${{ parameters.SCRIPTS }}" \
+      --features "${{ parameters.FEATURES }}" \
+      --scripts-exclude "${{ parameters.SCRIPTS_EXCLUDE }}" \
+      --features-exclude "${{ parameters.FEATURES_EXCLUDE }}" \
+      --repo-name ${{ parameters.REPO_NAME }} \
+      --mgmt-branch ${{ parameters.MGMT_BRANCH }} \
+      --stop-on-failure ${{ parameters.STOP_ON_FAILURE }} \
+      --retry-times ${{ parameters.RETRY_TIMES }} \
+      --dump-kvm-if-fail ${{ parameters.DUMP_KVM_IF_FAIL }} \
+      --requester "${{ parameters.REQUESTER }}" \
+      --max-execute-seconds $((${{ parameters.MAX_RUN_TEST_MINUTES }} * 60))
+
+      TEST_PLAN_ID=`cat new_test_plan_id.txt`
+
+      echo "Created test plan $TEST_PLAN_ID"
+
+      echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m"
+      echo -e -n "\033[33mPlease visit Elastictest page \033[0m"
+      echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
+      echo -e "\033[33mfor detailed test plan progress \033[0m"
+
+      echo "##vso[task.setvariable variable=TEST_PLAN_ID]$TEST_PLAN_ID"
+    displayName: "Trigger test"
+
+  - script: |
+      set -e
+      echo "Lock testbed"
+
+      echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m"
+      echo -e -n "\033[33mPlease visit Elastictest page \033[0m"
+      echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
+      echo -e "\033[33mfor detailed test plan progress \033[0m"
+
+      # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED"
+      echo "##[group][test_plan.py] poll LOCK_TESTBED status"
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state LOCK_TESTBED
+      echo "##[endgroup]"
+    displayName: "Lock testbed"
+
+  - script: |
+      set -e
+      echo "Prepare testbed"
+      echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient"
+
+      echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m"
+      echo -e -n "\033[33mPlease visit Elastictest page \033[0m"
+      echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
+      echo -e "\033[33mfor detailed test plan progress \033[0m"
+
+      # When "PREPARE_TESTBED" finish, it changes into "EXECUTING"
+      echo "##[group][test_plan.py] poll PREPARE_TESTBED status"
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state PREPARE_TESTBED
+      echo "##[endgroup]"
+    displayName: "Prepare testbed"
+
+  - script: |
+      set -e
+      echo "Run test"
+
+      echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m"
+      echo -e -n "\033[33mPlease visit Elastictest page \033[0m"
+      echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
+      echo -e "\033[33mfor detailed test plan progress \033[0m"
+
+      # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED"
+      echo "##[group][test_plan.py] poll EXECUTING status"
+      python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state EXECUTING --expected-result ${{ parameters.EXPECTED_RESULT }}
+      echo "##[endgroup]"
+    displayName: "Run test"
+    timeoutInMinutes: ${{ parameters.MAX_RUN_TEST_MINUTES }}
+
+  - ${{ if eq(parameters.DUMP_KVM_IF_FAIL, 'True') }}:
+      - script: |
+          set -e
+          echo "KVM dump"
+
+          echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m"
+          echo -e -n "\033[33mPlease visit Elastictest page \033[0m"
+          echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID "
+          echo -e "\033[33mfor detailed test plan progress \033[0m"
+
+          # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED"
+          echo "##[group][test_plan.py] poll KVMDUMP status"
+          python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --expected-state KVMDUMP
+          echo "##[endgroup]"
+        condition: succeededOrFailed()
+        displayName: "KVM dump"
+
+  - script: |
+      set -e
+      echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect."
+      python ./.azure-pipelines/test_plan.py cancel -i "$(TEST_PLAN_ID)"
+    condition: always()
+    displayName: "Finalize running test plan"

.azure-pipelines/run-test-scheduler-template.yml

@@ -105,6 +105,10 @@ parameters:
   type: number
   default: 480
 
+- name: KVM_IMAGE_BRANCH
+  type: string
+  default: ""
+
 
 steps:
   - script: |
@@ -116,6 +120,7 @@ steps:
 
       python ./.azure-pipelines/test_plan.py create -t ${{ parameters.TOPOLOGY }} -o new_test_plan_id.txt --min-worker ${{ parameters.MIN_WORKER }} --max-worker ${{ parameters.MAX_WORKER }} \
       --test-set ${{ parameters.TEST_SET }} --kvm-build-id $(KVM_BUILD_ID) \
+      --kvm-image-branch "${{ parameters.KVM_IMAGE_BRANCH }}" \
       --deploy-mg-extra-params "${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" \
       --common-extra-params "${{ parameters.COMMON_EXTRA_PARAMS }}" \
       --vm-type ${{ parameters.VM_TYPE }} --num-asic ${{ parameters.NUM_ASIC }} \