Example branch

.azure-pipelines/get_dut_version.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+import argparse
+import logging
+import os
+import sys
+import json
+import yaml
+
+_self_dir = os.path.dirname(os.path.abspath(__file__))
+base_path = os.path.realpath(os.path.join(_self_dir, ".."))
+if base_path not in sys.path:
+    sys.path.append(base_path)
+ansible_path = os.path.realpath(os.path.join(_self_dir, "../ansible"))
+if ansible_path not in sys.path:
+    sys.path.append(ansible_path)
+
+from devutil.devices.factory import init_localhost, init_testbed_sonichosts  # noqa E402
+
+logger = logging.getLogger(__name__)
+
+RC_INIT_FAILED = 1
+RC_GET_DUT_VERSION_FAILED = 2
+
+ASIC_NAME_PATH = '../ansible/group_vars/sonic/variables'
+
+
+def read_asic_name(hwsku):
+    asic_name_file = os.path.join(os.path.dirname(__file__), ASIC_NAME_PATH)
+    try:
+        with open(asic_name_file) as f:
+            asic_name = yaml.safe_load(f)
+
+        asic_name_dict = {}
+        for key, value in asic_name.items():
+            if "hwskus" in key:
+                asic_name_dict[key] = value
+
+        for name, hw in asic_name_dict.items():
+            if hwsku in hw:
+                return name.split('_')[1]
+
+        return "unknown"
+
+    except IOError:
+        return None
+
+
+def get_duts_version(sonichosts, output=None):
+    try:
+        ret = {}
+        duts_version = sonichosts.command("show version")
+        for dut, version in duts_version.items():
+            ret[dut] = {}
+            dut_version = version["stdout_lines"]
+
+            for line in dut_version:
+                if ":" in line:
+                    line_splitted = line.split(":", 1)
+                    key = line_splitted[0].strip()
+                    value = line_splitted[1].strip()
+                    if key == "Docker images":
+                        ret[dut]["Docker images"] = []
+                        continue
+                    elif key == "ASIC":
+                        ret[dut]["ASIC TYPE"] = value
+                        continue
+                    elif key == "HwSKU":
+                        ret[dut]["ASIC"] = read_asic_name(value)
+                    ret[dut][key] = value
+                elif "docker" in line:
+                    line_splitted = line.split()
+                    ret[dut]["Docker images"].append({"REPOSITORY": line_splitted[0],
+                                                      "TAG": line_splitted[1],
+                                                      "IMAGE ID": line_splitted[2],
+                                                      "SIZE": line_splitted[3]})
+
+        if output:
+            with open(output, "w") as f:
+                f.write(json.dumps(ret))
+                f.close()
+        else:
+            print(ret)
+    except Exception as e:
+        logger.error("Failed to get DUT version: {}".format(e))
+        sys.exit(RC_GET_DUT_VERSION_FAILED)
+
+
+def validate_args(args):
+    _log_level_map = {
+        "debug": logging.DEBUG,
+        "info": logging.INFO,
+        "warning": logging.WARNING,
+        "error": logging.ERROR,
+        "critical": logging.CRITICAL
+    }
+    logging.basicConfig(
+        stream=sys.stdout,
+        level=_log_level_map[args.log_level],
+        format="%(asctime)s %(filename)s#%(lineno)d %(levelname)s - %(message)s"
+    )
+
+
+def main(args):
+    logger.info("Validating arguments")
+    validate_args(args)
+
+    logger.info("Initializing hosts")
+    localhost = init_localhost(args.inventory, options={"verbosity": args.verbosity})
+    sonichosts = init_testbed_sonichosts(
+        args.inventory, args.testbed_name, testbed_file=args.tbfile, options={"verbosity": args.verbosity}
+    )
+
+    if not localhost or not sonichosts:
+        sys.exit(RC_INIT_FAILED)
+
+    get_duts_version(sonichosts, args.output)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Tool for getting sonic device version.")
+
+    parser.add_argument(
+        "-i", "--inventory",
+        dest="inventory",
+        nargs="+",
+        help="Ansible inventory file")
+
+    parser.add_argument(
+        "-t", "--testbed-name",
+        type=str,
+        required=True,
+        dest="testbed_name",
+        help="Testbed name."
+    )
+
+    parser.add_argument(
+        "--tbfile",
+        type=str,
+        dest="tbfile",
+        default="testbed.yaml",
+        help="Testbed definition file."
+    )
+
+    parser.add_argument(
+        "-v", "--verbosity",
+        type=int,
+        dest="verbosity",
+        default=2,
+        help="Log verbosity (0-3)."
+    )
+
+    parser.add_argument(
+        "--log-level",
+        type=str,
+        dest="log_level",
+        choices=["debug", "info", "warning", "error", "critical"],
+        default="debug",
+        help="Loglevel"
+    )
+
+    parser.add_argument(
+        "-o", "--output",
+        type=str,
+        dest="output",
+        required=False,
+        help="Output duts version to the specified file."
+    )
+
+    args = parser.parse_args()
+    main(args)

.azure-pipelines/pr_test_scripts.yaml

@@ -2,6 +2,7 @@ t0:
   - arp/test_arp_extended.py
   - arp/test_neighbor_mac.py
   - arp/test_neighbor_mac_noptf.py
+  - bgp/test_bgp_dual_asn.py
   - bgp/test_bgp_fact.py
   - bgp/test_bgp_gr_helper.py::test_bgp_gr_helper_routes_perserved
   - bgp/test_bgp_speaker.py
@@ -28,14 +29,16 @@ t0:
   - generic_config_updater/test_syslog.py
   - generic_config_updater/test_vlan_interface.py
   - generic_config_updater/test_mmu_dynamic_threshold_config_update.py
+  - generic_config_updater/test_ecn_config_update.py
   - iface_namingmode/test_iface_namingmode.py
   - lldp/test_lldp.py
+  - minigraph/test_masked_services.py
   - monit/test_monit_status.py
   - ntp/test_ntp.py
   - override_config_table/test_override_config_table.py
   - pc/test_po_cleanup.py
   - pc/test_po_update.py
-#  - platform_tests/test_advanced_reboot.py::test_warm_reboot
+  - platform_tests/test_advanced_reboot.py::test_warm_reboot
   - platform_tests/test_cpu_memory_usage.py
   - process_monitoring/test_critical_process_monitoring.py
   - radv/test_radv_ipv6_ra.py
@@ -45,6 +48,7 @@ t0:
   - snmp/test_snmp_cpu.py
   - snmp/test_snmp_default_route.py
   - snmp/test_snmp_interfaces.py
+  - snmp/test_snmp_link_local_ip.py
   - snmp/test_snmp_lldp.py
   - snmp/test_snmp_loopback.py
   - snmp/test_snmp_pfc_counters.py
@@ -60,9 +64,12 @@ t0:
   - tacacs/test_ro_user.py
   - tacacs/test_rw_user.py
   - telemetry/test_telemetry.py
+  - telemetry/test_events.py
   - test_features.py
   - test_interfaces.py
   - test_procdockerstatsd.py
+  - database/test_db_scripts.py
+  - system_health/test_watchdog.py
 
 t0-2vlans:
   - dhcp_relay/test_dhcp_relay.py
@@ -75,6 +82,8 @@ t0-sonic:
   - macsec/test_deployment.py
   - macsec/test_fault_handling.py
   - macsec/test_interop_protocol.py
+  - pc/test_retry_count.py
+  - platform_tests/test_advanced_reboot.py::test_warm_reboot
 
 dualtor:
   - arp/test_arp_extended.py
@@ -92,6 +101,7 @@ t1-lag:
   - container_checker/test_container_checker.py
   - generic_config_updater/test_mmu_dynamic_threshold_config_update.py
   - http/test_http_copy.py
+  - iface_namingmode/test_iface_namingmode.py
   - ipfwd/test_mtu.py
   - lldp/test_lldp.py
   - monit/test_monit_status.py
@@ -108,6 +118,7 @@ multi-asic-t1-lag:
   - bgp/test_bgp_fact.py
   - bgp/test_bgp_update_timer.py
   - snmp/test_snmp_default_route.py
+  - snmp/test_snmp_link_local_ip.py
   - snmp/test_snmp_loopback.py
   - snmp/test_snmp_pfc_counters.py
   - snmp/test_snmp_queue.py
@@ -121,7 +132,6 @@ multi-asic-t1-lag:
   - iface_namingmode/test_iface_namingmode.py
   - scp/test_scp_copy.py
   - test_interfaces.py
-  - override_config_table/test_override_config_table.py
   - process_monitoring/test_critical_process_monitoring.py
   - container_checker/test_container_checker.py
   - http/test_http_copy.py
@@ -146,3 +156,17 @@ wan-pub:
   - tacacs/test_ro_disk.py
   - tacacs/test_authorization.py
   - tacacs/test_accounting.py
+
+dpu:
+  - dash/test_dash_vnet.py
+
+specific_param:
+  t0-sonic:
+  - name: bgp/test_bgp_fact.py
+    param: "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"
+#  all the test modules under macsec directory
+  - name: macsec
+    param: "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"
+  dpu:
+  - name: dash/test_dash_vnet.py
+    param: " --skip_dataplane_checking "

.azure-pipelines/pre-commit-check.yml

@@ -58,10 +58,7 @@ steps:
     id:
     comment: |
       The [pre-commit](http://pre-commit.com/) check detected issues in the files touched by this pull request.
-      The detected issues may be old or new. For new issues, please try to fix them.
-
-      For old issues, it is not mandatory to fix them because they were not caused by this change. It is unfair to blame
-      author of this pull request. But if you can take extra effort to fix the old issues as well, that would be great!
+      The pre-commit check is a mandatory check, please fix detected issues.
 
       Detailed pre-commit check results:
       <samp>$(results)</samp>

.azure-pipelines/pytest-collect-only.yml

@@ -0,0 +1,44 @@
+steps:
+
+- script: |
+    sudo apt-get update
+    sudo apt-get install \
+      ca-certificates \
+      curl \
+      gnupg \
+      lsb-release -y
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor \
+      -o /usr/share/keyrings/docker-archive-keyring.gpg
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] \
+      https://download.docker.com/linux/ubuntu \
+      $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+    sudo apt-get update
+    sudo apt-get install docker-ce docker-ce-cli containerd.io -y
+  displayName: 'Install Docker'
+
+- checkout: self
+  clean: true
+  displayName: 'Checkout sonic-mgmt repo'
+
+- script: |
+    set -x
+
+    sudo docker pull sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt:latest
+
+    sudo docker run -dt --name sonic-mgmt-collect \
+      -v $(System.DefaultWorkingDirectory):/var/src/sonic-mgmt \
+      sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt:latest \
+      /bin/bash
+  displayName: 'Prepare sonic-mgmt docker container'
+
+- script: |
+    set -x
+
+    sudo docker exec -t -w /var/src/sonic-mgmt/tests sonic-mgmt-collect \
+      python3 -m pytest --inventory ../ansible/veos_vtb --host-pattern all \
+      --testbed_file vtestbed.yaml --testbed vms-kvm-t0 \
+      --ignore saitests --ignore ptftests --ignore acstests \
+      --ignore scripts --ignore k8s --ignore sai_qualify --ignore common \
+      --ignore-conditional-mark \
+      --color=no --collect-only --continue-on-collection-errors
+  displayName: 'Run pytest --collect-only'