From f27755dd99192b7150625b02a33abc50acf0112c Mon Sep 17 00:00:00 2001 From: SuvarnaMeenakshi <50386592+SuvarnaMeenakshi@users.noreply.github.com> Date: Thu, 17 Aug 2023 12:56:20 -0700 Subject: [PATCH] [drop_packets]: Fix to support testbed with different (#9262) What is the motivation for this PR? Seeing failures on testbed with different hwsku's with different asic count. In Chassis testbed, with different LC hwSKU , if one of the selected LCs have 3 asics and another LC has 2 asics, we see the below failure: drop_packets/test_drop_counters.py::test_src_ip_is_loopback_addr[rif_members] -------------------------------- live log call --------------------------------- 02:08:08 utilities.wait_until L0118 ERROR | Exception caught while checking :Traceback (most recent call last): File "/./tests/common/utilities.py", line 112, in wait_until check_result = condition(*args, **kwargs) File "/./tests/common/helpers/drop_counters/drop_counters.py", line 101, in check_drops_on_dut = lambda: packets_count in get_drops_across_all_duthosts() File "/./tests/common/helpers/drop_counters/drop_counters.py", line 94, in get_drops_across_all_duthosts pkt_drops = get_pkt_drops(duthost, get_cnt_cli_cmd, asic_index) File "/./tests/common/helpers/drop_counters/drop_counters.py", line 34, in get_pkt_drops namespace = duthost.get_namespace_from_asic_id(asic_index) File "/./tests/common/devices/multi_asic.py", line 230, in get_namespace_from_asic_id raise ValueError("Invalid asic_id '{}' passed as input".format(asic_id)) ValueError: Invalid asic_id '2' passed as input , error:Invalid asic_id '2' passed as input Issue is because the get_pkt_drop function takes the asic_index of the selected HwSKU or selected LC. If the random asic_index selected is 2, and if get_pkt_drops() is called with asic_index 2 for a different LC in the testbed with 2 asics, the test fails as asic_index 2 is not present in the different LC. How did you do it? Fixed get_pkt_drops to go through all asic namespaces and get drop count. This way get_pkt_drops will return packet drops for all interfaces from all namespaces. How did you verify/test it? Verified on packet chassis, voq chassis and single asic DUT testbed. --- .../helpers/drop_counters/drop_counters.py | 64 ++++++++++--------- tests/drop_packets/test_drop_counters.py | 14 ++-- 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/tests/common/helpers/drop_counters/drop_counters.py b/tests/common/helpers/drop_counters/drop_counters.py index b19dd9b9a1..40a82d69bd 100644 --- a/tests/common/helpers/drop_counters/drop_counters.py +++ b/tests/common/helpers/drop_counters/drop_counters.py @@ -23,7 +23,7 @@ COMBINED_ACL_DROP_COUNTER = False -def get_pkt_drops(duthost, cli_cmd, asic_index): +def get_pkt_drops(duthost, cli_cmd, asic_index=None): """ @summary: Parse output of "portstat" or "intfstat" commands and convert it to the dictionary. @param module: The AnsibleModule object @@ -31,32 +31,38 @@ def get_pkt_drops(duthost, cli_cmd, asic_index): @return: Return dictionary of parsed counters """ # Get namespace from asic_index. - namespace = duthost.get_namespace_from_asic_id(asic_index) - - # Frame the correct cli command - # the L2 commands need _SUFFIX and L3 commands need _PREFIX - if cli_cmd == GET_L3_COUNTERS: - CMD_PREFIX = NAMESPACE_PREFIX if (namespace is not None and duthost.is_multi_asic) else '' - cli_cmd = CMD_PREFIX + cli_cmd - elif cli_cmd == GET_L2_COUNTERS: - CMD_SUFFIX = NAMESPACE_SUFFIX if (namespace is not None and duthost.is_multi_asic) else '' - cli_cmd = cli_cmd + CMD_SUFFIX - - stdout = duthost.command(cli_cmd.format(namespace)) - stdout = stdout["stdout"] - match = re.search("Last cached time was.*\n", stdout) - if match: - stdout = re.sub("Last cached time was.*\n", "", stdout) - - try: - return json.loads(stdout) - except Exception as err: - raise Exception("Failed to parse output of '{}', err={}".format(cli_cmd, str(err))) - - -def ensure_no_l3_drops(duthost, asic_index, packets_count): + result = {} + for asic_id in duthost.get_asic_ids(): + if asic_index is not None and asic_index != asic_id: + continue + namespace = duthost.get_namespace_from_asic_id(asic_id) + + # Frame the correct cli command + # the L2 commands need _SUFFIX and L3 commands need _PREFIX + if cli_cmd == GET_L3_COUNTERS: + CMD_PREFIX = NAMESPACE_PREFIX if (namespace is not None and duthost.is_multi_asic) else '' + cli_cmd = CMD_PREFIX + cli_cmd + elif cli_cmd == GET_L2_COUNTERS: + CMD_SUFFIX = NAMESPACE_SUFFIX if (namespace is not None and duthost.is_multi_asic) else '' + cli_cmd = cli_cmd + CMD_SUFFIX + + stdout = duthost.command(cli_cmd.format(namespace)) + stdout = stdout["stdout"] + match = re.search("Last cached time was.*\n", stdout) + if match: + stdout = re.sub("Last cached time was.*\n", "", stdout) + + try: + namespace_result = json.loads(stdout) + result.update(namespace_result) + except Exception as err: + raise Exception("Failed to parse output of '{}', err={}".format(cli_cmd, str(err))) + return result + + +def ensure_no_l3_drops(duthost, packets_count): """ Verify L3 drop counters were not incremented """ - intf_l3_counters = get_pkt_drops(duthost, GET_L3_COUNTERS, asic_index) + intf_l3_counters = get_pkt_drops(duthost, GET_L3_COUNTERS) unexpected_drops = {} for iface, value in list(intf_l3_counters.items()): try: @@ -71,9 +77,9 @@ def ensure_no_l3_drops(duthost, asic_index, packets_count): pytest.fail("L3 'RX_ERR' was incremented for the following interfaces:\n{}".format(unexpected_drops)) -def ensure_no_l2_drops(duthost, asic_index, packets_count): +def ensure_no_l2_drops(duthost, packets_count): """ Verify L2 drop counters were not incremented """ - intf_l2_counters = get_pkt_drops(duthost, GET_L2_COUNTERS, asic_index) + intf_l2_counters = get_pkt_drops(duthost, GET_L2_COUNTERS) unexpected_drops = {} for iface, value in list(intf_l2_counters.items()): try: @@ -92,7 +98,7 @@ def verify_drop_counters(duthosts, asic_index, dut_iface, get_cnt_cli_cmd, colum def _get_drops_across_all_duthosts(): drop_list = [] for duthost in duthosts.frontend_nodes: - pkt_drops = get_pkt_drops(duthost, get_cnt_cli_cmd, asic_index) + pkt_drops = get_pkt_drops(duthost, get_cnt_cli_cmd) # we cannot assume the iface name will be same on all the devices for SONiC chassis # if the dut_iface is not found ignore this device if dut_iface not in pkt_drops: diff --git a/tests/drop_packets/test_drop_counters.py b/tests/drop_packets/test_drop_counters.py index bab671071c..d302b75149 100755 --- a/tests/drop_packets/test_drop_counters.py +++ b/tests/drop_packets/test_drop_counters.py @@ -119,13 +119,13 @@ def base_verification(discard_group, pkt, ptfadapter, duthosts, asic_index, port verify_drop_counters(duthosts, asic_index, ports_info["dut_iface"], GET_L2_COUNTERS, L2_COL_KEY, packets_count=PKT_NUMBER) for duthost in duthosts.frontend_nodes: - ensure_no_l3_drops(duthost, asic_index, packets_count=PKT_NUMBER) + ensure_no_l3_drops(duthost, packets_count=PKT_NUMBER) elif discard_group == "L3": if COMBINED_L2L3_DROP_COUNTER: verify_drop_counters(duthosts, asic_index, ports_info["dut_iface"], GET_L2_COUNTERS, L2_COL_KEY, packets_count=PKT_NUMBER) for duthost in duthosts.frontend_nodes: - ensure_no_l3_drops(duthost, asic_index, packets_count=PKT_NUMBER) + ensure_no_l3_drops(duthost, packets_count=PKT_NUMBER) else: if not tx_dut_ports: pytest.fail("No L3 interface specified") @@ -133,7 +133,7 @@ def base_verification(discard_group, pkt, ptfadapter, duthosts, asic_index, port verify_drop_counters(duthosts, asic_index, tx_dut_ports[ports_info["dut_iface"]], GET_L3_COUNTERS, L3_COL_KEY, packets_count=PKT_NUMBER) for duthost in duthosts.frontend_nodes: - ensure_no_l2_drops(duthost, asic_index, packets_count=PKT_NUMBER) + ensure_no_l2_drops(duthost, packets_count=PKT_NUMBER) elif discard_group == "ACL": if not tx_dut_ports: pytest.fail("No L3 interface specified") @@ -154,12 +154,12 @@ def base_verification(discard_group, pkt, ptfadapter, duthosts, asic_index, port pytest.fail(fail_msg) if not COMBINED_ACL_DROP_COUNTER: for duthost in duthosts.frontend_nodes: - ensure_no_l3_drops(duthost, asic_index, packets_count=PKT_NUMBER) - ensure_no_l2_drops(duthost, asic_index, packets_count=PKT_NUMBER) + ensure_no_l3_drops(duthost, packets_count=PKT_NUMBER) + ensure_no_l2_drops(duthost, packets_count=PKT_NUMBER) elif discard_group == "NO_DROPS": for duthost in duthosts.frontend_nodes: - ensure_no_l2_drops(duthost, asic_index, packets_count=PKT_NUMBER) - ensure_no_l3_drops(duthost, asic_index, packets_count=PKT_NUMBER) + ensure_no_l2_drops(duthost, packets_count=PKT_NUMBER) + ensure_no_l3_drops(duthost, packets_count=PKT_NUMBER) else: pytest.fail("Incorrect 'discard_group' specified. Supported values: 'L2', 'L3', 'ACL' or 'NO_DROPS'")