Skip to content

Commit

Permalink
[drop_packets]: Fix to support testbed with different (#9262)
Browse files Browse the repository at this point in the history 
What is the motivation for this PR?
Seeing failures on testbed with different hwsku's with different asic count.
In Chassis testbed, with different LC hwSKU , if one of the selected LCs have 3 asics and another LC has 2 asics, we see the below failure:

drop_packets/test_drop_counters.py::test_src_ip_is_loopback_addr[rif_members] 
-------------------------------- live log call ---------------------------------
02:08:08 utilities.wait_until                     L0118 ERROR  | Exception caught while checking <lambda>:Traceback (most recent call last):
  File "/./tests/common/utilities.py", line 112, in wait_until
    check_result = condition(*args, **kwargs)
  File "/./tests/common/helpers/drop_counters/drop_counters.py", line 101, in <lambda>
    check_drops_on_dut = lambda: packets_count in get_drops_across_all_duthosts()
  File "/./tests/common/helpers/drop_counters/drop_counters.py", line 94, in get_drops_across_all_duthosts
    pkt_drops = get_pkt_drops(duthost, get_cnt_cli_cmd, asic_index)
  File "/./tests/common/helpers/drop_counters/drop_counters.py", line 34, in get_pkt_drops
    namespace = duthost.get_namespace_from_asic_id(asic_index)
  File "/./tests/common/devices/multi_asic.py", line 230, in get_namespace_from_asic_id
    raise ValueError("Invalid asic_id '{}' passed as input".format(asic_id))
ValueError: Invalid asic_id '2' passed as input
, error:Invalid asic_id '2' passed as input
Issue is because the get_pkt_drop function takes the asic_index of the selected HwSKU or selected LC.
If the random asic_index selected is 2, and if get_pkt_drops() is called with asic_index 2 for a different LC in the testbed with 2 asics, the test fails as asic_index 2 is not present in the different LC.

How did you do it?
Fixed get_pkt_drops to go through all asic namespaces and get drop count.
This way get_pkt_drops will return packet drops for all interfaces from all namespaces.

How did you verify/test it?
Verified on packet chassis, voq chassis and single asic DUT testbed.
  • Loading branch information
@SuvarnaMeenakshi
SuvarnaMeenakshi authored Aug 17, 2023
1 parent c5bfe11 commit f27755d
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 36 deletions.
64 changes: 35 additions & 29 deletions tests/common/helpers/drop_counters/drop_counters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,40 +23,46 @@
COMBINED_ACL_DROP_COUNTER = False


def get_pkt_drops(duthost, cli_cmd, asic_index):
def get_pkt_drops(duthost, cli_cmd, asic_index=None):
"""
@summary: Parse output of "portstat" or "intfstat" commands and convert it to the dictionary.
@param module: The AnsibleModule object
@param cli_cmd: one of supported CLI commands - "portstat -j" or "intfstat -j"
@return: Return dictionary of parsed counters
"""
# Get namespace from asic_index.
namespace = duthost.get_namespace_from_asic_id(asic_index)

# Frame the correct cli command
# the L2 commands need _SUFFIX and L3 commands need _PREFIX
if cli_cmd == GET_L3_COUNTERS:
CMD_PREFIX = NAMESPACE_PREFIX if (namespace is not None and duthost.is_multi_asic) else ''
cli_cmd = CMD_PREFIX + cli_cmd
elif cli_cmd == GET_L2_COUNTERS:
CMD_SUFFIX = NAMESPACE_SUFFIX if (namespace is not None and duthost.is_multi_asic) else ''
cli_cmd = cli_cmd + CMD_SUFFIX

stdout = duthost.command(cli_cmd.format(namespace))
stdout = stdout["stdout"]
match = re.search("Last cached time was.*\n", stdout)
if match:
stdout = re.sub("Last cached time was.*\n", "", stdout)

try:
return json.loads(stdout)
except Exception as err:
raise Exception("Failed to parse output of '{}', err={}".format(cli_cmd, str(err)))


def ensure_no_l3_drops(duthost, asic_index, packets_count):
result = {}
for asic_id in duthost.get_asic_ids():
if asic_index is not None and asic_index != asic_id:
continue
namespace = duthost.get_namespace_from_asic_id(asic_id)

# Frame the correct cli command
# the L2 commands need _SUFFIX and L3 commands need _PREFIX
if cli_cmd == GET_L3_COUNTERS:
CMD_PREFIX = NAMESPACE_PREFIX if (namespace is not None and duthost.is_multi_asic) else ''
cli_cmd = CMD_PREFIX + cli_cmd
elif cli_cmd == GET_L2_COUNTERS:
CMD_SUFFIX = NAMESPACE_SUFFIX if (namespace is not None and duthost.is_multi_asic) else ''
cli_cmd = cli_cmd + CMD_SUFFIX

stdout = duthost.command(cli_cmd.format(namespace))
stdout = stdout["stdout"]
match = re.search("Last cached time was.*\n", stdout)
if match:
stdout = re.sub("Last cached time was.*\n", "", stdout)

try:
namespace_result = json.loads(stdout)
result.update(namespace_result)
except Exception as err:
raise Exception("Failed to parse output of '{}', err={}".format(cli_cmd, str(err)))
return result


def ensure_no_l3_drops(duthost, packets_count):
""" Verify L3 drop counters were not incremented """
intf_l3_counters = get_pkt_drops(duthost, GET_L3_COUNTERS, asic_index)
intf_l3_counters = get_pkt_drops(duthost, GET_L3_COUNTERS)
unexpected_drops = {}
for iface, value in list(intf_l3_counters.items()):
try:
Expand All @@ -71,9 +77,9 @@ def ensure_no_l3_drops(duthost, asic_index, packets_count):
pytest.fail("L3 'RX_ERR' was incremented for the following interfaces:\n{}".format(unexpected_drops))


def ensure_no_l2_drops(duthost, asic_index, packets_count):
def ensure_no_l2_drops(duthost, packets_count):
""" Verify L2 drop counters were not incremented """
intf_l2_counters = get_pkt_drops(duthost, GET_L2_COUNTERS, asic_index)
intf_l2_counters = get_pkt_drops(duthost, GET_L2_COUNTERS)
unexpected_drops = {}
for iface, value in list(intf_l2_counters.items()):
try:
Expand All @@ -92,7 +98,7 @@ def verify_drop_counters(duthosts, asic_index, dut_iface, get_cnt_cli_cmd, colum
def _get_drops_across_all_duthosts():
drop_list = []
for duthost in duthosts.frontend_nodes:
pkt_drops = get_pkt_drops(duthost, get_cnt_cli_cmd, asic_index)
pkt_drops = get_pkt_drops(duthost, get_cnt_cli_cmd)
# we cannot assume the iface name will be same on all the devices for SONiC chassis
# if the dut_iface is not found ignore this device
if dut_iface not in pkt_drops:
Expand Down
14 changes: 7 additions & 7 deletions tests/drop_packets/test_drop_counters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,21 +119,21 @@ def base_verification(discard_group, pkt, ptfadapter, duthosts, asic_index, port
verify_drop_counters(duthosts, asic_index, ports_info["dut_iface"],
GET_L2_COUNTERS, L2_COL_KEY, packets_count=PKT_NUMBER)
for duthost in duthosts.frontend_nodes:
ensure_no_l3_drops(duthost, asic_index, packets_count=PKT_NUMBER)
ensure_no_l3_drops(duthost, packets_count=PKT_NUMBER)
elif discard_group == "L3":
if COMBINED_L2L3_DROP_COUNTER:
verify_drop_counters(duthosts, asic_index, ports_info["dut_iface"],
GET_L2_COUNTERS, L2_COL_KEY, packets_count=PKT_NUMBER)
for duthost in duthosts.frontend_nodes:
ensure_no_l3_drops(duthost, asic_index, packets_count=PKT_NUMBER)
ensure_no_l3_drops(duthost, packets_count=PKT_NUMBER)
else:
if not tx_dut_ports:
pytest.fail("No L3 interface specified")

verify_drop_counters(duthosts, asic_index, tx_dut_ports[ports_info["dut_iface"]],
GET_L3_COUNTERS, L3_COL_KEY, packets_count=PKT_NUMBER)
for duthost in duthosts.frontend_nodes:
ensure_no_l2_drops(duthost, asic_index, packets_count=PKT_NUMBER)
ensure_no_l2_drops(duthost, packets_count=PKT_NUMBER)
elif discard_group == "ACL":
if not tx_dut_ports:
pytest.fail("No L3 interface specified")
Expand All @@ -154,12 +154,12 @@ def base_verification(discard_group, pkt, ptfadapter, duthosts, asic_index, port
pytest.fail(fail_msg)
if not COMBINED_ACL_DROP_COUNTER:
for duthost in duthosts.frontend_nodes:
ensure_no_l3_drops(duthost, asic_index, packets_count=PKT_NUMBER)
ensure_no_l2_drops(duthost, asic_index, packets_count=PKT_NUMBER)
ensure_no_l3_drops(duthost, packets_count=PKT_NUMBER)
ensure_no_l2_drops(duthost, packets_count=PKT_NUMBER)
elif discard_group == "NO_DROPS":
for duthost in duthosts.frontend_nodes:
ensure_no_l2_drops(duthost, asic_index, packets_count=PKT_NUMBER)
ensure_no_l3_drops(duthost, asic_index, packets_count=PKT_NUMBER)
ensure_no_l2_drops(duthost, packets_count=PKT_NUMBER)
ensure_no_l3_drops(duthost, packets_count=PKT_NUMBER)
else:
pytest.fail("Incorrect 'discard_group' specified. Supported values: 'L2', 'L3', 'ACL' or 'NO_DROPS'")

Expand Down

0 comments on commit f27755d

Please sign in to comment.