diff --git a/tests/common/plugins/conditional_mark/__init__.py b/tests/common/plugins/conditional_mark/__init__.py index 366d50cf8b..cb7d6b5769 100644 --- a/tests/common/plugins/conditional_mark/__init__.py +++ b/tests/common/plugins/conditional_mark/__init__.py @@ -419,7 +419,7 @@ def find_longest_matches(nodeid, conditions): for condition in conditions: # condition is a dict which has only one item, so we use condition.keys()[0] to get its key. if nodeid.startswith(list(condition.keys())[0]): - length = len(condition) + length = len(list(condition.keys())[0]) if length > max_length: max_length = length longest_matches = [] diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index d24f38615e..bfa71f05c9 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -1284,6 +1284,12 @@ qos/test_qos_sai.py::TestQosSai::testQosSaiDwrrWeightChange: conditions: - "asic_type in ['mellanox']" +qos/test_qos_sai.py::TestQosSai::testQosSaiFullMeshTrafficSanity: + skip: + reason: "Unsupported platform or testbed type." + conditions: + - "asic_type not in ['cisco-8000'] or topo_name not in ['ptf64']" + qos/test_qos_sai.py::TestQosSai::testQosSaiHeadroomPoolSize: skip: reason: "Headroom pool size not supported." diff --git a/tests/qos/qos_sai_base.py b/tests/qos/qos_sai_base.py index 457e7a2f8a..ed4c17573f 100644 --- a/tests/qos/qos_sai_base.py +++ b/tests/qos/qos_sai_base.py @@ -132,13 +132,17 @@ def runPtfTest(self, ptfhost, testCase='', testParams={}, relax=False, pdb=False """ custom_options = " --disable-ipv6 --disable-vxlan --disable-geneve" \ " --disable-erspan --disable-mpls --disable-nvgre" + # Append a suffix to the logfile name if log_suffix is present in testParams + log_suffix = testParams.get("log_suffix", "") + logfile_suffix = "_{0}".format(log_suffix) if log_suffix else "" + ptf_runner( ptfhost, "saitests", testCase, platform_dir="ptftests", params=testParams, - log_file="/tmp/{0}.log".format(testCase), + log_file="/tmp/{0}{1}.log".format(testCase, logfile_suffix), # Include suffix in the logfile name, qlen=10000, is_python3=True, relax=relax, @@ -812,9 +816,32 @@ def __buildPortSpeeds(self, config_facts): port_speeds[attr['speed']].append(etp) return port_speeds + @pytest.fixture(scope='class', autouse=False) + def configure_ip_on_ptf_intfs(self, ptfhost, get_src_dst_asic_and_duts, tbinfo): + src_dut = get_src_dst_asic_and_duts['src_dut'] + src_mgFacts = src_dut.get_extended_minigraph_facts(tbinfo) + topo = tbinfo["topo"]["name"] + + # if PTF64 and is Cisco, set ip IP address on eth interfaces of the ptf" + if topo == 'ptf64' and is_cisco_device(src_dut): + minigraph_ip_interfaces = src_mgFacts['minigraph_interfaces'] + for entry in minigraph_ip_interfaces: + ptfhost.shell("ip addr add {}/31 dev eth{}".format( + entry['peer_addr'], src_mgFacts["minigraph_ptf_indices"][entry['attachto']]) + ) + yield + for entry in minigraph_ip_interfaces: + ptfhost.shell("ip addr del {}/31 dev eth{}".format( + entry['peer_addr'], src_mgFacts["minigraph_ptf_indices"][entry['attachto']]) + ) + return + else: + yield + return + @pytest.fixture(scope='class', autouse=True) def dutConfig( - self, request, duthosts, get_src_dst_asic_and_duts, + self, request, duthosts, configure_ip_on_ptf_intfs, get_src_dst_asic_and_duts, lower_tor_host, tbinfo, dualtor_ports_for_duts, dut_qos_maps): # noqa F811 """ Build DUT host config pertaining to QoS SAI tests @@ -934,7 +961,7 @@ def dutConfig( testPortIds[src_dut_index][src_asic_index] = sorted( list(testPortIps[src_dut_index][src_asic_index].keys())) - elif topo in self.SUPPORTED_T1_TOPOS: + elif topo in self.SUPPORTED_T1_TOPOS or (topo in self.SUPPORTED_PTF_TOPOS and is_cisco_device(src_dut)): # T1 is supported only for 'single_asic' or 'single_dut_multi_asic'. # So use src_dut as the dut use_separated_upkink_dscp_tc_map = separated_dscp_to_tc_map_on_uplink(dut_qos_maps) @@ -2380,6 +2407,67 @@ def populate_arp_entries( ptfhost, testCase=saiQosTest, testParams=testParams ) + @pytest.fixture(scope="function", autouse=False) + def set_static_route_ptf64(self, dutConfig, get_src_dst_asic_and_duts, dutTestParams, enum_frontend_asic_index): + def generate_ip_address(base_ip, new_first_octet): + octets = base_ip.split('.') + if len(octets) != 4: + raise ValueError("Invalid IP address format") + octets[0] = str(new_first_octet) + octets[2] = octets[3] + octets[3] = '1' + return '.'.join(octets) + + def combine_ips(src_ips, dst_ips, new_first_octet): + combined_ips_map = {} + + for key, src_info in src_ips.items(): + src_ip = src_info['peer_addr'] + new_ip = generate_ip_address(src_ip, new_first_octet) + combined_ips_map[key] = {'original_ip': src_ip, 'generated_ip': new_ip} + + for key, dst_info in dst_ips.items(): + dst_ip = dst_info['peer_addr'] + new_ip = generate_ip_address(dst_ip, new_first_octet) + combined_ips_map[key] = {'original_ip': dst_ip, 'generated_ip': new_ip} + + return combined_ips_map + + def configRoutePrefix(add_route): + action = "add" if add_route else "del" + for port, entry in combined_ips_map.items(): + if enum_frontend_asic_index is None: + src_asic.shell("config route {} prefix {}.0/24 nexthop {}".format( + action, '.'.join(entry['generated_ip'].split('.')[:3]), entry['original_ip'])) + else: + src_asic.shell("ip netns exec asic{} config route {} prefix {}.0/24 nexthop {}".format( + enum_frontend_asic_index, + action, '.'.join(entry['generated_ip'].split('.')[:3]), + entry['original_ip']) + ) + + if dutTestParams["basicParams"]["sonic_asic_type"] != "cisco-8000": + pytest.skip("Traffic sanity test is not supported") + + if dutTestParams["topo"] != "ptf64": + pytest.skip("Test not supported in {} topology. Use ptf64 topo".format(dutTestParams["topo"])) + + src_dut_index = get_src_dst_asic_and_duts['src_dut_index'] + dst_dut_index = get_src_dst_asic_and_duts['dst_dut_index'] + src_asic_index = get_src_dst_asic_and_duts['src_asic_index'] + dst_asic_index = get_src_dst_asic_and_duts['dst_asic_index'] + src_asic = get_src_dst_asic_and_duts['src_asic'] + + src_testPortIps = dutConfig["testPortIps"][src_dut_index][src_asic_index] + dst_testPortIps = dutConfig["testPortIps"][dst_dut_index][dst_asic_index] + + new_first_octet = 100 + combined_ips_map = combine_ips(src_testPortIps, dst_testPortIps, new_first_octet) + + configRoutePrefix(True) + yield combined_ips_map + configRoutePrefix(False) + @pytest.fixture(scope="function", autouse=False) def skip_longlink(self, dutQosConfig): portSpeedCableLength = dutQosConfig["portSpeedCableLength"] diff --git a/tests/qos/test_qos_sai.py b/tests/qos/test_qos_sai.py index 01db476c03..c592c9cf50 100644 --- a/tests/qos/test_qos_sai.py +++ b/tests/qos/test_qos_sai.py @@ -2262,3 +2262,89 @@ def testQosSaiLossyQueueVoqMultiSrc( ptfhost, testCase="sai_qos_tests.LossyQueueVoqMultiSrcTest", testParams=testParams ) + + def testQosSaiFullMeshTrafficSanity( + self, ptfhost, dutTestParams, dutConfig, dutQosConfig, + get_src_dst_asic_and_duts, dut_qos_maps, # noqa F811 + set_static_route_ptf64 + ): + """ + Test QoS SAI traffic sanity + Args: + ptfhost (AnsibleHost): Packet Test Framework (PTF) + dutTestParams (Fixture, dict): DUT host test params + dutConfig (Fixture, dict): Map of DUT config containing dut interfaces, test port IDs, test port IPs, + and test ports + dutQosConfig (Fixture, dict): Map containing DUT host QoS configuration + Returns: + None + Raises: + RunAnsibleModuleFail if ptf test fails + """ + # Execution with a specific set of dst port + def run_test_for_dst_port(start, end): + test_params = dict() + test_params.update(dutTestParams["basicParams"]) + test_params.update({ + "testbed_type": dutTestParams["topo"], + "all_src_port_id_to_ip": all_src_port_id_to_ip, + "all_src_port_id_to_name": all_src_port_id_to_name, + "all_dst_port_id_to_ip": {port_id: all_dst_port_id_to_ip[port_id] for port_id in range(start, end)}, + "all_dst_port_id_to_name": {port_id: all_dst_port_id_to_name[port_id] for port_id in range(start, end)}, + "dscp_to_q_map": dscp_to_q_map, + # Add a log_suffix to have separate log and pcap file name + "log_suffix": "_".join([str(port_id) for port_id in range(start, end)]), + "hwsku": dutTestParams['hwsku'] + }) + + self.runPtfTest(ptfhost, testCase="sai_qos_tests.FullMeshTrafficSanity", testParams=test_params) + + src_dut_index = get_src_dst_asic_and_duts['src_dut_index'] + dst_dut_index = get_src_dst_asic_and_duts['dst_dut_index'] + src_asic_index = get_src_dst_asic_and_duts['src_asic_index'] + dst_asic_index = get_src_dst_asic_and_duts['dst_asic_index'] + + src_testPortIps = dutConfig["testPortIps"][src_dut_index][src_asic_index] + dst_testPortIps = dutConfig["testPortIps"][dst_dut_index][dst_asic_index] + + # Fetch all port IDs and IPs + all_src_port_id_to_ip = {port_id: src_testPortIps[port_id]['peer_addr'] for port_id in src_testPortIps.keys()} + + all_src_port_id_to_name = { + port_id: dutConfig["dutInterfaces"][port_id] + for port_id in all_src_port_id_to_ip.keys() + } + + all_dst_port_id_to_ip = { + port_id: set_static_route_ptf64[port_id]['generated_ip'] + for port_id in dst_testPortIps.keys() + } + + all_dst_port_id_to_name = { + port_id: dutConfig["dutInterfaces"][port_id] + for port_id in all_dst_port_id_to_ip.keys() + } + + try: + tc_to_q_map = dut_qos_maps['tc_to_queue_map']['AZURE'] + tc_to_dscp_map = {v: k for k, v in dut_qos_maps['dscp_to_tc_map']['AZURE'].items()} + except KeyError: + pytest.skip( + "Need both TC_TO_PRIORITY_GROUP_MAP and DSCP_TO_TC_MAP" + "and key AZURE to run this test.") + + dscp_to_q_map = {tc_to_dscp_map[tc]: tc_to_q_map[tc] for tc in tc_to_dscp_map if tc != 7} + + # Define the number of splits + # for the dst port list + num_splits = 4 + + # Get all keys and sort them + all_keys = sorted(all_dst_port_id_to_ip.keys()) + + # Calculate the split points + split_points = [all_keys[i * len(all_keys) // num_splits] for i in range(1, num_splits)] + + # Execute with one set of dst port at a time, avoids ptf run getting timed out + for start, end in zip([0] + split_points, split_points + [len(all_keys)]): + run_test_for_dst_port(start, end) diff --git a/tests/saitests/py3/sai_qos_tests.py b/tests/saitests/py3/sai_qos_tests.py index e31050c633..7c31865bba 100755 --- a/tests/saitests/py3/sai_qos_tests.py +++ b/tests/saitests/py3/sai_qos_tests.py @@ -12,6 +12,7 @@ import texttable import math import os +import concurrent.futures from ptf.testutils import (ptf_ports, dp_poll, simple_arp_packet, @@ -311,10 +312,10 @@ def get_ip_addr(): def get_tcp_port(): val = 1234 while True: - if val == 65535: - raise RuntimeError("We ran out of tcp ports!") - val = max(val, (val+10) % 65535) yield val + val += 10 + if val > 65534: + val = 1234 TCP_PORT_GEN = get_tcp_port() @@ -366,7 +367,8 @@ def construct_tcp_pkt(pkt_len, dst_mac, src_mac, src_ip, dst_ip, dscp, src_vlan, return pkt -def get_multiple_flows(dp, dst_mac, dst_id, dst_ip, src_vlan, dscp, ecn, ttl, pkt_len, src_details, packets_per_port=1): +def get_multiple_flows(dp, dst_mac, dst_id, dst_ip, src_vlan, dscp, ecn, ttl, + pkt_len, src_details, packets_per_port=1, check_actual_dst_id=True): ''' Returns a dict of format: src_id : [list of (pkt, exp_pkt) pairs that go to the given dst_id] @@ -421,7 +423,10 @@ def get_rx_port_pkt(dp, src_port_id, pkt, exp_pkt): all_pkts[src_tuple[0]] except KeyError: all_pkts[src_tuple[0]] = [] - actual_dst_id = get_rx_port_pkt(dp, src_tuple[0], pkt, masked_exp_pkt) + if check_actual_dst_id is False: + actual_dst_id = dst_id + else: + actual_dst_id = get_rx_port_pkt(dp, src_tuple[0], pkt, masked_exp_pkt) if actual_dst_id == dst_id: all_pkts[src_tuple[0]].append((pkt, masked_exp_pkt, dst_id)) num_of_pkts += 1 @@ -5870,3 +5875,180 @@ def runTest(self): print("Successfully dropped {} packets".format(drops), file=sys.stderr) finally: self.sai_thrift_port_tx_enable(self.dst_client, self.asic_type, [self.dst_port_id]) + + +class FullMeshTrafficSanity(sai_base_test.ThriftInterfaceDataPlane): + def setUp(self): + sai_base_test.ThriftInterfaceDataPlane.setUp(self) + time.sleep(5) + switch_init(self.clients) + + # Parse input parameters + self.testbed_type = self.test_params['testbed_type'] + self.router_mac = self.test_params['router_mac'] + self.sonic_version = self.test_params['sonic_version'] + + dscp_to_q_map = self.test_params['dscp_to_q_map'] + self.dscps = [int(key) for key in dscp_to_q_map.keys()] + self.queues = [int(value) for value in dscp_to_q_map.values()] + self.all_src_port_id_to_ip = self.test_params['all_src_port_id_to_ip'] + self.all_src_port_id_to_name = self.test_params['all_src_port_id_to_name'] + self.all_dst_port_id_to_ip = self.test_params['all_dst_port_id_to_ip'] + self.all_dst_port_id_to_name = self.test_params['all_dst_port_id_to_name'] + + self.all_port_id_to_ip = dict() + self.all_port_id_to_ip.update(self.all_src_port_id_to_ip) + self.all_port_id_to_ip.update(self.all_dst_port_id_to_ip) + + self.all_port_id_to_name = dict() + self.all_port_id_to_name.update(self.all_src_port_id_to_name) + self.all_port_id_to_name.update(self.all_dst_port_id_to_name) + + self.src_port_ids = list(self.all_src_port_id_to_ip.keys()) + self.dst_port_ids = list(self.all_dst_port_id_to_ip.keys()) + self.all_port_ids = self.src_port_ids + list(set(self.dst_port_ids) - set(self.src_port_ids)) + + self.asic_type = self.test_params['sonic_asic_type'] + self.packet_size = 100 + logging.info("Using packet size", self.packet_size) + self.flows_per_port = 6 + + self.all_port_id_to_mac = {port_id: self.dataplane.get_mac(0, port_id) + for port_id in self.all_port_id_to_ip.keys()} + + def tearDown(self): + sai_base_test.ThriftInterfaceDataPlane.tearDown(self) + + def config_traffic(self, dst_port_id, dscp, ecn_bit): + if type(ecn_bit) == bool: + ecn_bit = 1 if ecn_bit else 0 + self.dscp = dscp + self.dst_port_id = dst_port_id + self.tos = (dscp << 2) | ecn_bit + self.ttl = 64 + logging.debug("Getting multiple flows to {:>2}, dscp={}, dst_ip={}".format( + self.dst_port_id, self.dscp, self.dst_port_ip) + ) + self.pkt = get_multiple_flows( + self, + self.dst_port_mac, + dst_port_id, + self.dst_port_ip, + None, + dscp, + ecn_bit, + 64, + self.packet_size, + [(src_port_id, src_port_ip) for src_port_id, src_port_ip in self.all_src_port_id_to_ip.items()], + self.flows_per_port, + False) + logging.debug("Got multiple flows to {:>2}, dscp={}, dst_ip={}".format( + self.dst_port_id, self.dscp, self.dst_port_ip) + ) + + def runTest(self): + failed_pairs = set() + logging.info("Total traffic src_dst_pairs being tested {}".format( + len(self.src_port_ids)*len(self.dst_port_ids)) + ) + pkt_count = 10 + + # Split the src port list for concurrent pkt injection + num_splits = 2 + split_points = [i * len(self.src_port_ids) // num_splits for i in range(1, num_splits)] + parts = [self.src_port_ids[i:j] for i, j in zip([0] + split_points, split_points + [None])] + + def runTestPerSrcList(src_port_list, checkCounter=False): + for src_port_id in src_port_list: + logging.debug( + "Sending {} packets X {} flows with dscp/queue {}/{} from src {} -> dst {}".format( + pkt_count, + len(self.pkt[src_port_id]), dscp, queue, + self.all_port_id_to_name.get(src_port_id, 'Not Found'), + dst_port_name) + ) + if checkCounter: + port_cnt_base, q_cntrs_base = sai_thrift_read_port_counters( + self.dst_client, self.asic_type, + port_list['dst'][real_dst_port_id] + ) + + for pkt_tuple in self.pkt[src_port_id]: + logging.debug( + "Sending {} packets with dscp/queue {}/{} from src {} -> dst {} Pkt {}".format( + pkt_count, dscp, queue, + self.all_port_id_to_name.get(src_port_id, 'Not Found'), + dst_port_name, pkt_tuple[0]) + ) + send_packet(self, src_port_id, pkt_tuple[0], pkt_count) + + if checkCounter: + time.sleep(1) + port_cntrs, q_cntrs = sai_thrift_read_port_counters( + self.dst_client, self.asic_type, + port_list['dst'][real_dst_port_id] + ) + pkts_enqueued = q_cntrs[queue] - q_cntrs_base[queue] + if pkts_enqueued < self.flows_per_port*pkt_count: + logging.info("Faulty src/dst {}/{} pair on queue {}".format( + self.all_port_id_to_name.get(src_port_id, 'Not Found'), + dst_port_name, queue + )) + logging.info("q_cntrs_base {}".format(q_cntrs_base)) + logging.info("q_cntrs {}".format(q_cntrs)) + logging.info("port_cnt_base {}".format(port_cnt_base)) + logging.info("port_cntrs {}".format(port_cntrs)) + failed_pairs.add( + ( + self.all_port_id_to_name.get(src_port_id, 'Not Found'), + dst_port_name, queue + ) + ) + + def findFaultySrcDstPair(dscp, queue): + ecn_bit = 1 if queue in [3, 4] else 0 + self.config_traffic(real_dst_port_id, dscp, ecn_bit) + runTestPerSrcList(self.src_port_ids, True) + + for dst_port_id in self.dst_port_ids: + real_dst_port_id = dst_port_id + dst_port_name = self.all_port_id_to_name.get(real_dst_port_id, 'Not Found') + logging.info("Starting Test for dst {}".format(dst_port_name)) + dst_port_mac = self.all_port_id_to_mac[real_dst_port_id] + self.dst_port_mac = self.router_mac if self.router_mac != '' else dst_port_mac + self.dst_port_ip = self.all_port_id_to_ip[real_dst_port_id] + + for i, dscp in enumerate(self.dscps): + queue = self.queues[i] # Need queue for occupancy verification + ecn_bit = 1 if queue in [3, 4] else 0 + self.config_traffic(real_dst_port_id, dscp, ecn_bit) + + port_cnt_base, q_cntrs_base = sai_thrift_read_port_counters( + self.dst_client, self.asic_type, + port_list['dst'][real_dst_port_id] + ) + + with concurrent.futures.ThreadPoolExecutor(max_workers=num_splits) as executor: + # Submit the tasks to the executor + futures = [executor.submit(runTestPerSrcList, part) for part in parts] + + # Wait for all tasks to complete + concurrent.futures.wait(futures) + + time.sleep(1) + port_cntrs, q_cntrs = sai_thrift_read_port_counters( + self.dst_client, self.asic_type, + port_list['dst'][real_dst_port_id] + ) + pkts_enqueued = q_cntrs[queue] - q_cntrs_base[queue] + logging.info("Enqueued on queue {} pkts {}".format(queue, pkts_enqueued)) + if pkts_enqueued < self.flows_per_port*pkt_count*len(self.src_port_ids): + logging.info("q_cntrs_base {}".format(q_cntrs_base)) + logging.info("q_cntrs {}".format(q_cntrs)) + logging.info("port_cnt_base {}".format(port_cnt_base)) + logging.info("port_cntrs {}".format(port_cntrs)) + # Craft pkt for given queue and + # inject from each src to find which src/dst pair is dropping pkt + findFaultySrcDstPair(dscp, queue) + + assert len(failed_pairs) == 0, "Traffic failed between {}".format(failed_pairs)