Minigraph ECMP parsing support (cleaner format)

Author: kktheballer

src/sonic-config-engine/minigraph.py

@@ -10,6 +10,7 @@
 from lxml import etree as ET
 from lxml.etree import QName
 
+
 from portconfig import get_port_config
 from sonic_py_common.multi_asic import get_asic_id_from_name
 from sonic_py_common.interface import backplane_prefix
@@ -105,7 +106,73 @@ def parse_device(device):
 
     return (lo_prefix, lo_prefix_v6, mgmt_prefix, name, hwsku, d_type, deployment_id)
 
-def parse_png(png, hname):
+def calculate_lcm_for_ecmp (nhdevices_bank_map, nhip_bank_map):
+    banks_enumerated = {}
+    lcm_array = []
+    for value in nhdevices_bank_map.values():
+        for key in nhip_bank_map.keys():
+            if nhip_bank_map[key] == value:
+                if value not in banks_enumerated:
+                    banks_enumerated[value] = 1
+                else:
+                    banks_enumerated[value] = banks_enumerated[value] + 1
+    for bank_enumeration in banks_enumerated.values():
+        lcm_list = range(1, bank_enumeration+1)
+        lcm_comp = lcm_list[0]
+        for i in lcm_list[1:]:
+            lcm_comp = lcm_comp * i / calculate_gcd(lcm_comp, i)
+        lcm_array.append(lcm_comp)
+
+    LCM = sum(lcm_array)
+    return int(LCM)
+
+def calculate_gcd(x, y):
+    while y != 0:
+        (x, y) = (y, x % y)
+    return int(x)
+
+def formulate_fine_grained_ecmp(version, dpg_ecmp_content, port_device_map, port_alias_map):
+    family = ""
+    tag = ""
+    neigh_key = []
+    if version == "ipv4":
+        family = "IPV4"
+        tag = "fgnhg_v4"
+    elif version == "ipv6":
+        family = "IPV6"
+        tag = "fgnhg_v6"
+
+    port_nhip_map = dpg_ecmp_content['port_nhip_map']
+    nhgaddr = dpg_ecmp_content['nhgaddr']
+    nhg_int = dpg_ecmp_content['nhg_int']
+
+    nhip_device_map = {port_nhip_map[x]: port_device_map[x] for x in port_device_map
+                         if x in port_nhip_map}
+    nhip_devices = sorted(list(set(nhip_device_map.values())))
+    nhdevices_ip_bank_map = {device: bank for bank, device in enumerate(nhip_devices)}
+    nhip_bank_map = {ip: nhdevices_ip_bank_map[device] for ip, device in nhip_device_map.items()}
+    LCM = calculate_lcm_for_ecmp(nhdevices_ip_bank_map, nhip_bank_map)
+
+    FG_NHG_MEMBER = {ip: {"FG_NHG": tag, "bank": bank} for ip, bank in nhip_bank_map.items()}
+    nhip_port_map = dict(zip(port_nhip_map.values(), port_nhip_map.keys()))
+
+    
+
+    for nhip, memberinfo in FG_NHG_MEMBER.items():
+        if nhip in nhip_port_map:
+            memberinfo["link"] = port_alias_map[nhip_port_map[nhip]]
+            FG_NHG_MEMBER[nhip] = memberinfo
+
+    FG_NHG_PREFIX = {nhgaddr: {"FG_NHG": tag}}
+    FG_NHG = {tag: {"bucket_size": LCM}}
+    for ip in nhip_bank_map:
+        neigh_key.append(str(nhg_int + "|" + ip))
+    NEIGH = {neigh_key: {"family": family} for neigh_key in neigh_key}
+
+    fine_grained_content = {"FG_NHG_MEMBER": FG_NHG_MEMBER, "FG_NHG": FG_NHG, "FG_NHG_PREFIX": FG_NHG_PREFIX, "NEIGH": NEIGH}
+    return fine_grained_content
+
+def parse_png(png, hname, dpg_ecmp_content = None):
     neighbors = {}
     devices = {}
     console_dev = ''
@@ -116,6 +183,13 @@ def parse_png(png, hname):
     console_ports = {}
     mux_cable_ports = {}
     is_storage_device = False
+    port_device_map = {}
+    png_ecmp_content = {}
+    FG_NHG_MEMBER = {}
+    FG_NHG_PREFIX = {}
+    FG_NHG = {}
+    NEIGH = {}
+
     for child in png:
         if child.tag == str(QName(ns, "DeviceInterfaceLinks")):
             for link in child.findall(str(QName(ns, "DeviceLinkBase"))):
@@ -141,6 +215,11 @@ def parse_png(png, hname):
                             }
                     continue
 
+                if linktype == "DeviceInterfaceLink":
+                    endport = link.find(str(QName(ns, "EndPort"))).text
+                    startdevice = link.find(str(QName(ns, "StartDevice"))).text
+                    port_device_map[endport] = startdevice
+
                 if linktype != "DeviceInterfaceLink" and linktype != "UnderlayInterfaceLink":
                     continue
 
@@ -196,6 +275,7 @@ def parse_png(png, hname):
                             elif node.tag == str(QName(ns, "EndDevice")):
                                 mgmt_dev = node.text
 
+
         if child.tag == str(QName(ns, "DeviceInterfaceLinks")):
             for link in child.findall(str(QName(ns, 'DeviceLinkBase'))):
                 if link.find(str(QName(ns, "ElementType"))).text == "LogicalLink":
@@ -205,7 +285,19 @@ def parse_png(png, hname):
 
                     mux_cable_ports[intf_name] = "true"
 
-    return (neighbors, devices, console_dev, console_port, mgmt_dev, mgmt_port, port_speeds, console_ports, mux_cable_ports, is_storage_device)
+        if (len(dpg_ecmp_content)):
+            for version, content in dpg_ecmp_content.items():  # version is ipv4 or ipv6
+                fine_grained_content = formulate_fine_grained_ecmp(version, content, port_device_map, port_alias_map)  # port_alias_map
+                FG_NHG_MEMBER.update(fine_grained_content['FG_NHG_MEMBER'])
+                FG_NHG_PREFIX.update(fine_grained_content['FG_NHG_PREFIX'])
+                FG_NHG.update(fine_grained_content['FG_NHG'])
+                NEIGH.update(fine_grained_content['NEIGH'])
+
+            png_ecmp_content = {"FG_NHG_PREFIX": FG_NHG_PREFIX, "FG_NHG_MEMBER": FG_NHG_MEMBER, "FG_NHG": FG_NHG,
+                            "NEIGH": NEIGH}
+
+    return (neighbors, devices, console_dev, console_port, mgmt_dev, mgmt_port, port_speeds, console_ports, mux_cable_ports, is_storage_device, png_ecmp_content)
+
 
 def parse_asic_external_link(link, asic_name, hostname):
     neighbors = {}
@@ -295,6 +387,8 @@ def parse_asic_png(png, asic_name, hostname):
                 if lo_prefix_v6:
                     device_data['lo_addr_v6']= lo_prefix_v6
                 devices[name] = device_data
+
+
     return (neighbors, devices, port_speeds)
 
 def parse_loopback_intf(child):
@@ -339,12 +433,13 @@ def parse_dpg(dpg, hname):
 
         ipintfs = child.find(str(QName(ns, "IPInterfaces")))
         intfs = {}
+        ip_intfs_map = {}
         for ipintf in ipintfs.findall(str(QName(ns, "IPInterface"))):
             intfalias = ipintf.find(str(QName(ns, "AttachTo"))).text
             intfname = port_alias_map.get(intfalias, intfalias)
             ipprefix = ipintf.find(str(QName(ns, "Prefix"))).text
             intfs[(intfname, ipprefix)] = {}
-
+            ip_intfs_map[ipprefix] = intfalias
         lo_intfs =  parse_loopback_intf(child)
 
         mvrfConfigs = child.find(str(QName(ns, "MgmtVrfConfigs")))
@@ -381,7 +476,46 @@ def parse_dpg(dpg, hname):
                 pcs[pcintfname] = {'members': pcmbr_list, 'fallback': pcintf.find(str(QName(ns, "Fallback"))).text, 'min_links': str(int(math.ceil(len() * 0.75)))}
             else:
                 pcs[pcintfname] = {'members': pcmbr_list, 'min_links': str(int(math.ceil(len(pcmbr_list) * 0.75)))}
-
+        port_nhipv4_map = {}
+        port_nhipv6_map = {}
+        nhgaddr = ["", ""]
+        nhg_int = ""
+        nhportlist = []
+        dpg_ecmp_content = {}
+        ipnhs = child.find(str(QName(ns, "IPNextHops")))
+        if ipnhs is not None:
+            for ipnh in ipnhs.findall(str(QName(ns, "IPNextHop"))):
+                if ipnh.find(str(QName(ns, "Type"))).text == 'FineGrainedECMPGroupMember':
+                    ipnhfmbr = ipnh.find(str(QName(ns, "AttachTo"))).text
+                    ipnhaddr = ipnh.find(str(QName(ns, "Address"))).text
+                    nhportlist.append(ipnhfmbr)
+                    if "." in ipnhaddr:
+                        port_nhipv4_map[ipnhfmbr] = ipnhaddr
+                    elif ":" in ipnhaddr:
+                        port_nhipv6_map[ipnhfmbr] = ipnhaddr
+
+            if port_nhipv4_map is not None and port_nhipv6_map is not None:
+                subnet_check_ip = list(port_nhipv4_map.values())[0]
+                for subnet_range in ip_intfs_map:
+                    if ("." in subnet_range):
+                        a = ipaddress.ip_address(UNICODE_TYPE(subnet_check_ip))
+                        n = list(ipaddress.ip_network(UNICODE_TYPE(subnet_range), False).hosts())
+                        if a in n:
+                            nhg_int = ip_intfs_map[subnet_range]
+                dwnstrms = child.find(str(QName(ns, "DownstreamSummarySet")))
+                for dwnstrm in dwnstrms.findall(str(QName(ns, "DownstreamSummary"))):
+                    dwnstrmentry = str(ET.tostring(dwnstrm))
+                    if ("FineGrainedECMPGroupDestination" in dwnstrmentry):
+                        subnet_ip = dwnstrm.find(str(QName(ns1, "Subnet"))).text
+                        truncsubnet_ip = subnet_ip.split("/")[0]
+                        if "." in (truncsubnet_ip):
+                            nhgaddr[0] = subnet_ip
+                        elif ":" in (truncsubnet_ip):
+                            nhgaddr[1] = subnet_ip
+                ipv4_content = {"port_nhip_map": port_nhipv4_map, "nhgaddr": nhgaddr[0], "nhg_int": nhg_int}
+                ipv6_content = {"port_nhip_map": port_nhipv6_map, "nhgaddr": nhgaddr[1], "nhg_int": nhg_int}
+                dpg_ecmp_content['ipv4'] = ipv4_content
+                dpg_ecmp_content['ipv6'] = ipv6_content
         vlanintfs = child.find(str(QName(ns, "VlanInterfaces")))
         vlan_intfs = []
         vlans = {}
@@ -503,6 +637,7 @@ def parse_dpg(dpg, hname):
                 except:
                     print("Warning: Ignoring Control Plane ACL %s without type" % aclname, file=sys.stderr)
 
+
         mg_tunnels = child.find(str(QName(ns, "TunnelInterfaces")))
         if mg_tunnels is not None:
             table_key_to_mg_key_map = {"encap_ecn_mode": "EcnEncapsulationMode", 
@@ -521,7 +656,7 @@ def parse_dpg(dpg, hname):
                     if mg_key in mg_tunnel.attrib:
                         tunnelintfs[tunnel_type][tunnel_name][table_key] = mg_tunnel.attrib[mg_key]
 
-        return intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnelintfs
+        return intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnelintfs, dpg_ecmp_content
     return None, None, None, None, None, None, None, None, None, None
 
 def parse_host_loopback(dpg, hname):
@@ -927,6 +1062,8 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
     bgp_monitors = []
     bgp_asn = None
     intfs = None
+    dpg_ecmp_content = {}
+    png_ecmp_content = {}
     vlan_intfs = None
     pc_intfs = None
     tunnel_intfs = None
@@ -988,13 +1125,13 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
     for child in root:
         if asic_name is None:
             if child.tag == str(QName(ns, "DpgDec")):
-                (intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnel_intfs) = parse_dpg(child, hostname)
+                (intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnel_intfs, dpg_ecmp_content) = parse_dpg(child, hostname)
             elif child.tag == str(QName(ns, "CpgDec")):
                 (bgp_sessions, bgp_internal_sessions, bgp_asn, bgp_peers_with_range, bgp_monitors) = parse_cpg(child, hostname)
             elif child.tag == str(QName(ns, "PngDec")):
-                (neighbors, devices, console_dev, console_port, mgmt_dev, mgmt_port, port_speed_png, console_ports, mux_cable_ports, is_storage_device) = parse_png(child, hostname)
+                (neighbors, devices, console_dev, console_port, mgmt_dev, mgmt_port, port_speed_png, console_ports, mux_cable_ports, is_storage_device, png_ecmp_content) = parse_png(child, hostname, dpg_ecmp_content)
             elif child.tag == str(QName(ns, "UngDec")):
-                (u_neighbors, u_devices, _, _, _, _, _, _) = parse_png(child, hostname)
+                (u_neighbors, u_devices, _, _, _, _, _, _) = parse_png(child, hostname, None)
             elif child.tag == str(QName(ns, "MetadataDeclaration")):
                 (syslog_servers, dhcp_servers, ntp_servers, tacacs_servers, mgmt_routes, erspan_dst, deployment_id, region, cloudtype, resource_type) = parse_meta(child, hostname)
             elif child.tag == str(QName(ns, "LinkMetadataDeclaration")):
@@ -1003,7 +1140,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
                 (port_speeds_default, port_descriptions) = parse_deviceinfo(child, hwsku)
         else:
             if child.tag == str(QName(ns, "DpgDec")):
-                (intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnel_intfs) = parse_dpg(child, asic_name)
+                (intfs, lo_intfs, mvrf, mgmt_intf, vlans, vlan_members, pcs, pc_members, acls, vni, tunnel_intfs, dpg_ecmp_content) = parse_dpg(child, asic_name)
                 host_lo_intfs = parse_host_loopback(child, hostname)
             elif child.tag == str(QName(ns, "CpgDec")):
                 (bgp_sessions, bgp_internal_sessions, bgp_asn, bgp_peers_with_range, bgp_monitors) = parse_cpg(child, asic_name, local_devices)
@@ -1305,6 +1442,13 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hw
             'client_crt_cname': 'client.restapi.sonic'
         }
     }
+
+    if len(png_ecmp_content):
+        results['FG_NHG_MEMBER'] = png_ecmp_content['FG_NHG_MEMBER']
+        results['FG_NHG_PREFIX'] = png_ecmp_content['FG_NHG_PREFIX']
+        results['FG_NHG'] = png_ecmp_content['FG_NHG']
+        results['NEIGH'] = png_ecmp_content['NEIGH']
+
     # Do not configure the minigraph's mirror session, which is currently unused
     # mirror_sessions = {}
     # if erspan_dst:

src/sonic-config-engine/tests/mellanox-sample-port-config.ini

@@ -0,0 +1,65 @@
+# name         lanes              alias     index
+Ethernet0      0,1,2,3            etp1      1
+Ethernet4      4,5,6,7            etp2      2
+Ethernet8      8,9,10,11          etp3      3
+Ethernet12     12,13,14,15        etp4      4
+Ethernet16     16,17,18,19        etp5      5
+Ethernet20     20,21,22,23        etp6      6
+Ethernet24     24,25,26,27        etp7      7
+Ethernet28     28,29,30,31        etp8      8
+Ethernet32     32,33,34,35        etp9      9
+Ethernet36     36,37,38,39        etp10     10
+Ethernet40     40,41,42,43        etp11     11
+Ethernet44     44,45,46,47        etp12     12
+Ethernet48     48,49,50,51        etp13     13
+Ethernet52     52,53,54,55        etp14     14
+Ethernet56     56,57,58,59        etp15     15
+Ethernet60     60,61,62,63        etp16     16
+Ethernet64     64,65,66,67        etp17     17
+Ethernet68     68,69,70,71        etp18     18
+Ethernet72     72,73,74,75        etp19     19
+Ethernet76     76,77,78,79        etp20     20
+Ethernet80     80,81,82,83        etp21     21
+Ethernet84     84,85,86,87        etp22     22
+Ethernet88     88,89,90,91        etp23     23
+Ethernet92     92,93,94,95        etp24     24
+Ethernet96     96,97,98,99        etp25     25
+Ethernet100    100,101,102,103    etp26     26
+Ethernet104    104,105,106,107    etp27     27
+Ethernet108    108,109,110,111    etp28     28
+Ethernet112    112,113,114,115    etp29     29
+Ethernet116    116,117,118,119    etp30     30
+Ethernet120    120,121,122,123    etp31     31
+Ethernet124    124,125,126,127    etp32     32
+Ethernet128    128,129,130,131    etp33     33
+Ethernet132    132,133,134,135    etp34     34
+Ethernet136    136,137,138,139    etp35     35
+Ethernet140    140,141,142,143    etp36     36
+Ethernet144    144,145,146,147    etp37     37
+Ethernet148    148,149,150,151    etp38     38
+Ethernet152    152,153,154,155    etp39     39
+Ethernet156    156,157,158,159    etp40     40
+Ethernet160    160,161,162,163    etp41     41
+Ethernet164    164,165,166,167    etp42     42
+Ethernet168    168,169,170,171    etp43     43
+Ethernet172    172,173,174,175    etp44     44
+Ethernet176    176,177,178,179    etp45     45
+Ethernet180    180,181,182,183    etp46     46
+Ethernet184    184,185,186,187    etp47     47
+Ethernet188    188,189,190,191    etp48     48
+Ethernet192    192,193,194,195    etp49     49
+Ethernet196    196,197,198,199    etp50     50
+Ethernet200    200,201,202,203    etp51     51
+Ethernet204    204,205,206,207    etp52     52
+Ethernet208    208,209,210,211    etp53     53
+Ethernet212    212,213,214,215    etp54     54
+Ethernet216    216,217,218,219    etp55     55
+Ethernet220    220,221,222,223    etp56     56
+Ethernet224    224,225,226,227    etp57     57
+Ethernet228    228,229,230,231    etp58     58
+Ethernet232    232,233,234,235    etp59     59
+Ethernet236    236,237,238,239    etp60     60
+Ethernet240    240,241,242,243    etp61     61
+Ethernet244    244,245,246,247    etp62     62
+Ethernet248    248,249,250,251    etp63     63
+Ethernet252    252,253,254,255    etp64     64