diff --git a/orchagent/neighorch.cpp b/orchagent/neighorch.cpp index 5b414d9eaa..4ddedaa080 100644 --- a/orchagent/neighorch.cpp +++ b/orchagent/neighorch.cpp @@ -3,6 +3,7 @@ #include "logger.h" #include "swssnet.h" #include "crmorch.h" +#include "routeorch.h" extern sai_neighbor_api_t* sai_neighbor_api; extern sai_next_hop_api_t* sai_next_hop_api; @@ -10,6 +11,7 @@ extern sai_next_hop_api_t* sai_next_hop_api; extern PortsOrch *gPortsOrch; extern sai_object_id_t gSwitchId; extern CrmOrch *gCrmOrch; +extern RouteOrch *gRouteOrch; NeighOrch::NeighOrch(DBConnector *db, string tableName, IntfsOrch *intfsOrch) : Orch(db, tableName), m_intfsOrch(intfsOrch) @@ -59,6 +61,8 @@ bool NeighOrch::addNextHop(IpAddress ipAddress, string alias) NextHopEntry next_hop_entry; next_hop_entry.next_hop_id = next_hop_id; next_hop_entry.ref_count = 0; + next_hop_entry.nh_flags = 0; + next_hop_entry.if_alias = alias; m_syncdNextHops[ipAddress] = next_hop_entry; m_intfsOrch->increaseRouterIntfsRefCount(alias); @@ -75,6 +79,114 @@ bool NeighOrch::addNextHop(IpAddress ipAddress, string alias) return true; } +bool NeighOrch::setNextHopFlag(const IpAddress &ipaddr, const uint32_t nh_flag) +{ + SWSS_LOG_ENTER(); + + auto nhop = m_syncdNextHops.find(ipaddr); + bool rc = false; + + assert(nhop != m_syncdNextHops.end()); + + if (nhop->second.nh_flags & nh_flag) + { + return true; + } + + nhop->second.nh_flags |= nh_flag; + + switch (nh_flag) + { + case NHFLAGS_IFDOWN: + rc = gRouteOrch->invalidnexthopinNextHopGroup(ipaddr); + break; + default: + assert(0); + break; + } + + return rc; +} + +bool NeighOrch::clearNextHopFlag(const IpAddress &ipaddr, const uint32_t nh_flag) +{ + SWSS_LOG_ENTER(); + + auto nhop = m_syncdNextHops.find(ipaddr); + bool rc = false; + + assert(nhop != m_syncdNextHops.end()); + + if (!(nhop->second.nh_flags & nh_flag)) + { + return true; + } + + nhop->second.nh_flags &= ~nh_flag; + + switch (nh_flag) + { + case NHFLAGS_IFDOWN: + rc = gRouteOrch->validnexthopinNextHopGroup(ipaddr); + break; + default: + assert(0); + break; + } + + return rc; +} + +bool NeighOrch::isNextHopFlagSet(const IpAddress &ipaddr, const uint32_t nh_flag) +{ + SWSS_LOG_ENTER(); + + auto nhop = m_syncdNextHops.find(ipaddr); + + assert(nhop != m_syncdNextHops.end()); + + if (nhop->second.nh_flags & nh_flag) + { + return true; + } + + return false; +} + +bool NeighOrch::ifChangeInformNextHop(const string &alias, bool if_up) +{ + SWSS_LOG_ENTER(); + bool rc = true; + + for (auto nhop = m_syncdNextHops.begin(); nhop != m_syncdNextHops.end(); ++nhop) + { + if (nhop->second.if_alias != alias) + { + continue; + } + + if (if_up) + { + rc = clearNextHopFlag(nhop->first, NHFLAGS_IFDOWN); + } + else + { + rc = setNextHopFlag(nhop->first, NHFLAGS_IFDOWN); + } + + if (rc == true) + { + continue; + } + else + { + break; + } + } + + return rc; +} + bool NeighOrch::removeNextHop(IpAddress ipAddress, string alias) { SWSS_LOG_ENTER(); diff --git a/orchagent/neighorch.h b/orchagent/neighorch.h index 47cbd1f59f..71257d1415 100644 --- a/orchagent/neighorch.h +++ b/orchagent/neighorch.h @@ -8,6 +8,8 @@ #include "ipaddress.h" +#define NHFLAGS_IFDOWN 0x1 // nexthop's outbound i/f is down + struct NeighborEntry { IpAddress ip_address; // neighbor IP address @@ -33,6 +35,8 @@ struct NextHopEntry { sai_object_id_t next_hop_id; // next hop id int ref_count; // reference count + uint32_t nh_flags; // flags + string if_alias; // i/f name alias }; /* NeighborTable: NeighborEntry, neighbor MAC address */ @@ -62,6 +66,9 @@ class NeighOrch : public Orch, public Subject bool getNeighborEntry(const IpAddress&, NeighborEntry&, MacAddress&); + bool ifChangeInformNextHop(const string &, bool); + bool isNextHopFlagSet(const IpAddress &, const uint32_t); + private: IntfsOrch *m_intfsOrch; @@ -74,6 +81,9 @@ class NeighOrch : public Orch, public Subject bool addNeighbor(NeighborEntry, MacAddress); bool removeNeighbor(NeighborEntry); + bool setNextHopFlag(const IpAddress &, const uint32_t); + bool clearNextHopFlag(const IpAddress &, const uint32_t); + void doTask(Consumer &consumer); }; diff --git a/orchagent/orchdaemon.cpp b/orchagent/orchdaemon.cpp index abc4ab29a8..3c9fe381e9 100644 --- a/orchagent/orchdaemon.cpp +++ b/orchagent/orchdaemon.cpp @@ -17,19 +17,19 @@ using namespace swss; extern sai_switch_api_t* sai_switch_api; extern sai_object_id_t gSwitchId; -/* Global variable gPortsOrch declared */ +/* + * Global orch daemon variables + */ PortsOrch *gPortsOrch; -/* Global variable gFdbOrch declared */ FdbOrch *gFdbOrch; -/*Global variable gAclOrch declared*/ +NeighOrch *gNeighOrch; +RouteOrch *gRouteOrch; AclOrch *gAclOrch; -/*Global variable gCrmOrch declared*/ CrmOrch *gCrmOrch; OrchDaemon::OrchDaemon(DBConnector *applDb, DBConnector *configDb) : m_applDb(applDb), m_configDb(configDb) - { SWSS_LOG_ENTER(); } @@ -64,8 +64,8 @@ bool OrchDaemon::init() gPortsOrch = new PortsOrch(m_applDb, ports_tables); gFdbOrch = new FdbOrch(m_applDb, APP_FDB_TABLE_NAME, gPortsOrch); IntfsOrch *intfs_orch = new IntfsOrch(m_applDb, APP_INTF_TABLE_NAME); - NeighOrch *neigh_orch = new NeighOrch(m_applDb, APP_NEIGH_TABLE_NAME, intfs_orch); - RouteOrch *route_orch = new RouteOrch(m_applDb, APP_ROUTE_TABLE_NAME, neigh_orch); + gNeighOrch = new NeighOrch(m_applDb, APP_NEIGH_TABLE_NAME, intfs_orch); + gRouteOrch = new RouteOrch(m_applDb, APP_ROUTE_TABLE_NAME, gNeighOrch); CoppOrch *copp_orch = new CoppOrch(m_applDb, APP_COPP_TABLE_NAME); TunnelDecapOrch *tunnel_decap_orch = new TunnelDecapOrch(m_applDb, APP_TUNNEL_DECAP_TABLE_NAME); @@ -94,16 +94,16 @@ bool OrchDaemon::init() TableConnector appDbMirrorSession(m_applDb, APP_MIRROR_SESSION_TABLE_NAME); TableConnector confDbMirrorSession(m_configDb, CFG_MIRROR_SESSION_TABLE_NAME); - MirrorOrch *mirror_orch = new MirrorOrch(appDbMirrorSession, confDbMirrorSession, gPortsOrch, route_orch, neigh_orch, gFdbOrch); + MirrorOrch *mirror_orch = new MirrorOrch(appDbMirrorSession, confDbMirrorSession, gPortsOrch, gRouteOrch, gNeighOrch, gFdbOrch); VRFOrch *vrf_orch = new VRFOrch(m_configDb, CFG_VRF_TABLE_NAME); vector acl_tables = { CFG_ACL_TABLE_NAME, CFG_ACL_RULE_TABLE_NAME }; - gAclOrch = new AclOrch(m_configDb, acl_tables, gPortsOrch, mirror_orch, neigh_orch, route_orch); + gAclOrch = new AclOrch(m_configDb, acl_tables, gPortsOrch, mirror_orch, gNeighOrch, gRouteOrch); - m_orchList = { switch_orch, gCrmOrch, gPortsOrch, intfs_orch, neigh_orch, route_orch, copp_orch, tunnel_decap_orch, qos_orch, buffer_orch, mirror_orch, gAclOrch, gFdbOrch, vrf_orch }; + m_orchList = { switch_orch, gCrmOrch, gPortsOrch, intfs_orch, gNeighOrch, gRouteOrch, copp_orch, tunnel_decap_orch, qos_orch, buffer_orch, mirror_orch, gAclOrch, gFdbOrch, vrf_orch }; m_select = new Select(); vector pfc_wd_tables = { diff --git a/orchagent/portsorch.cpp b/orchagent/portsorch.cpp index fecbec1fdb..00284af5d7 100644 --- a/orchagent/portsorch.cpp +++ b/orchagent/portsorch.cpp @@ -1,4 +1,5 @@ #include "portsorch.h" +#include "neighorch.h" #include #include @@ -27,6 +28,7 @@ extern sai_hostif_api_t* sai_hostif_api; extern sai_acl_api_t* sai_acl_api; extern sai_queue_api_t *sai_queue_api; extern sai_object_id_t gSwitchId; +extern NeighOrch *gNeighOrch; extern CrmOrch *gCrmOrch; #define VLAN_PREFIX "Vlan" @@ -852,23 +854,30 @@ bool PortsOrch::setHostIntfsOperStatus(sai_object_id_t port_id, bool up) for (auto it = m_portList.begin(); it != m_portList.end(); it++) { - if (it->second.m_port_id == port_id) + if (it->second.m_port_id != port_id) { - sai_attribute_t attr; - attr.id = SAI_HOSTIF_ATTR_OPER_STATUS; - attr.value.booldata = up; + continue; + } - sai_status_t status = sai_hostif_api->set_hostif_attribute(it->second.m_hif_id, &attr); - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_WARN("Failed to set operation status %s to host interface %s", - up ? "UP" : "DOWN", it->second.m_alias.c_str()); - return false; - } - SWSS_LOG_NOTICE("Set operation status %s to host interface %s", - up ? "UP" : "DOWN", it->second.m_alias.c_str()); - return true; + sai_attribute_t attr; + attr.id = SAI_HOSTIF_ATTR_OPER_STATUS; + attr.value.booldata = up; + + sai_status_t status = sai_hostif_api->set_hostif_attribute(it->second.m_hif_id, &attr); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_WARN("Failed to set operation status %s to host interface %s", + up ? "UP" : "DOWN", it->second.m_alias.c_str()); + return false; } + SWSS_LOG_NOTICE("Set operation status %s to host interface %s", + up ? "UP" : "DOWN", it->second.m_alias.c_str()); + if (gNeighOrch->ifChangeInformNextHop(it->second.m_alias, up) == false) + { + SWSS_LOG_WARN("Inform nexthop operation failed for interface %s", + it->second.m_alias.c_str()); + } + return true; } return false; } diff --git a/orchagent/routeorch.cpp b/orchagent/routeorch.cpp index a3b7e235fc..ec8aeb5295 100644 --- a/orchagent/routeorch.cpp +++ b/orchagent/routeorch.cpp @@ -166,6 +166,83 @@ void RouteOrch::detach(Observer *observer, const IpAddress& dstAddr) } } +bool RouteOrch::validnexthopinNextHopGroup(const IpAddress &ipaddr) +{ + SWSS_LOG_ENTER(); + + sai_object_id_t nexthop_id; + sai_status_t status; + + for (auto nhopgroup = m_syncdNextHopGroups.begin(); + nhopgroup != m_syncdNextHopGroups.end(); ++nhopgroup) + { + + if (!(nhopgroup->first.contains(ipaddr))) + { + continue; + } + + vector nhgm_attrs; + sai_attribute_t nhgm_attr; + + nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID; + nhgm_attr.value.oid = nhopgroup->second.next_hop_group_id; + nhgm_attrs.push_back(nhgm_attr); + + nhgm_attr.id = SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_ID; + nhgm_attr.value.oid = m_neighOrch->getNextHopId(ipaddr); + nhgm_attrs.push_back(nhgm_attr); + + status = sai_next_hop_group_api->create_next_hop_group_member(&nexthop_id, gSwitchId, + (uint32_t)nhgm_attrs.size(), + nhgm_attrs.data()); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to add next hop member to group %lx: %d\n", + nhopgroup->second.next_hop_group_id, status); + return false; + } + + gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); + nhopgroup->second.nhopgroup_members[ipaddr] = nexthop_id; + } + + return true; +} + +bool RouteOrch::invalidnexthopinNextHopGroup(const IpAddress &ipaddr) +{ + SWSS_LOG_ENTER(); + + sai_object_id_t nexthop_id; + sai_status_t status; + + for (auto nhopgroup = m_syncdNextHopGroups.begin(); + nhopgroup != m_syncdNextHopGroups.end(); ++nhopgroup) + { + + if (!(nhopgroup->first.contains(ipaddr))) + { + continue; + } + + nexthop_id = nhopgroup->second.nhopgroup_members[ipaddr]; + status = sai_next_hop_group_api->remove_next_hop_group_member(nexthop_id); + + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove next hop member %lx from group %lx: %d\n", + nexthop_id, nhopgroup->second.next_hop_group_id, status); + return false; + } + + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); + } + + return true; +} + void RouteOrch::doTask(Consumer& consumer) { SWSS_LOG_ENTER(); @@ -436,6 +513,8 @@ bool RouteOrch::addNextHopGroup(IpAddresses ipAddresses) vector next_hop_ids; set next_hop_set = ipAddresses.getIpAddresses(); + sai_object_id_t next_hop_id; + std::map nhopgroup_members_set; /* Assert each IP address exists in m_syncdNextHops table, * and add the corresponding next_hop_id to next_hop_ids. */ @@ -450,6 +529,7 @@ bool RouteOrch::addNextHopGroup(IpAddresses ipAddresses) sai_object_id_t next_hop_id = m_neighOrch->getNextHopId(it); next_hop_ids.push_back(next_hop_id); + nhopgroup_members_set[next_hop_id] = it; } sai_attribute_t nhg_attr; @@ -460,8 +540,10 @@ bool RouteOrch::addNextHopGroup(IpAddresses ipAddresses) nhg_attrs.push_back(nhg_attr); sai_object_id_t next_hop_group_id; - sai_status_t status = sai_next_hop_group_api-> - create_next_hop_group(&next_hop_group_id, gSwitchId, (uint32_t)nhg_attrs.size(), nhg_attrs.data()); + sai_status_t status = sai_next_hop_group_api->create_next_hop_group(&next_hop_group_id, + gSwitchId, + (uint32_t)nhg_attrs.size(), + nhg_attrs.data()); if (status != SAI_STATUS_SUCCESS) { @@ -493,8 +575,10 @@ bool RouteOrch::addNextHopGroup(IpAddresses ipAddresses) nhgm_attrs.push_back(nhgm_attr); sai_object_id_t next_hop_group_member_id; - status = sai_next_hop_group_api-> - create_next_hop_group_member(&next_hop_group_member_id, gSwitchId, (uint32_t)nhgm_attrs.size(), nhgm_attrs.data()); + status = sai_next_hop_group_api->create_next_hop_group_member(&next_hop_group_member_id, + gSwitchId, + (uint32_t)nhgm_attrs.size(), + nhgm_attrs.data()); if (status != SAI_STATUS_SUCCESS) { @@ -507,10 +591,11 @@ bool RouteOrch::addNextHopGroup(IpAddresses ipAddresses) gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); // Save the membership into next hop structure - next_hop_group_entry.next_hop_group_members.insert(next_hop_group_member_id); + next_hop_group_entry.nhopgroup_members[nhopgroup_members_set.find(nhid)->second] = + next_hop_group_member_id; } - /* Increate the ref_count for the next hops used by the next hop group. */ + /* Increment the ref_count for the next hops used by the next hop group. */ for (auto it : next_hop_set) m_neighOrch->increaseNextHopRefCount(it); @@ -521,6 +606,22 @@ bool RouteOrch::addNextHopGroup(IpAddresses ipAddresses) next_hop_group_entry.ref_count = 0; m_syncdNextHopGroups[ipAddresses] = next_hop_group_entry; + for (auto nhop : next_hop_set) { + if (!m_neighOrch->isNextHopFlagSet(nhop, NHFLAGS_IFDOWN)) { + continue; + } + + next_hop_id = next_hop_group_entry.nhopgroup_members[nhop]; + status = sai_next_hop_group_api->remove_next_hop_group_member(next_hop_id); + + if (status != SAI_STATUS_SUCCESS) { + SWSS_LOG_ERROR("Failed to remove next hop group member %lx: %d\n", + next_hop_id, status); + } + + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); + } + return true; } @@ -528,43 +629,57 @@ bool RouteOrch::removeNextHopGroup(IpAddresses ipAddresses) { SWSS_LOG_ENTER(); + sai_object_id_t next_hop_group_id; + auto next_hop_group_entry = m_syncdNextHopGroups.find(ipAddresses); sai_status_t status; - assert(hasNextHopGroup(ipAddresses)); - if (m_syncdNextHopGroups[ipAddresses].ref_count == 0) + assert(next_hop_group_entry != m_syncdNextHopGroups.end()); + + if (next_hop_group_entry->second.ref_count != 0) { - auto next_hop_group_entry = m_syncdNextHopGroups[ipAddresses]; - sai_object_id_t next_hop_group_id = next_hop_group_entry.next_hop_group_id; + return true; + } - for (auto next_hop_group_member_id: next_hop_group_entry.next_hop_group_members) - { - status = sai_next_hop_group_api->remove_next_hop_group_member(next_hop_group_member_id); - if (status != SAI_STATUS_SUCCESS) - { - SWSS_LOG_ERROR("Failed to remove next hop group member %lx, rv:%d", next_hop_group_member_id, status); - return false; - } + next_hop_group_id = next_hop_group_entry->second.next_hop_group_id; + SWSS_LOG_NOTICE("Delete next hop group %s", ipAddresses.to_string().c_str()); - gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); - } + for (auto nhop = next_hop_group_entry->second.nhopgroup_members.begin(); + nhop != next_hop_group_entry->second.nhopgroup_members.end();) + { - sai_status_t status = sai_next_hop_group_api->remove_next_hop_group(next_hop_group_id); + if (m_neighOrch->isNextHopFlagSet(nhop->first, NHFLAGS_IFDOWN)) + { + nhop = next_hop_group_entry->second.nhopgroup_members.erase(nhop); + continue; + } + status = sai_next_hop_group_api->remove_next_hop_group_member(nhop->second); if (status != SAI_STATUS_SUCCESS) { - SWSS_LOG_ERROR("Failed to remove next hop group %lx, rv:%d", next_hop_group_id, status); + SWSS_LOG_ERROR("Failed to remove next hop group member %lx, rv:%d", + nhop->second, status); return false; } - m_nextHopGroupCount --; + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP_MEMBER); + nhop = next_hop_group_entry->second.nhopgroup_members.erase(nhop); + } - gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); + status = sai_next_hop_group_api->remove_next_hop_group(next_hop_group_id); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("Failed to remove next hop group %lx, rv:%d", next_hop_group_id, status); + return false; + } - set ip_address_set = ipAddresses.getIpAddresses(); - for (auto it : ip_address_set) - m_neighOrch->decreaseNextHopRefCount(it); + m_nextHopGroupCount --; + gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_NEXTHOP_GROUP); - m_syncdNextHopGroups.erase(ipAddresses); + set ip_address_set = ipAddresses.getIpAddresses(); + for (auto it : ip_address_set) + { + m_neighOrch->decreaseNextHopRefCount(it); } + m_syncdNextHopGroups.erase(ipAddresses); return true; } diff --git a/orchagent/routeorch.h b/orchagent/routeorch.h index b34ac6aa9f..af17b627a4 100644 --- a/orchagent/routeorch.h +++ b/orchagent/routeorch.h @@ -15,11 +15,13 @@ /* Maximum next hop group number */ #define NHGRP_MAX_SIZE 128 +typedef std::map NextHopGroupMembers; + struct NextHopGroupEntry { sai_object_id_t next_hop_group_id; // next hop group id - std::set next_hop_group_members; // next hop group member ids int ref_count; // reference count + NextHopGroupMembers nhopgroup_members; // ids of members indexed by ip address }; struct NextHopUpdate @@ -61,6 +63,9 @@ class RouteOrch : public Orch, public Subject bool addNextHopGroup(IpAddresses); bool removeNextHopGroup(IpAddresses); + bool validnexthopinNextHopGroup(const IpAddress &); + bool invalidnexthopinNextHopGroup(const IpAddress &); + private: NeighOrch *m_neighOrch; diff --git a/tests/test_nhg.py b/tests/test_nhg.py new file mode 100644 index 0000000000..bf41c793d2 --- /dev/null +++ b/tests/test_nhg.py @@ -0,0 +1,128 @@ +from swsscommon import swsscommon +import os +import re +import time +import json + +def test_route_nhg(dvs): + + dvs.runcmd("ifconfig Ethernet0 10.0.0.0/31 up") + dvs.runcmd("ifconfig Ethernet4 10.0.0.2/31 up") + dvs.runcmd("ifconfig Ethernet8 10.0.0.4/31 up") + + dvs.runcmd("arp -s 10.0.0.1 00:00:00:00:00:01") + dvs.runcmd("arp -s 10.0.0.3 00:00:00:00:00:02") + dvs.runcmd("arp -s 10.0.0.5 00:00:00:00:00:03") + + dvs.servers[0].runcmd("ip link set down dev eth0") == 0 + dvs.servers[1].runcmd("ip link set down dev eth0") == 0 + dvs.servers[2].runcmd("ip link set down dev eth0") == 0 + + dvs.servers[0].runcmd("ip link set up dev eth0") == 0 + dvs.servers[1].runcmd("ip link set up dev eth0") == 0 + dvs.servers[2].runcmd("ip link set up dev eth0") == 0 + + db = swsscommon.DBConnector(0, dvs.redis_sock, 0) + ps = swsscommon.ProducerStateTable(db, "ROUTE_TABLE") + fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.3,10.0.0.5"), ("ifname", "Ethernet0,Ethernet4,Ethernet8")]) + + ps.set("2.2.2.0/24", fvs) + + time.sleep(1) + + # check if route was propagated to ASIC DB + + adb = swsscommon.DBConnector(1, dvs.redis_sock, 0) + + rtbl = swsscommon.Table(adb, "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY") + nhgtbl = swsscommon.Table(adb, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP") + nhg_member_tbl = swsscommon.Table(adb, "ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP_GROUP_MEMBER") + + keys = rtbl.getKeys() + + found_route = False + for k in keys: + rt_key = json.loads(k) + + if rt_key['dest'] == "2.2.2.0/24": + found_route = True + break + + assert found_route + + # assert the route points to next hop group + (status, fvs) = rtbl.get(k) + + for v in fvs: + if v[0] == "SAI_ROUTE_ENTRY_ATTR_NEXT_HOP_ID": + nhgid = v[1] + + (status, fvs) = nhgtbl.get(nhgid) + + assert status + + keys = nhg_member_tbl.getKeys() + + assert len(keys) == 3 + + for k in keys: + (status, fvs) = nhg_member_tbl.get(k) + + for v in fvs: + if v[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID": + assert v[1] == nhgid + + # bring links down one-by-one + for i in [0, 1, 2]: + dvs.servers[i].runcmd("ip link set down dev eth0") == 0 + + time.sleep(1) + + tbl = swsscommon.Table(db, "PORT_TABLE") + (status, fvs) = tbl.get("Ethernet%d" % (i * 4)) + + assert status == True + + oper_status = "unknown" + + for v in fvs: + if v[0] == "oper_status": + oper_status = v[1] + break + + assert oper_status == "down" + + keys = nhg_member_tbl.getKeys() + + assert len(keys) == 2 - i + + # bring links up one-by-one + for i in [0, 1, 2]: + dvs.servers[i].runcmd("ip link set up dev eth0") == 0 + + time.sleep(1) + + tbl = swsscommon.Table(db, "PORT_TABLE") + (status, fvs) = tbl.get("Ethernet0") + + assert status == True + + oper_status = "unknown" + + for v in fvs: + if v[0] == "oper_status": + oper_status = v[1] + break + + assert oper_status == "up" + + keys = nhg_member_tbl.getKeys() + + assert len(keys) == i + 1 + + for k in keys: + (status, fvs) = nhg_member_tbl.get(k) + + for v in fvs: + if v[0] == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID": + assert v[1] == nhgid