Skip to content

Commit 500e2e9

Browse files
vganesan-nokiavganesan-nokia
authored
[voq/inbandif] Voq inbandif port (sonic-net#1602)
What I did Changes to support inband interface configuration Why I did it The VOQ chassis needs inband interface in the kernel for cpu to cpu communication across different asic instance and remote neighbor programming in the kernel. This PR adds support for configuring inband interface so that kernel interface can be created via sonic configuration. How I verified it swss vs test (test_virtual_chassis.py) to verify the remote neigh programming.
1 parent 9f22ba7 commit 500e2e9

11 files changed

+586
-221
lines changed

‎cfgmgr/nbrmgr.cpp

+38-5
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,8 @@ void NbrMgr::doStateSystemNeighTask(Consumer &consumer)
348348
//Get the name of the device on which the neigh and route are
349349
//going to be programmed.
350350
string nbr_odev;
351-
if(!getVoqInbandInterfaceName(nbr_odev))
351+
string ibif_type;
352+
if(!getVoqInbandInterfaceName(nbr_odev, ibif_type))
352353
{
353354
//The inband interface is not available yet
354355
return;
@@ -380,16 +381,19 @@ void NbrMgr::doStateSystemNeighTask(Consumer &consumer)
380381
mac_address = MacAddress(fvValue(*i));
381382
}
382383

383-
if (!isIntfStateOk(nbr_odev))
384+
if (ibif_type == "port" && !isIntfOperUp(nbr_odev))
384385
{
385-
SWSS_LOG_DEBUG("Interface %s is not ready, skipping system neigh %s'", nbr_odev.c_str(), kfvKey(t).c_str());
386+
SWSS_LOG_DEBUG("Device %s is not oper up, skipping system neigh %s'", nbr_odev.c_str(), kfvKey(t).c_str());
386387
it++;
387388
continue;
388389
}
389390

390391
if (!addKernelNeigh(nbr_odev, ip_address, mac_address))
391392
{
392393
SWSS_LOG_ERROR("Neigh entry add on dev %s failed for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
394+
// Delete neigh to take care of deletion of exiting nbr for mac change. This makes sure that
395+
// re-try will be successful and route addtion (below) will be attempted and be successful
396+
delKernelNeigh(nbr_odev, ip_address);
393397
it++;
394398
continue;
395399
}
@@ -402,6 +406,8 @@ void NbrMgr::doStateSystemNeighTask(Consumer &consumer)
402406
{
403407
SWSS_LOG_ERROR("Route entry add on dev %s failed for '%s'", nbr_odev.c_str(), kfvKey(t).c_str());
404408
delKernelNeigh(nbr_odev, ip_address);
409+
// Delete route to take care of deletion of exiting route of nbr for mac change.
410+
delKernelRoute(ip_address);
405411
it++;
406412
continue;
407413
}
@@ -437,9 +443,24 @@ void NbrMgr::doStateSystemNeighTask(Consumer &consumer)
437443
}
438444
}
439445

440-
bool NbrMgr::getVoqInbandInterfaceName(string &ibif)
446+
bool NbrMgr::isIntfOperUp(const string &alias)
441447
{
448+
string oper;
449+
450+
if (m_statePortTable.hget(alias, "netdev_oper_status", oper))
451+
{
452+
if (oper == "up")
453+
{
454+
SWSS_LOG_DEBUG("NetDev %s is oper up", alias.c_str());
455+
return true;
456+
}
457+
}
442458

459+
return false;
460+
}
461+
462+
bool NbrMgr::getVoqInbandInterfaceName(string &ibif, string &type)
463+
{
443464
vector<string> keys;
444465
m_cfgVoqInbandInterfaceTable->getKeys(keys);
445466

@@ -448,9 +469,21 @@ bool NbrMgr::getVoqInbandInterfaceName(string &ibif)
448469
SWSS_LOG_NOTICE("Voq Inband interface is not configured!");
449470
return false;
450471
}
451-
//key:"alias" = inband interface name
472+
473+
// key:"alias" = inband interface name
474+
452475
vector<string> if_keys = tokenize(keys[0], config_db_key_delimiter);
476+
453477
ibif = if_keys[0];
478+
479+
// Get the type of the inband interface
480+
481+
if (!m_cfgVoqInbandInterfaceTable->hget(ibif, "inband_type", type))
482+
{
483+
SWSS_LOG_ERROR("Getting Voq Inband interface type failed for %s", ibif.c_str());
484+
return false;
485+
}
486+
454487
return true;
455488
}
456489

‎cfgmgr/nbrmgr.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,12 @@ class NbrMgr : public Orch
3232
void doSetNeighTask(Consumer &consumer);
3333
void doTask(Consumer &consumer);
3434
void doStateSystemNeighTask(Consumer &consumer);
35-
bool getVoqInbandInterfaceName(string &nbr_odev);
35+
bool getVoqInbandInterfaceName(string &nbr_odev, string &ibiftype);
3636
bool addKernelRoute(string odev, IpAddress ip_addr);
3737
bool delKernelRoute(IpAddress ip_addr);
3838
bool addKernelNeigh(string odev, IpAddress ip_addr, MacAddress mac_addr);
3939
bool delKernelNeigh(string odev, IpAddress ip_addr);
40+
bool isIntfOperUp(const std::string &alias);
4041
unique_ptr<Table> m_cfgVoqInbandInterfaceTable;
4142

4243
Table m_statePortTable, m_stateLagTable, m_stateVlanTable, m_stateIntfTable, m_stateNeighRestoreTable;

‎orchagent/neighorch.cpp

+122-6
Original file line numberDiff line numberDiff line change
@@ -1028,6 +1028,16 @@ void NeighOrch::doVoqSystemNeighTask(Consumer &consumer)
10281028
return;
10291029
}
10301030

1031+
// For "port" type inband interface, wait till the Inband interface is both admin up and oper up
1032+
if (ibif.m_type != Port::VLAN)
1033+
{
1034+
if (ibif.m_admin_state_up != true || ibif.m_oper_status != SAI_PORT_OPER_STATUS_UP)
1035+
{
1036+
// Inband port is not operational yet
1037+
return;
1038+
}
1039+
}
1040+
10311041
auto it = consumer.m_toSync.begin();
10321042
while (it != consumer.m_toSync.end())
10331043
{
@@ -1161,20 +1171,126 @@ void NeighOrch::doVoqSystemNeighTask(Consumer &consumer)
11611171

11621172
bool NeighOrch::addInbandNeighbor(string alias, IpAddress ip_address)
11631173
{
1164-
//For "port" type inband, the inband reachability info syncing can be done through static
1165-
//configureation or CHASSIS_APP_DB sync (this function)
1174+
//Add neighbor record in SAI without adding host route for local inband to avoid route
1175+
//looping for packets destined to the Inband interface if the Inband is port type
1176+
1177+
if(gIntfsOrch->isRemoteSystemPortIntf(alias))
1178+
{
1179+
//Remote Inband interface. Skip
1180+
return true;
1181+
}
1182+
1183+
sai_status_t status;
1184+
MacAddress inband_mac = gMacAddress;
1185+
1186+
sai_object_id_t rif_id = gIntfsOrch->getRouterIntfsId(alias);
1187+
if (rif_id == SAI_NULL_OBJECT_ID)
1188+
{
1189+
SWSS_LOG_INFO("Failed to get rif_id for %s", alias.c_str());
1190+
return false;
1191+
}
1192+
1193+
//Make the object key
1194+
sai_neighbor_entry_t neighbor_entry;
1195+
neighbor_entry.rif_id = rif_id;
1196+
neighbor_entry.switch_id = gSwitchId;
1197+
copy(neighbor_entry.ip_address, ip_address);
1198+
1199+
vector<sai_attribute_t> neighbor_attrs;
1200+
sai_attribute_t attr;
1201+
attr.id = SAI_NEIGHBOR_ENTRY_ATTR_DST_MAC_ADDRESS;
1202+
memcpy(attr.value.mac, inband_mac.getMac(), 6);
1203+
neighbor_attrs.push_back(attr);
1204+
1205+
//No host route for neighbor of the Inband IP address
1206+
attr.id = SAI_NEIGHBOR_ENTRY_ATTR_NO_HOST_ROUTE;
1207+
attr.value.booldata = true;
1208+
neighbor_attrs.push_back(attr);
1209+
1210+
status = sai_neighbor_api->create_neighbor_entry(&neighbor_entry, static_cast<uint32_t>(neighbor_attrs.size()), neighbor_attrs.data());
1211+
if (status != SAI_STATUS_SUCCESS)
1212+
{
1213+
if (status == SAI_STATUS_ITEM_ALREADY_EXISTS)
1214+
{
1215+
SWSS_LOG_ERROR("Entry exists: neighbor %s on %s, rv:%d", inband_mac.to_string().c_str(), alias.c_str(), status);
1216+
return true;
1217+
}
1218+
else
1219+
{
1220+
SWSS_LOG_ERROR("Failed to create neighbor %s on %s, rv:%d", inband_mac.to_string().c_str(), alias.c_str(), status);
1221+
return false;
1222+
}
1223+
}
1224+
1225+
SWSS_LOG_NOTICE("Created inband neighbor %s on %s", inband_mac.to_string().c_str(), alias.c_str());
11661226

1167-
//For "vlan" type inband, the inband reachability info syncinng can be ARP learning of other
1168-
//asics inband or static configuration or through CHASSIS_APP_DB sync (this function)
1227+
gIntfsOrch->increaseRouterIntfsRefCount(alias);
11691228

1170-
//May implement inband rechability info syncing through CHASSIS_APP_DB sync here
1229+
if (neighbor_entry.ip_address.addr_family == SAI_IP_ADDR_FAMILY_IPV4)
1230+
{
1231+
gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV4_NEIGHBOR);
1232+
}
1233+
else
1234+
{
1235+
gCrmOrch->incCrmResUsedCounter(CrmResourceType::CRM_IPV6_NEIGHBOR);
1236+
}
1237+
1238+
//Sync the neighbor to add to the CHASSIS_APP_DB
1239+
voqSyncAddNeigh(alias, ip_address, inband_mac, neighbor_entry);
11711240

11721241
return true;
11731242
}
11741243

11751244
bool NeighOrch::delInbandNeighbor(string alias, IpAddress ip_address)
11761245
{
1177-
//Remove inband rechability info sync
1246+
// Remove local inband neighbor from SAI
1247+
1248+
if(gIntfsOrch->isRemoteSystemPortIntf(alias))
1249+
{
1250+
//Remote Inband interface. Skip
1251+
return true;
1252+
}
1253+
1254+
MacAddress inband_mac = gMacAddress;
1255+
1256+
sai_object_id_t rif_id = gIntfsOrch->getRouterIntfsId(alias);
1257+
1258+
sai_neighbor_entry_t neighbor_entry;
1259+
neighbor_entry.rif_id = rif_id;
1260+
neighbor_entry.switch_id = gSwitchId;
1261+
copy(neighbor_entry.ip_address, ip_address);
1262+
1263+
sai_status_t status;
1264+
status = sai_neighbor_api->remove_neighbor_entry(&neighbor_entry);
1265+
if (status != SAI_STATUS_SUCCESS)
1266+
{
1267+
if (status == SAI_STATUS_ITEM_NOT_FOUND)
1268+
{
1269+
SWSS_LOG_ERROR("Failed to locate neigbor %s on %s, rv:%d", inband_mac.to_string().c_str(), alias.c_str(), status);
1270+
return true;
1271+
}
1272+
else
1273+
{
1274+
SWSS_LOG_ERROR("Failed to remove neighbor %s on %s, rv:%d", inband_mac.to_string().c_str(), alias.c_str(), status);
1275+
return false;
1276+
}
1277+
}
1278+
1279+
if (neighbor_entry.ip_address.addr_family == SAI_IP_ADDR_FAMILY_IPV4)
1280+
{
1281+
gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV4_NEIGHBOR);
1282+
}
1283+
else
1284+
{
1285+
gCrmOrch->decCrmResUsedCounter(CrmResourceType::CRM_IPV6_NEIGHBOR);
1286+
}
1287+
1288+
SWSS_LOG_NOTICE("Removed neighbor %s on %s", inband_mac.to_string().c_str(), alias.c_str());
1289+
1290+
gIntfsOrch->decreaseRouterIntfsRefCount(alias);
1291+
1292+
//Sync the neighbor to delete from the CHASSIS_APP_DB
1293+
voqSyncDelNeigh(alias, ip_address);
11781294

11791295
return true;
11801296
}

‎orchagent/portsorch.cpp

+11-10
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ extern int32_t gVoqMySwitchId;
5252
extern string gMyHostName;
5353
extern string gMyAsicName;
5454

55+
#define DEFAULT_SYSTEM_PORT_MTU 9100
5556
#define VLAN_PREFIX "Vlan"
5657
#define DEFAULT_VLAN_ID 1
5758
#define MAX_VALID_VLAN_ID 4094
@@ -5462,6 +5463,7 @@ bool PortsOrch::addSystemPorts()
54625463
port.m_admin_state_up = true;
54635464
port.m_oper_status = SAI_PORT_OPER_STATUS_UP;
54645465
port.m_speed = attrs[1].value.sysportconfig.speed;
5466+
port.m_mtu = DEFAULT_SYSTEM_PORT_MTU;
54655467
if (attrs[0].value.s32 == SAI_SYSTEM_PORT_TYPE_LOCAL)
54665468
{
54675469
//Get the local port oid
@@ -5543,21 +5545,20 @@ bool PortsOrch::setVoqInbandIntf(string &alias, string &type)
55435545
return true;
55445546
}
55455547

5548+
//Make sure port and host if exists for the configured inband interface
55465549
Port port;
5547-
if(type == "port")
5550+
if (!getPort(alias, port))
55485551
{
5549-
if (!getPort(alias, port))
5550-
{
5551-
SWSS_LOG_NOTICE("Port configured for inband intf %s is not ready!", alias.c_str());
5552-
return false;
5553-
}
5552+
SWSS_LOG_ERROR("Port/Vlan configured for inband intf %s is not ready!", alias.c_str());
5553+
return false;
55545554
}
55555555

5556-
// Check for existence of host interface. If does not exist, may create
5557-
// host if for the inband here
5556+
if(type == "port" && !port.m_hif_id)
5557+
{
5558+
SWSS_LOG_ERROR("Host interface is not available for port %s", alias.c_str());
5559+
return false;
5560+
}
55585561

5559-
// May do the processing for other inband type like type=vlan here
5560-
55615562
//Store the name of the local inband port
55625563
m_inbandPortName = alias;
55635564

‎orchagent/routeorch.cpp

+9
Original file line numberDiff line numberDiff line change
@@ -1392,6 +1392,15 @@ bool RouteOrch::addRoute(RouteBulkContext& ctx, const NextHopGroupKey &nextHops)
13921392

13931393
if (nexthop.ip_address.isZero())
13941394
{
1395+
if(gPortsOrch->isInbandPort(nexthop.alias))
1396+
{
1397+
//This routes is the static route added for the remote system neighbors
1398+
//We do not need this route in the ASIC since the static neighbor creation
1399+
//in ASIC adds the same full mask route (host route) in ASIC automatically
1400+
//So skip.
1401+
return true;
1402+
}
1403+
13951404
next_hop_id = m_intfsOrch->getRouterIntfsId(nexthop.alias);
13961405
/* rif is not created yet */
13971406
if (next_hop_id == SAI_NULL_OBJECT_ID)

‎portsyncd/linksync.cpp

+3-1
Original file line numberDiff line numberDiff line change
@@ -253,8 +253,10 @@ void LinkSync::onMsg(int nlmsg_type, struct nl_object *obj)
253253
FieldValueTuple tuple("state", "ok");
254254
vector<FieldValueTuple> vector;
255255
vector.push_back(tuple);
256+
FieldValueTuple op("netdev_oper_status", oper ? "up" : "down");
257+
vector.push_back(op);
256258
m_statePortTable.set(key, vector);
257-
SWSS_LOG_NOTICE("Publish %s(ok) to state db", key.c_str());
259+
SWSS_LOG_NOTICE("Publish %s(ok:%s) to state db", key.c_str(), oper ? "up" : "down");
258260
}
259261
else
260262
{

‎tests/conftest.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1446,7 +1446,7 @@ def handle_neighconn(self):
14461446
def get_chassis_instance_port_statuses(self):
14471447
instance_to_port_status_map = {}
14481448
if "neighbor_connections" not in self.virt_topo:
1449-
return instance_to_neighbor_map
1449+
return instance_to_port_status_map
14501450

14511451
working_dir = os.getcwd()
14521452
for conn, endpoints in self.virt_topo["neighbor_connections"].items():

0 commit comments

Comments
 (0)