Support for in-band-mgmt via management VRF

cfgmgr/intfmgr.cpp

@@ -19,6 +19,7 @@ using namespace swss;
 #define VNET_PREFIX         "Vnet"
 #define MTU_INHERITANCE     "0"
 #define VRF_PREFIX          "Vrf"
+#define VRF_MGMT            "mgmt"
 
 #define LOOPBACK_DEFAULT_MTU_STR "65536"
 
@@ -399,7 +400,8 @@ bool IntfMgr::isIntfStateOk(const string &alias)
             return true;
         }
     }
-    else if (!alias.compare(0, strlen(VRF_PREFIX), VRF_PREFIX))
+    else if ((!alias.compare(0, strlen(VRF_PREFIX), VRF_PREFIX)) ||
+            (alias == VRF_MGMT))
     {
         if (m_stateVrfTable.get(alias, temp))
         {

cfgmgr/vrfmgr.cpp

@@ -12,6 +12,8 @@
 #define VRF_TABLE_START 1001
 #define VRF_TABLE_END 2000
 #define TABLE_LOCAL_PREF 1001 // after l3mdev-table
+#define MGMT_VRF_TABLE_ID 5000
+#define MGMT_VRF          "mgmt"
 
 using namespace swss;
 
@@ -143,6 +145,13 @@ bool VrfMgr::delLink(const string& vrfName)
         return false;
     }
 
+    if (vrfName == MGMT_VRF)
+    {
+        recycleTable(m_vrfTableMap[vrfName]);
+        m_vrfTableMap.erase(vrfName);
+        return true;
+    }
+
     cmd << IP_CMD << " link del " << shellquote(vrfName);
     EXEC_WITH_ERROR_THROW(cmd.str(), res);
 
@@ -163,6 +172,15 @@ bool VrfMgr::setLink(const string& vrfName)
     {
         return true;
     }
+
+    if (vrfName == MGMT_VRF)
+    {
+        // Mgmt VRF is initialised as part of hostcfgd, 
+        // just return the reserved table_id for mgmt VRF from here.
+        uint32_t table_id = MGMT_VRF_TABLE_ID;
+        m_vrfTableMap.emplace(vrfName, table_id);
+        return true;
+    }
 
     uint32_t table = getFreeTable();
     if (table == 0)
@@ -207,6 +225,51 @@ void VrfMgr::doTask(Consumer &consumer)
         auto vrfName = kfvKey(t);
 
         string op = kfvOp(t);
+        // Mgmt VRF table event handling for in-band management
+        if (consumer.getTableName() == CFG_MGMT_VRF_CONFIG_TABLE_NAME)
+        {
+            SWSS_LOG_DEBUG("Event for mgmt VRF op %s", op.c_str());
+            if (op == SET_COMMAND) 
+            {
+                bool in_band_mgmt_enabled = false;
+                bool mgmt_vrf_enabled = false;
+                for (auto i : kfvFieldsValues(t))
+                {
+                    if (fvField(i) == "mgmtVrfEnabled")
+                    {
+                        if (fvValue(i) == "true")
+                        {
+                            mgmt_vrf_enabled = true;
+                        }
+                        SWSS_LOG_DEBUG("Event for mgmt VRF table mgmt_vrf_enabled is set val:%s", fvValue(i).c_str());
+                    }
+                    else if (fvField(i) == "in_band_mgmt_enabled")
+                    {
+                        if (fvValue(i) == "true")
+                        {
+                            in_band_mgmt_enabled = true;
+                        }
+                        SWSS_LOG_DEBUG("Event for mgmt VRF table in_band_mgmt_enabled is set val:%s", fvValue(i).c_str());
+                    }
+                }
+                // If mgmt VRF is not enabled or in-band-mgmt is not enabled delete the in-band-mgmt 
+                // related VRF table map information
+                if ((op == SET_COMMAND) && ((mgmt_vrf_enabled == false) || (in_band_mgmt_enabled == false)))
+                {
+                    op = DEL_COMMAND;
+                }
+            }
+            vrfName = MGMT_VRF;
+            if (((op == DEL_COMMAND) && (m_vrfTableMap.find(vrfName) == m_vrfTableMap.end())) ||
+                    ((op == SET_COMMAND) && (m_vrfTableMap.find(vrfName) != m_vrfTableMap.end())))
+            {
+                // If the mgmt VRF is not populated already, return
+                it = consumer.m_toSync.erase(it);
+                continue;
+            }
+            SWSS_LOG_DEBUG("Event for mgmt VRF op %s", op.c_str());
+        }
+        SWSS_LOG_DEBUG("Event for table %s vrf netdev %s id %s", consumer.getTableName().c_str(), vrfName.c_str(), op.c_str());
         if (op == SET_COMMAND)
         {
             if (consumer.getTableName() == CFG_VXLAN_EVPN_NVO_TABLE_NAME)
@@ -226,7 +289,8 @@ void VrfMgr::doTask(Consumer &consumer)
                 m_stateVrfTable.set(vrfName, fvVector);
 
                 SWSS_LOG_NOTICE("Created vrf netdev %s", vrfName.c_str());
-                if (consumer.getTableName() == CFG_VRF_TABLE_NAME)
+                if ((consumer.getTableName() == CFG_VRF_TABLE_NAME) ||
+                    (consumer.getTableName() == CFG_MGMT_VRF_CONFIG_TABLE_NAME))
                 {
                     status  = doVrfVxlanTableCreateTask (t);
                     if (status == false)
@@ -256,7 +320,8 @@ void VrfMgr::doTask(Consumer &consumer)
             {
                 doVrfEvpnNvoDelTask (t);
             }
-            else if (consumer.getTableName() == CFG_VRF_TABLE_NAME)
+            else if ((consumer.getTableName() == CFG_VRF_TABLE_NAME) ||
+                     (consumer.getTableName() == CFG_MGMT_VRF_CONFIG_TABLE_NAME))
             {
                 vector<FieldValueTuple> temp;
 

cfgmgr/vrfmgrd.cpp

@@ -46,6 +46,7 @@ int main(int argc, char **argv)
             CFG_VRF_TABLE_NAME,
             CFG_VNET_TABLE_NAME,
             CFG_VXLAN_EVPN_NVO_TABLE_NAME,
+            CFG_MGMT_VRF_CONFIG_TABLE_NAME
         };
 
         DBConnector cfgDb("CONFIG_DB", 0);

fpmsyncd/routesync.cpp

@@ -19,6 +19,7 @@ using namespace swss;
 #define VXLAN_IF_NAME_PREFIX    "Brvxlan"
 #define VNET_PREFIX             "Vnet"
 #define VRF_PREFIX              "Vrf"
+#define MGMT_VRF_PREFIX         "mgmt"
 
 #ifndef ETH_ALEN
 #define ETH_ALEN 6
@@ -625,7 +626,17 @@ void RouteSync::onRouteMsg(int nlmsg_type, struct nl_object *obj, char *vrf)
          */
         if (memcmp(vrf, VRF_PREFIX, strlen(VRF_PREFIX)))
         {
-            SWSS_LOG_ERROR("Invalid VRF name %s (ifindex %u)", vrf, rtnl_route_get_table(route_obj));
+            if(memcmp(vrf, MGMT_VRF_PREFIX, strlen(MGMT_VRF_PREFIX)))
+            {
+                SWSS_LOG_ERROR("Invalid VRF name %s (ifindex %u)", vrf, rtnl_route_get_table(route_obj));
+            }
+            else
+            {
+                dip = rtnl_route_get_dst(route_obj);
+                nl_addr2str(dip, destipprefix, MAX_ADDR_SIZE);
+                SWSS_LOG_INFO("Skip routes for Mgmt VRF name %s (ifindex %u) prefix: %s", vrf,
+                        rtnl_route_get_table(route_obj), destipprefix);
+            }
             return;
         }
         memcpy(destipprefix, vrf, strlen(vrf));

orchagent/intfsorch.cpp

@@ -42,6 +42,7 @@ const int intfsorch_pri = 35;
 #define RIF_FLEX_STAT_COUNTER_POLL_MSECS "1000"
 #define UPDATE_MAPS_SEC 1
 
+#define MGMT_VRF            "mgmt"
 
 static const vector<sai_router_interface_stat_t> rifStatIds =
 {
@@ -159,6 +160,23 @@ string IntfsOrch::getRouterIntfsAlias(const IpAddress &ip, const string &vrf_nam
     return string();
 }
 
+bool IntfsOrch::isInbandIntfInMgmtVrf(const string& alias)
+{
+    if (m_syncdIntfses.find(alias) == m_syncdIntfses.end())
+    {
+        return false;
+    }
+
+    string vrf_name = "";
+    vrf_name = m_vrfOrch->getVRFname(m_syncdIntfses[alias].vrf_id);
+    if ((!vrf_name.empty()) && (vrf_name == MGMT_VRF))
+    {
+        return true;
+    }
+
+    return false;
+}
+
 void IntfsOrch::increaseRouterIntfsRefCount(const string &alias)
 {
     SWSS_LOG_ENTER();

orchagent/intfsorch.h

@@ -36,6 +36,7 @@ class IntfsOrch : public Orch
 
     sai_object_id_t getRouterIntfsId(const string&);
     bool isPrefixSubnet(const IpPrefix&, const string&);
+    bool isInbandIntfInMgmtVrf(const string& alias);
     string getRouterIntfsAlias(const IpAddress &ip, const string &vrf_name = "");
     string getRifRateFlexCounterTableKey(string key);
     void increaseRouterIntfsRefCount(const string&);

orchagent/neighorch.cpp

@@ -551,7 +551,8 @@ void NeighOrch::doTask(Consumer &consumer)
 
         string alias = key.substr(0, found);
 
-        if (alias == "eth0" || alias == "lo" || alias == "docker0")
+        if (alias == "eth0" || alias == "lo" || alias == "docker0"
+            || ((op == SET_COMMAND) && m_intfsOrch->isInbandIntfInMgmtVrf(alias)))
         {
             it = consumer.m_toSync.erase(it);
             continue;

tests/test_inband_intf_mgmt_vrf.py

@@ -0,0 +1,160 @@
+import time
+import pytest
+
+from swsscommon import swsscommon
+
+MGMT_VRF_NAME = 'mgmt'
+INBAND_INTF_NAME = 'Ethernet4'
+
+class TestInbandInterface(object):
+    def setup_db(self, dvs):
+        self.appl_db = swsscommon.DBConnector(0, dvs.redis_sock, 0)
+        self.asic_db = swsscommon.DBConnector(1, dvs.redis_sock, 0)
+        self.cfg_db = swsscommon.DBConnector(4, dvs.redis_sock, 0)
+
+    def add_mgmt_vrf(self):
+        tbl = swsscommon.Table(self.asic_db, 'ASIC_STATE:SAI_OBJECT_TYPE_VIRTUAL_ROUTER')
+        initial_entries = set(tbl.getKeys())
+
+        tbl = swsscommon.Table(self.cfg_db, 'MGMT_VRF_CONFIG')
+        fvs = swsscommon.FieldValuePairs([('mgmtVrfEnabled', 'true'), ('in_band_mgmt_enabled', 'true')])
+        tbl.set('vrf_global', fvs)
+        time.sleep(1)
+
+        # check application database
+        tbl = swsscommon.Table(self.appl_db, 'VRF_TABLE')
+        vrf_keys = tbl.getKeys()
+        assert len(vrf_keys) == 1
+        assert vrf_keys[0] == MGMT_VRF_NAME
+
+        # check SAI database
+        tbl = swsscommon.Table(self.asic_db, 'ASIC_STATE:SAI_OBJECT_TYPE_VIRTUAL_ROUTER')
+        current_entries = set(tbl.getKeys())
+        assert len(current_entries - initial_entries) == 1
+        return list(current_entries - initial_entries)[0]
+
+    def del_mgmt_vrf(self):
+        tbl = swsscommon.Table(self.cfg_db, 'MGMT_VRF_CONFIG')
+        tbl._del('vrf_global')
+        time.sleep(1)
+
+        # check application database
+        tbl = swsscommon.Table(self.appl_db, 'VRF_TABLE')
+        vrf_keys = tbl.getKeys()
+        assert len(vrf_keys) == 0
+
+        # check SAI database
+        tbl = swsscommon.Table(self.asic_db, 'ASIC_STATE:SAI_OBJECT_TYPE_VIRTUAL_ROUTER')
+        current_entries = set(tbl.getKeys())
+        assert len(current_entries) == 1
+
+    def create_inband_intf(self, interface):
+        cfg_tbl = cfg_key = cfg_fvs = None
+        if interface.startswith('PortChannel'):
+            tbl_name = 'PORTCHANNEL_INTERFACE'
+            cfg_tbl = 'PORTCHANNEL'
+            cfg_key = interface
+            cfg_fvs = swsscommon.FieldValuePairs([("admin_status", "up"),
+                                              ("mtu", "9100")])
+        elif interface.startswith('Vlan'):
+            tbl_name = 'VLAN_INTERFACE'
+            cfg_tbl = 'VLAN'
+            vlan_id = interface[len('Vlan'):]
+            cfg_key = 'Vlan' + vlan_id
+            cfg_fvs = swsscommon.FieldValuePairs([("vlanid", vlan_id)])
+        elif interface.startswith('Loopback'):
+            tbl_name = 'LOOPBACK_INTERFACE'
+        else:
+            tbl_name = 'INTERFACE'
+        if cfg_tbl is not None:
+            tbl = swsscommon.Table(self.cfg_db, cfg_tbl)
+            tbl.set(cfg_key, cfg_fvs)
+            time.sleep(1)
+        fvs = swsscommon.FieldValuePairs([('vrf_name', MGMT_VRF_NAME)])
+        tbl = swsscommon.Table(self.cfg_db, tbl_name)
+        tbl.set(interface, fvs)
+        time.sleep(1)
+
+    def remove_inband_intf(self, interface):
+        cfg_tbl = cfg_key = None
+        if interface.startswith('PortChannel'):
+            tbl_name = 'PORTCHANNEL_INTERFACE'
+            cfg_tbl = 'PORTCHANNEL'
+            cfg_key = interface
+        elif interface.startswith('Vlan'):
+            tbl_name = 'VLAN_INTERFACE'
+            cfg_tbl = 'VLAN'
+            vlan_id = interface[len('Vlan'):]
+            cfg_key = 'Vlan' + vlan_id
+        elif interface.startswith('Loopback'):
+            tbl_name = 'LOOPBACK_INTERFACE'
+        else:
+            tbl_name = 'INTERFACE'
+        tbl = swsscommon.Table(self.cfg_db, tbl_name)
+        tbl._del(interface)
+        time.sleep(1)
+        if cfg_tbl is not None:
+            tbl = swsscommon.Table(self.cfg_db, cfg_tbl)
+            tbl._del(cfg_key)
+            time.sleep(1)
+
+    def test_MgmtVrf(self, dvs, testlog):
+        self.setup_db(dvs)
+
+        vrf_oid = self.add_mgmt_vrf()
+        self.del_mgmt_vrf()
+
+    @pytest.mark.parametrize('intf_name', ['Ethernet4', 'Vlan100', 'PortChannel5', 'Loopback1'])
+    def test_InbandIntf(self, intf_name, dvs, testlog):
+        self.setup_db(dvs)
+
+        vrf_oid = self.add_mgmt_vrf()
+        self.create_inband_intf(intf_name)
+
+        # check application database
+        tbl = swsscommon.Table(self.appl_db, 'INTF_TABLE')
+        intf_keys = tbl.getKeys()
+        status, fvs = tbl.get(intf_name)
+        assert status == True
+        for fv in fvs:
+            if fv[0] == 'vrf_name':
+                assert fv[1] == MGMT_VRF_NAME
+
+        if not intf_name.startswith('Loopback'):
+            # check ASIC router interface database
+            tbl = swsscommon.Table(self.asic_db, 'ASIC_STATE:SAI_OBJECT_TYPE_ROUTER_INTERFACE')
+            intf_keys = tbl.getKeys()
+            # one loopback router interface one port based router interface
+            assert len(intf_keys) == 2
+            for key in intf_keys:
+                status, fvs = tbl.get(key)
+                assert status == True
+                loopback = False
+                intf_vrf_oid = None
+                for fv in fvs:
+                    if (fv[0] == 'SAI_ROUTER_INTERFACE_ATTR_TYPE' and
+                        fv[1] == 'SAI_ROUTER_INTERFACE_TYPE_LOOPBACK'):
+                        loopback = True
+                        break
+                    if fv[0] == 'SAI_ROUTER_INTERFACE_ATTR_VIRTUAL_ROUTER_ID':
+                        intf_vrf_oid = fv[1]
+                if loopback:
+                    continue
+                assert intf_vrf_oid == vrf_oid
+
+        self.remove_inband_intf(intf_name)
+
+        # check application database
+        tbl = swsscommon.Table(self.appl_db, 'INTF_TABLE')
+        intf_keys = tbl.getKeys()
+        assert len(intf_keys) == 0
+
+        if not intf_name.startswith('Loopback'):
+            # check ASIC router interface database
+            tbl = swsscommon.Table(self.asic_db, 'ASIC_STATE:SAI_OBJECT_TYPE_ROUTER_INTERFACE')
+            intf_keys = tbl.getKeys()
+            # one loopback router interface
+            assert len(intf_keys) ==1 
+
+        self.del_mgmt_vrf()
+