Skip to content

Commit

Permalink
Add warm boot support with removed/created port (#515)
Browse files Browse the repository at this point in the history
* Add warm boot support with removed/created port

* Fix aspell

* Update sai_warmboot.bin location to local folder
  • Loading branch information
kcudnik authored Sep 24, 2019
1 parent 59e530a commit 1f4a1d7
Show file tree
Hide file tree
Showing 16 changed files with 424 additions and 22 deletions.
32 changes: 27 additions & 5 deletions syncd/syncd.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1257,6 +1257,7 @@ void get_port_related_objects(

void post_port_remove(
_In_ std::shared_ptr<SaiSwitch> sw,
_In_ sai_object_id_t port_rid,
_In_ const std::vector<sai_object_id_t>& relatedRids)
{
SWSS_LOG_ENTER();
Expand Down Expand Up @@ -1307,16 +1308,16 @@ void post_port_remove(

sai_object_type_t ot = redis_sai_object_type_query(vid);

std::string key = sai_serialize_object_type(ot) + ":" + str_vid;
std::string key = ASIC_STATE_TABLE + std::string(":") + sai_serialize_object_type(ot) + ":" + str_vid;

SWSS_LOG_INFO("removing ASIC DB key: %s", key.c_str());

g_redisClient->del(key);
}

SWSS_LOG_NOTICE("post port remove actions succeeded");
sw->onPostPortRemove(port_rid);

// TODO lane map must be updated (for warm boot)
SWSS_LOG_NOTICE("post port remove actions succeeded");
}

void post_port_create(
Expand All @@ -1328,7 +1329,7 @@ void post_port_create(

sw->onPostPortCreate(port_rid, port_vid);

// TODO lane map must be updated (for warm boot)
SWSS_LOG_NOTICE("post port create actions succeeded");
}

sai_status_t handle_generic(
Expand Down Expand Up @@ -1560,7 +1561,7 @@ sai_status_t handle_generic(

if (object_type == SAI_OBJECT_TYPE_PORT)
{
post_port_remove(switches.at(switch_vid), related);
post_port_remove(switches.at(switch_vid), rid, related);
}
}
}
Expand Down Expand Up @@ -2418,6 +2419,16 @@ sai_status_t processEventInInitViewMode(
{
case SAI_COMMON_API_CREATE:

if (object_type == SAI_OBJECT_TYPE_PORT)
{
// reason for this is that if user will create port,
// new port is not actually created so when for example
// querying new queues for new created port, there are
// not there, since no actual port create was issued on
// the ASIC
SWSS_LOG_THROW("port object can't be created in init view mode");
}

if (info->isnonobjectid)
{
/*
Expand Down Expand Up @@ -2450,6 +2461,17 @@ sai_status_t processEventInInitViewMode(

case SAI_COMMON_API_REMOVE:

if (object_type == SAI_OBJECT_TYPE_PORT)
{
// reason for this is that if user will remove port, actual
// resources for it wont be release, lanes would be still
// occupied and there is extra logic required in post port
// remove which clears OIDs (ipgs,queues,SGs) from redis db
// that are automatically removed by vendor SAI, and comparison
// logic don't support that
SWSS_LOG_THROW("port object can't be removed in init view mode");
}

if (object_type == SAI_OBJECT_TYPE_SWITCH)
{
/*
Expand Down
41 changes: 40 additions & 1 deletion syncd/syncd_applyview.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6985,6 +6985,7 @@ void populateExistingObjects(
auto sw = switches.begin()->second;

auto coldBootDiscoveredVids = sw->getColdBootDiscoveredVids();
auto warmBootDiscoveredVids = sw->getWarmBootDiscoveredVids();

/*
* If some objects that are existing objects on switch are not present in
Expand Down Expand Up @@ -7058,9 +7059,47 @@ void populateExistingObjects(
* NOTE: If we are here, then this RID exists only in current view, and
* if this object contains any OID attributes, discovery logic queried
* them so they are also existing in current view.
*
* Also in warm boot, when user removed port, and then created some new
* ports, new QUEUEs, IPGs and SGs will be created automatically by
* SAI. Those new created objects mot likely will have different RID
* values then previous instances for given port. Those values should
* also be copied to temporary view, since they will not exist on cold
* boot discovered VIDs. If not, then comparison logic will try to remove
* them which is not what we want.
*
* This is tricky scenario, and there could be some issues also when
* other object types would be created by user.
*/

if (coldBootDiscoveredVids.find(vid) == coldBootDiscoveredVids.end())
bool performColdCheck = true;

if (warmBootDiscoveredVids.find(vid) != warmBootDiscoveredVids.end())
{
sai_object_type_t ot = redis_sai_object_type_query(vid);

switch (ot)
{
case SAI_OBJECT_TYPE_QUEUE:
case SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP:
case SAI_OBJECT_TYPE_SCHEDULER_GROUP:

// TODO this case may require adjustment, if user will do a
// warm boot then remove/add some ports and make another
// warm boot, it may happen that current logic will be
// confused which of those objects are from previous warm
// boot or second one, need better way to mark changes to
// those objects in redis DB between warm boots

performColdCheck = false;

break;
default:
break;
}
}

if (performColdCheck && coldBootDiscoveredVids.find(vid) == coldBootDiscoveredVids.end())
{
SWSS_LOG_INFO("object is not on default existing list: %s RID %s VID %s",
sai_serialize_object_type(sai_object_type_query(rid)).c_str(),
Expand Down
2 changes: 1 addition & 1 deletion syncd/syncd_hard_reinit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1349,7 +1349,7 @@ void performWarmRestart()
* Perform all get operations on existing switch.
*/

auto sw = switches[switch_vid] = std::make_shared<SaiSwitch>(switch_vid, switch_rid);
auto sw = switches[switch_vid] = std::make_shared<SaiSwitch>(switch_vid, switch_rid, true);

g_switch_rid = switch_rid;
g_switch_vid = switch_vid;
Expand Down
139 changes: 133 additions & 6 deletions syncd/syncd_saiswitch.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#include <unordered_map>
#include <set>

const int maxLanesPerPort = 8;

/*
* NOTE: all those methods could be implemented inside SaiSwitch class so then
* we could skip using switch_id in params and even they could be public then.
Expand Down Expand Up @@ -148,18 +150,16 @@ std::unordered_map<sai_uint32_t, sai_object_id_t> SaiSwitch::saiGetHardwareLaneM
* addressed in future.
*/

const int lanesPerPort = 8;

for (const auto &port_rid : portList)
{
sai_uint32_t lanes[lanesPerPort];
sai_uint32_t lanes[maxLanesPerPort];

memset(lanes, 0, sizeof(lanes));

sai_attribute_t attr;

attr.id = SAI_PORT_ATTR_HW_LANE_LIST;
attr.value.u32list.count = lanesPerPort;
attr.value.u32list.count = maxLanesPerPort;
attr.value.u32list.list = lanes;

sai_status_t status = sai_metadata_sai_port_api->get_port_attribute(port_rid, 1, &attr);
Expand Down Expand Up @@ -1059,6 +1059,13 @@ std::set<sai_object_id_t> SaiSwitch::getColdBootDiscoveredVids() const
return discoveredVids;
}

std::set<sai_object_id_t> SaiSwitch::getWarmBootDiscoveredVids() const
{
SWSS_LOG_ENTER();

return m_warmBootDiscoveredVids;
}

void SaiSwitch::redisSaveColdBootDiscoveredVids() const
{
SWSS_LOG_ENTER();
Expand Down Expand Up @@ -1224,6 +1231,21 @@ sai_object_id_t SaiSwitch::getDefaultValueForOidAttr(
return ita->second;
}

void SaiSwitch::helperPopulateWarmBootVids()
{
SWSS_LOG_ENTER();

if (!m_warmBoot)
return;

for (sai_object_id_t rid: m_discovered_rids)
{
sai_object_id_t vid = translate_rid_to_vid(rid, m_switch_vid);

m_warmBootDiscoveredVids.insert(vid);
}
}

/*
* NOTE: If real ID will change during hard restarts, then we need to remap all
* VID/RID, but we can only do that if we will save entire tree with all
Expand All @@ -1232,7 +1254,9 @@ sai_object_id_t SaiSwitch::getDefaultValueForOidAttr(

SaiSwitch::SaiSwitch(
_In_ sai_object_id_t switch_vid,
_In_ sai_object_id_t switch_rid)
_In_ sai_object_id_t switch_rid,
_In_ bool warmBoot):
m_warmBoot(warmBoot)
{
SWSS_LOG_ENTER();

Expand Down Expand Up @@ -1263,9 +1287,68 @@ SaiSwitch::SaiSwitch(

helperLoadColdVids();

helperPopulateWarmBootVids();

saiGetMacAddress(m_default_mac_address);
}

std::vector<uint32_t> SaiSwitch::saiGetPortLanes(
_In_ sai_object_id_t port_rid)
{
SWSS_LOG_ENTER();

std::vector<uint32_t> lanes;

lanes.resize(maxLanesPerPort);

sai_attribute_t attr;

attr.id = SAI_PORT_ATTR_HW_LANE_LIST;
attr.value.u32list.count = maxLanesPerPort;
attr.value.u32list.list = lanes.data();

sai_status_t status = sai_metadata_sai_port_api->get_port_attribute(port_rid, 1, &attr);

if (status != SAI_STATUS_SUCCESS)
{
SWSS_LOG_THROW("failed to get hardware lane list port RID %s: %s",
sai_serialize_object_id(port_rid).c_str(),
sai_serialize_status(status).c_str());
}

if (attr.value.u32list.count == 0)
{
SWSS_LOG_THROW("switch returned lane count ZERO for port RID %s",
sai_serialize_object_id(port_rid).c_str());
}

lanes.resize(attr.value.u32list.count);

return lanes;
}

void SaiSwitch::redisUpdatePortLaneMap(
_In_ sai_object_id_t port_rid)
{
SWSS_LOG_ENTER();

auto lanes = saiGetPortLanes(port_rid);

for (uint32_t lane: lanes)
{
std::string strLane = sai_serialize_number(lane);
std::string strPortId = sai_serialize_object_id(port_rid);

auto key = getRedisLanesKey();

g_redisClient->hset(key, strLane, strPortId);
}

SWSS_LOG_NOTICE("added %zu lanes to redis lane map for port RID %s",
lanes.size(),
sai_serialize_object_id(port_rid).c_str());
}

void SaiSwitch::onPostPortCreate(
_In_ sai_object_id_t port_rid,
_In_ sai_object_id_t port_vid)
Expand All @@ -1282,7 +1365,8 @@ void SaiSwitch::onPostPortCreate(

m_discovered_rids.insert(discovered.begin(), discovered.end());

SWSS_LOG_NOTICE("putting ALL new discovered objects to redis");
SWSS_LOG_NOTICE("putting ALL new discovered objects to redis for port %s",
sai_serialize_object_id(port_vid).c_str());

for (sai_object_id_t rid: discovered)
{
Expand All @@ -1296,5 +1380,48 @@ void SaiSwitch::onPostPortCreate(

redisSetDummyAsicStateForRealObjectId(rid);
}

redisUpdatePortLaneMap(port_rid);
}

void SaiSwitch::onPostPortRemove(
_In_ sai_object_id_t port_rid)
{
SWSS_LOG_ENTER();

int removed = 0;

// key - lane number, value - port RID
auto map = redisGetLaneMap();

for (auto& kv: map)
{
if (kv.second == port_rid)
{
auto key = getRedisLanesKey();

std::string strLane = sai_serialize_number(kv.first);

g_redisClient->hdel(key, strLane);

removed++;
}
}

SWSS_LOG_NOTICE("removed %u lanes from redis lane map for port RID %s",
removed,
sai_serialize_object_id(port_rid).c_str());

if (removed == 0)
{
SWSS_LOG_THROW("NO LANES found in redis lane map for given port RID %s",
sai_serialize_object_id(port_rid).c_str());
}
}

bool SaiSwitch::isWarmBoot() const
{
SWSS_LOG_ENTER();

return m_warmBoot;
}
Loading

0 comments on commit 1f4a1d7

Please sign in to comment.