From dcab09f2881a0b45a2c03c091e641ff66dfada8c Mon Sep 17 00:00:00 2001 From: Wenda Ni Date: Sat, 25 May 2019 18:09:57 -0700 Subject: [PATCH] Add buffer pool stat to flex counter architecture (#451) --- meta/sai_serialize.h | 4 + meta/saiserialize.cpp | 9 ++ syncd/syncd.cpp | 28 ++++ syncd/syncd_flex_counter.cpp | 255 +++++++++++++++++++++++++++++++++-- syncd/syncd_flex_counter.h | 25 ++++ 5 files changed, 308 insertions(+), 13 deletions(-) diff --git a/meta/sai_serialize.h b/meta/sai_serialize.h index eb3bfd66a768..4f2ee5d261e0 100644 --- a/meta/sai_serialize.h +++ b/meta/sai_serialize.h @@ -277,6 +277,10 @@ void sai_deserialize_ingress_priority_group_attr( _In_ const std::string& s, _Out_ sai_ingress_priority_group_attr_t& attr); +void sai_deserialize_buffer_pool_stat( + _In_ const std::string& s, + _Out_ sai_buffer_pool_stat_t& stat); + void sai_deserialize_queue_attr( _In_ const std::string& s, _Out_ sai_queue_attr_t& attr); diff --git a/meta/saiserialize.cpp b/meta/saiserialize.cpp index 605da1d6ff95..881452a270c4 100644 --- a/meta/saiserialize.cpp +++ b/meta/saiserialize.cpp @@ -2968,6 +2968,15 @@ void sai_deserialize_ingress_priority_group_attr( sai_deserialize_enum(s, &sai_metadata_enum_sai_ingress_priority_group_attr_t, (int32_t&)attr); } +void sai_deserialize_buffer_pool_stat( + _In_ const std::string& s, + _Out_ sai_buffer_pool_stat_t& stat) +{ + SWSS_LOG_ENTER(); + + sai_deserialize_enum(s, &sai_metadata_enum_sai_buffer_pool_stat_t, (int32_t&)stat); +} + void sai_deserialize_queue_attr( _In_ const std::string& s, _Out_ sai_queue_attr_t& attr) diff --git a/syncd/syncd.cpp b/syncd/syncd.cpp index 4a4e59623bee..b0f204fc4098 100644 --- a/syncd/syncd.cpp +++ b/syncd/syncd.cpp @@ -2831,6 +2831,14 @@ void processFlexCounterGroupEvent( FlexCounter::addPortCounterPlugin(sha, groupName); } } + else if (field == BUFFER_POOL_PLUGIN_FIELD) + { + auto shaStrings = swss::tokenize(value, ','); + for (const auto &sha : shaStrings) + { + FlexCounter::addBufferPoolCounterPlugin(sha, groupName); + } + } else if (field == FLEX_COUNTER_STATUS_FIELD) { FlexCounter::updateFlexCounterStatus(value, groupName); @@ -2894,6 +2902,10 @@ void processFlexCounterEvent( { FlexCounter::removeRif(vid, groupName); } + else if (objectType == SAI_OBJECT_TYPE_BUFFER_POOL) + { + FlexCounter::removeBufferPool(vid, groupName); + } else { SWSS_LOG_ERROR("Object type for removal not supported, %s", objectTypeStr.c_str()); @@ -2919,6 +2931,7 @@ void processFlexCounterEvent( sai_deserialize_port_stat(str.c_str(), &stat); portCounterIds.push_back(stat); } + FlexCounter::setPortCounterList(vid, rid, groupName, portCounterIds); } else if (objectType == SAI_OBJECT_TYPE_QUEUE && field == QUEUE_COUNTER_ID_LIST) @@ -2930,6 +2943,7 @@ void processFlexCounterEvent( sai_deserialize_queue_stat(str.c_str(), &stat); queueCounterIds.push_back(stat); } + FlexCounter::setQueueCounterList(vid, rid, groupName, queueCounterIds); } else if (objectType == SAI_OBJECT_TYPE_QUEUE && field == QUEUE_ATTR_ID_LIST) @@ -2953,6 +2967,7 @@ void processFlexCounterEvent( sai_deserialize_ingress_priority_group_stat(str.c_str(), &stat); pgCounterIds.push_back(stat); } + FlexCounter::setPriorityGroupCounterList(vid, rid, groupName, pgCounterIds); } else if (objectType == SAI_OBJECT_TYPE_INGRESS_PRIORITY_GROUP && field == PG_ATTR_ID_LIST) @@ -2976,8 +2991,21 @@ void processFlexCounterEvent( sai_deserialize_router_interface_stat(str.c_str(), &stat); rifCounterIds.push_back(stat); } + FlexCounter::setRifCounterList(vid, rid, groupName, rifCounterIds); } + else if (objectType == SAI_OBJECT_TYPE_BUFFER_POOL && field == BUFFER_POOL_COUNTER_ID_LIST) + { + std::vector bufferPoolCounterIds; + for (const auto &str : idStrings) + { + sai_buffer_pool_stat_t stat; + sai_deserialize_buffer_pool_stat(str, stat); + bufferPoolCounterIds.push_back(stat); + } + + FlexCounter::setBufferPoolCounterList(vid, rid, groupName, bufferPoolCounterIds); + } else { SWSS_LOG_ERROR("Object type and field combination is not supported, object type %s, field %s", objectTypeStr.c_str(), field.c_str()); diff --git a/syncd/syncd_flex_counter.cpp b/syncd/syncd_flex_counter.cpp index 52e8899d6c2a..18814a5c901c 100644 --- a/syncd/syncd_flex_counter.cpp +++ b/syncd/syncd_flex_counter.cpp @@ -9,6 +9,7 @@ static std::set supportedPortCounters; static std::set supportedQueueCounters; static std::set supportedPriorityGroupCounters; static std::set supportedRifCounters; +static std::set supportedBufferPoolCounters; FlexCounter::PortCounterIds::PortCounterIds( _In_ sai_object_id_t port, @@ -58,6 +59,14 @@ FlexCounter::RifCounterIds::RifCounterIds( SWSS_LOG_ENTER(); } +FlexCounter::BufferPoolCounterIds::BufferPoolCounterIds( + _In_ sai_object_id_t bufferPool, + _In_ const std::vector &bufferPoolIds): + bufferPoolId(bufferPool), bufferPoolCounterIds(bufferPoolIds) +{ + SWSS_LOG_ENTER(); +} + void FlexCounter::setPollInterval( _In_ uint32_t pollInterval, _In_ std::string instanceId) @@ -429,6 +438,58 @@ void FlexCounter::setRifCounterList( } } +void FlexCounter::setBufferPoolCounterList( + _In_ sai_object_id_t bufferPoolVid, + _In_ sai_object_id_t bufferPoolId, + _In_ std::string instanceId, + _In_ const std::vector &counterIds) +{ + SWSS_LOG_ENTER(); + + FlexCounter &fc = getInstance(instanceId); + + fc.saiUpdateSupportedBufferPoolCounters(bufferPoolId, counterIds); + + // Filter unsupported counters + std::vector supportedIds; + for (auto counterId : counterIds) + { + if (fc.isBufferPoolCounterSupported(counterId)) + { + supportedIds.push_back(counterId); + } + } + + if (supportedIds.size() == 0) + { + SWSS_LOG_NOTICE("Buffer pool %s does not has supported counters", sai_serialize_object_id(bufferPoolId).c_str()); + + // Remove flex counter if all counter IDs and plugins are unregistered + if (fc.isEmpty()) + { + removeInstance(instanceId); + } + + return; + } + + auto it = fc.m_bufferPoolCounterIdsMap.find(bufferPoolVid); + if (it != fc.m_bufferPoolCounterIdsMap.end()) + { + it->second->bufferPoolCounterIds = supportedIds; + return; + } + + auto bufferPoolCounterIds = std::make_shared(bufferPoolId, supportedIds); + fc.m_bufferPoolCounterIdsMap.emplace(bufferPoolVid, bufferPoolCounterIds); + + // Start flex counter thread in case it was not running due to empty counter IDs map + if (fc.m_pollInterval > 0) + { + fc.startFlexCounterThread(); + } +} + void FlexCounter::removePort( _In_ sai_object_id_t portVid, _In_ std::string instanceId) @@ -559,6 +620,35 @@ void FlexCounter::removeRif( } } +void FlexCounter::removeBufferPool( + _In_ sai_object_id_t bufferPoolVid, + _In_ std::string instanceId) +{ + SWSS_LOG_ENTER(); + + bool found = false; + FlexCounter &fc = getInstance(instanceId); + + auto it = fc.m_bufferPoolCounterIdsMap.find(bufferPoolVid); + if (it != fc.m_bufferPoolCounterIdsMap.end()) + { + fc.m_bufferPoolCounterIdsMap.erase(it); + found = true; + } + + if (!found) + { + SWSS_LOG_NOTICE("Trying to remove nonexisting buffer pool 0x%lx from flex counter %s", bufferPoolVid, fc.m_instanceId.c_str()); + return; + } + + // Remove flex counter if all counter IDs and plugins are unregistered + if (fc.isEmpty()) + { + removeInstance(instanceId); + } +} + void FlexCounter::addPortCounterPlugin( _In_ std::string sha, _In_ std::string instanceId) @@ -616,6 +706,23 @@ void FlexCounter::addPriorityGroupCounterPlugin( SWSS_LOG_NOTICE("Priority group counters plugin %s registered", sha.c_str()); } +void FlexCounter::addBufferPoolCounterPlugin( + _In_ std::string sha, + _In_ std::string instanceId) +{ + SWSS_LOG_ENTER(); + + FlexCounter &fc = getInstance(instanceId); + + if (fc.m_bufferPoolPlugins.find(sha) != fc.m_bufferPoolPlugins.end()) + { + SWSS_LOG_ERROR("Plugin %s already registered", sha.c_str()); + } + + fc.m_bufferPoolPlugins.insert(sha); + SWSS_LOG_NOTICE("Buffer pool counters plugin %s registered", sha.c_str()); +} + void FlexCounter::removeCounterPlugin( _In_ std::string sha, _In_ std::string instanceId) @@ -627,6 +734,7 @@ void FlexCounter::removeCounterPlugin( fc.m_queuePlugins.erase(sha); fc.m_portPlugins.erase(sha); fc.m_priorityGroupPlugins.erase(sha); + fc.m_bufferPoolPlugins.erase(sha); // Remove flex counter if all counter IDs and plugins are unregistered if (fc.isEmpty()) @@ -645,6 +753,7 @@ void FlexCounter::removeCounterPlugin( fc.m_queuePlugins.clear(); fc.m_portPlugins.clear(); fc.m_priorityGroupPlugins.clear(); + fc.m_bufferPoolPlugins.clear(); // Remove flex counter if all counter IDs and plugins are unregistered if (fc.isEmpty()) @@ -680,7 +789,9 @@ bool FlexCounter::isEmpty() m_rifCounterIdsMap.empty() && m_queueAttrIdsMap.empty() && m_queuePlugins.empty() && - m_portPlugins.empty(); + m_portPlugins.empty() && + m_bufferPoolCounterIdsMap.empty() && + m_bufferPoolPlugins.empty(); } bool FlexCounter::isPortCounterSupported(sai_port_stat_t counter) const @@ -711,6 +822,13 @@ bool FlexCounter::isRifCounterSupported(sai_router_interface_stat_t counter) con return supportedRifCounters.count(counter) != 0; } +bool FlexCounter::isBufferPoolCounterSupported(sai_buffer_pool_stat_t counter) const +{ + SWSS_LOG_ENTER(); + + return supportedBufferPoolCounters.count(counter) != 0; +} + FlexCounter::FlexCounter(std::string instanceId) : m_instanceId(instanceId) { SWSS_LOG_ENTER(); @@ -747,6 +865,7 @@ void FlexCounter::collectCounters( std::map> priorityGroupCounterIdsMap; std::map> priorityGroupAttrIdsMap; std::map> rifCounterIdsMap; + std::map> bufferPoolCounterIdsMap; { std::lock_guard lock(g_mutex); @@ -756,6 +875,7 @@ void FlexCounter::collectCounters( priorityGroupCounterIdsMap = m_priorityGroupCounterIdsMap; priorityGroupAttrIdsMap = m_priorityGroupAttrIdsMap; rifCounterIdsMap = m_rifCounterIdsMap; + bufferPoolCounterIdsMap = m_bufferPoolCounterIdsMap; } // Collect stats for every registered port @@ -805,14 +925,14 @@ void FlexCounter::collectCounters( // Get queue stats sai_status_t status = -1; -// TODO: replace if with get_queue_stats_ext() call when it's fully supported -// Example: -// sai_status_t status = sai_metadata_sai_queue_api->get_queue_stats_ext( -// queueId, -// static_cast(queueCounterIds.size()), -// queueCounterIds.data(), -// m_statsMode, -// queueStats.data()); + // TODO: replace if with get_queue_stats_ext() call when it is fully supported + // Example: + // sai_status_t status = sai_metadata_sai_queue_api->get_queue_stats_ext( + // queueId, + // static_cast(queueCounterIds.size()), + // queueCounterIds.data(), + // m_statsMode, + // queueStats.data()); status = sai_metadata_sai_queue_api->get_queue_stats( queueId, static_cast(queueCounterIds.size()), @@ -903,7 +1023,7 @@ void FlexCounter::collectCounters( // Get PG stats sai_status_t status = -1; -// TODO: replace if with get_ingress_priority_group_stats_ext() call when it's fully supported + // TODO: replace if with get_ingress_priority_group_stats_ext() call when it is fully supported status = sai_metadata_sai_buffer_api->get_ingress_priority_group_stats( priorityGroupId, static_cast(priorityGroupCounterIds.size()), @@ -1018,6 +1138,61 @@ void FlexCounter::collectCounters( countersTable.set(rifVidStr, values, ""); } + // Collect stats for every registered buffer pool + for (const auto &it : bufferPoolCounterIdsMap) + { + const auto &bufferPoolVid = it.first; + const auto &bufferPoolId = it.second->bufferPoolId; + const auto &bufferPoolCounterIds = it.second->bufferPoolCounterIds; + + std::vector bufferPoolStats(bufferPoolCounterIds.size()); + + // Get buffer pool stats + sai_status_t status = -1; + + status = sai_metadata_sai_buffer_api->get_buffer_pool_stats( + bufferPoolId, + static_cast(bufferPoolCounterIds.size()), + reinterpret_cast(bufferPoolCounterIds.data()), + bufferPoolStats.data()); + if (status != SAI_STATUS_SUCCESS) + { + // Because of stat pre-qualification in setting the buffer pool counter list, + // it is less likely that we will get a failure return here in polling the stats + SWSS_LOG_ERROR("%s: failed to get stats of buffer pool %s, rv: %s", + m_instanceId.c_str(), + sai_serialize_object_id(bufferPoolId).c_str(), + sai_serialize_status(status).c_str()); + continue; + } + if (m_statsMode == SAI_STATS_MODE_READ_AND_CLEAR) + { + status = sai_metadata_sai_buffer_api->clear_buffer_pool_stats( + bufferPoolId, + static_cast(bufferPoolCounterIds.size()), + reinterpret_cast(bufferPoolCounterIds.data())); + } + if (status != SAI_STATUS_SUCCESS) + { + // Because of stat pre-qualification in setting the buffer pool counter list, + // it is less likely that we will get a failure return here in clearing the stats + SWSS_LOG_ERROR("%s: failed to clear stats of buffer pool %s, rv: %s", + m_instanceId.c_str(), + sai_serialize_object_id(bufferPoolId).c_str(), + sai_serialize_status(status).c_str()); + continue; + } + + // Write counter values to DB table + std::vector fvTuples; + for (size_t i = 0; i < bufferPoolCounterIds.size(); ++i) + { + fvTuples.emplace_back(sai_serialize_buffer_pool_stat(bufferPoolCounterIds[i]), std::to_string(bufferPoolStats[i])); + } + + countersTable.set(sai_serialize_object_id(bufferPoolVid), fvTuples); + } + countersTable.flush(); } @@ -1031,9 +1206,11 @@ void FlexCounter::runPlugins( std::map> queueAttrIdsMap; std::map> priorityGroupCounterIdsMap; std::map> priorityGroupAttrIdsMap; + std::map> bufferPoolCounterIdsMap; std::set queuePlugins; std::set portPlugins; std::set priorityGroupPlugins; + std::set bufferPoolPlugins; { std::lock_guard lock(g_mutex); @@ -1045,6 +1222,8 @@ void FlexCounter::runPlugins( priorityGroupCounterIdsMap = m_priorityGroupCounterIdsMap; priorityGroupAttrIdsMap = m_priorityGroupAttrIdsMap; priorityGroupPlugins = m_priorityGroupPlugins; + bufferPoolCounterIdsMap = m_bufferPoolCounterIdsMap; + bufferPoolPlugins = m_bufferPoolPlugins; } const std::vector argv = @@ -1060,7 +1239,6 @@ void FlexCounter::runPlugins( { portList.push_back(sai_serialize_object_id(kv.first)); } - for (const auto& sha : portPlugins) { runRedisScript(db, sha, portList, argv); @@ -1072,7 +1250,6 @@ void FlexCounter::runPlugins( { queueList.push_back(sai_serialize_object_id(kv.first)); } - for (const auto& sha : queuePlugins) { runRedisScript(db, sha, queueList, argv); @@ -1084,11 +1261,21 @@ void FlexCounter::runPlugins( { priorityGroupList.push_back(sai_serialize_object_id(kv.first)); } - for (const auto& sha : priorityGroupPlugins) { runRedisScript(db, sha, priorityGroupList, argv); } + + std::vector bufferPoolVids; + bufferPoolVids.reserve(bufferPoolCounterIdsMap.size()); + for (const auto& it : bufferPoolCounterIdsMap) + { + bufferPoolVids.push_back(sai_serialize_object_id(it.first)); + } + for (const auto& sha : bufferPoolPlugins) + { + runRedisScript(db, sha, bufferPoolVids, argv); + } } void FlexCounter::flexCounterThread(void) @@ -1294,3 +1481,45 @@ void FlexCounter::saiUpdateSupportedRifCounters(sai_object_id_t rifId) supportedRifCounters.insert(counter); } } + +void FlexCounter::saiUpdateSupportedBufferPoolCounters( + _In_ sai_object_id_t bufferPoolId, + _In_ const std::vector &counterIds) +{ + SWSS_LOG_ENTER(); + + uint64_t value; + supportedBufferPoolCounters.clear(); + + for (const auto &counterId : counterIds) + { + sai_status_t status = sai_metadata_sai_buffer_api->get_buffer_pool_stats(bufferPoolId, 1, (const sai_stat_id_t *)&counterId, &value); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("%s: counter %s is not supported on buffer pool %s, rv: %s", + m_instanceId.c_str(), + sai_serialize_buffer_pool_stat(counterId).c_str(), + sai_serialize_object_id(bufferPoolId).c_str(), + sai_serialize_status(status).c_str()); + + continue; + } + + if (m_statsMode == SAI_STATS_MODE_READ_AND_CLEAR) + { + status = sai_metadata_sai_buffer_api->clear_buffer_pool_stats(bufferPoolId, 1, (const sai_stat_id_t *)&counterId); + if (status != SAI_STATUS_SUCCESS) + { + SWSS_LOG_ERROR("%s: clear counter %s is not supported on buffer pool %s, rv: %s", + m_instanceId.c_str(), + sai_serialize_buffer_pool_stat(counterId).c_str(), + sai_serialize_object_id(bufferPoolId).c_str(), + sai_serialize_status(status).c_str()); + + continue; + } + } + + supportedBufferPoolCounters.insert(counterId); + } +} diff --git a/syncd/syncd_flex_counter.h b/syncd/syncd_flex_counter.h index 233d48932ca9..8ff7e1bee3e5 100644 --- a/syncd/syncd_flex_counter.h +++ b/syncd/syncd_flex_counter.h @@ -37,6 +37,11 @@ class FlexCounter _In_ sai_object_id_t rifId, _In_ std::string instanceId, _In_ const std::vector &counterIds); + static void setBufferPoolCounterList( + _In_ sai_object_id_t bufferPoolVid, + _In_ sai_object_id_t bufferPoolId, + _In_ std::string instanceId, + _In_ const std::vector &counterIds); static void setQueueAttrList( _In_ sai_object_id_t queueVid, _In_ sai_object_id_t queueId, @@ -66,6 +71,9 @@ class FlexCounter static void removeRif( _In_ sai_object_id_t rifVid, _In_ std::string instanceId); + static void removeBufferPool( + _In_ sai_object_id_t bufferPoolVid, + _In_ std::string instanceId); static void addPortCounterPlugin( _In_ std::string sha, @@ -76,6 +84,9 @@ class FlexCounter static void addPriorityGroupCounterPlugin( _In_ std::string sha, _In_ std::string instanceId); + static void addBufferPoolCounterPlugin( + _In_ std::string sha, + _In_ std::string instanceId); static void removeCounterPlugin( _In_ std::string sha, _In_ std::string instanceId); @@ -128,6 +139,16 @@ class FlexCounter std::vector priorityGroupAttrIds; }; + struct BufferPoolCounterIds + { + BufferPoolCounterIds( + _In_ sai_object_id_t bufferPool, + _In_ const std::vector &bufferPoolIds); + + sai_object_id_t bufferPoolId; + std::vector bufferPoolCounterIds; + }; + struct PortCounterIds { PortCounterIds( @@ -162,10 +183,12 @@ class FlexCounter void saiUpdateSupportedQueueCounters(sai_object_id_t queueId, const std::vector &counterIds); void saiUpdateSupportedPriorityGroupCounters(sai_object_id_t priorityGroupId, const std::vector &counterIds); void saiUpdateSupportedRifCounters(sai_object_id_t rifId); + void saiUpdateSupportedBufferPoolCounters(sai_object_id_t bufferPoolId, const std::vector &counterIds); bool isPortCounterSupported(sai_port_stat_t counter) const; bool isQueueCounterSupported(sai_queue_stat_t counter) const; bool isPriorityGroupCounterSupported(sai_ingress_priority_group_stat_t counter) const; bool isRifCounterSupported(sai_router_interface_stat_t counter) const; + bool isBufferPoolCounterSupported(sai_buffer_pool_stat_t counter) const; bool isEmpty(); // Key is a Virtual ID @@ -175,11 +198,13 @@ class FlexCounter std::map> m_priorityGroupCounterIdsMap; std::map> m_priorityGroupAttrIdsMap; std::map> m_rifCounterIdsMap; + std::map> m_bufferPoolCounterIdsMap; // Plugins std::set m_queuePlugins; std::set m_portPlugins; std::set m_priorityGroupPlugins; + std::set m_bufferPoolPlugins; std::atomic_bool m_runFlexCounterThread = { false }; std::shared_ptr m_flexCounterThread = nullptr;