One more commit to autoscaling

ydb/core/base/hive.h

@@ -880,7 +880,14 @@ namespace NKikimr {
             NKikimrHive::TEvResponseTabletDistribution, EvResponseTabletDistribution> {};
 
         struct TEvRequestScaleRecommendation : TEventPB<TEvRequestScaleRecommendation,
-            NKikimrHive::TEvRequestScaleRecommendation, EvRequestScaleRecommendation> {};
+            NKikimrHive::TEvRequestScaleRecommendation, EvRequestScaleRecommendation> {
+
+            TEvRequestScaleRecommendation() = default;
+
+            TEvRequestScaleRecommendation(TSubDomainKey domainKey) {
+                Record.MutableDomainKey()->CopyFrom(domainKey);
+            }
+        };
 
         struct TEvResponseScaleRecommendation : TEventPB<TEvResponseScaleRecommendation,
             NKikimrHive::TEvResponseScaleRecommendation, EvResponseScaleRecommendation> {};

ydb/core/mind/hive/domain_info.h

@@ -3,8 +3,6 @@
 #include "hive.h"
 #include "metrics.h"
 
-#include <library/cpp/containers/ring_buffer/ring_buffer.h>
-
 namespace NKikimr {
 namespace NHive {
 
@@ -27,7 +25,7 @@ struct TDomainInfo {
     ui64 TabletsAlive = 0;
     ui64 TabletsAliveInObjectDomain = 0;
 
-    TStaticRingBuffer<double, 20> AvgCpuUsageHistory;
+    std::deque<double> AvgCpuUsageHistory;
     TMaybeFail<TScaleRecommendation> LastScaleRecommendation;
 
     ENodeSelectionPolicy GetNodeSelectionPolicy() const;

ydb/core/mind/hive/hive_impl.cpp

@@ -3468,46 +3468,117 @@ void THive::Handle(TEvPrivate::TEvUpdateDataCenterFollowers::TPtr& ev) {
     Execute(CreateUpdateDcFollowers(ev->Get()->DataCenter));
 }
 
-void THive::MakeScaleRecommendation() {
-    BLOG_D("THive::MakeScaleRecommendation()");
+template<typename It>
+ui32 THive::CalculateRecommendedNodes(It windowBegin, It windowEnd, size_t readyNodes, double target) {
+    double maxOnWindow = *std::max_element(windowBegin, windowEnd);
+    double ratio = maxOnWindow / target;
+    return std::ceil(readyNodes * ratio);
+}
 
-    std::unordered_map<TSubDomainKey, double> subdomainToCpuUsageSum;
-    std::unordered_map<TSubDomainKey, size_t> subdomainToReadyNodesCount;
-    for (auto& [_, node] : Nodes) {
+void THive::MakeScaleRecommendation() {
+    BLOG_D("[MSR] Started making scale recommendation");
+
+    // TODO(pixcc): make following variables as configurable settings
+    constexpr TDuration NODE_INITILIZATION_TIME = TDuration::Seconds(30);
+    constexpr size_t MAX_HISTORY_SIZE = 15;
+    constexpr size_t SCALE_IN_WINDOW_SIZE = 5;
+    constexpr size_t SCALE_OUT_WINDOW_SIZE = 15;
+    constexpr double TARGET_AVG_CPU_USAGE_PERCENT = 0.66;
+    constexpr double CPU_USAGE_MARGIN = 0.2;
+
+    double cpuUsageSum = 0;
+    size_t readyNodesCount = 0;
+    for (auto& [id, node] : Nodes) {
         if (!node.IsAlive()) {
+            BLOG_TRACE("[MSR] Skip node " << id << ", not alive");
             continue;
         }
 
-        if (node.StartTime + TDuration::Seconds(30) < TActivationContext::Now()) {
+        if (node.StartTime + NODE_INITILIZATION_TIME > TActivationContext::Now()) {
+            BLOG_TRACE("[MSR] Skip node " << id << ", in initialization");
             continue;
         }
 
         if (!node.AveragedNodeTotalCpuUsage.IsValueReady()) {
+            BLOG_TRACE("[MSR] Skip node " << id << ", no CPU usage value");
+            continue;
+        }
+
+        if (node.GetServicedDomain() != GetMySubDomainKey()) {
+            BLOG_TRACE("[MSR] Skip node " << id << ", serviced domain doesn't match");
             continue;
         }
 
-        const auto nodeServicedDomain = node.GetServicedDomain();
-        ++subdomainToReadyNodesCount[nodeServicedDomain];
-        subdomainToCpuUsageSum[nodeServicedDomain] += node.AveragedNodeTotalCpuUsage.GetValue();
+        double avgCpuUsage = node.AveragedNodeTotalCpuUsage.GetValue();
+        BLOG_TRACE("[MSR] Node " << id << " is ready, Avg CPU Usage " << avgCpuUsage);
+        ++readyNodesCount;
+
+        cpuUsageSum += avgCpuUsage;
         node.AveragedNodeTotalCpuUsage.Clear();
     }
 
-    for (auto& [domainKey, domainInfo] : Domains) {
-        // if (autoscaling_enabled)
-
-        double cpuUsageSum = subdomainToCpuUsageSum[domainKey];
-        size_t readyNodes = subdomainToReadyNodesCount[domainKey];
-        // if (readyNodes == 0) {
-
-        // }
-        double avgCpuUsage = cpuUsageSum / readyNodes;
-        domainInfo.AvgCpuUsageHistory.PushBack(avgCpuUsage);
+    auto& domain = Domains[GetMySubDomainKey()];
+    auto& avgCpuUsageHistory = domain.AvgCpuUsageHistory;
+
+    double avgCpuUsage = readyNodesCount != 0 ? cpuUsageSum / readyNodesCount : 0;
+    BLOG_D("[MSR] Total Avg CPU Usage " << avgCpuUsage);
+
+    avgCpuUsageHistory.push_back(avgCpuUsage);
+    if (avgCpuUsageHistory.size() > MAX_HISTORY_SIZE) {
+        avgCpuUsageHistory.pop_front();
     }
 
-    Domains[GetMySubDomainKey()].LastScaleRecommendation = TScaleRecommendation{
-        .Nodes = TAppData::RandomProvider.Get()->Uniform(100),
-        .Timestamp = TActivationContext::Now()
-    };
+    ui32 recommendedNodes = 0;
+
+    if (avgCpuUsageHistory.size() >= SCALE_IN_WINDOW_SIZE) {
+        auto scaleInWindowBegin = avgCpuUsageHistory.end() - SCALE_IN_WINDOW_SIZE;
+        auto scaleInWindowEnd = avgCpuUsageHistory.end();
+        double usageBottomThreshold = TARGET_AVG_CPU_USAGE_PERCENT - CPU_USAGE_MARGIN;
+
+        bool needScaleIn = std::all_of(
+            scaleInWindowBegin,
+            scaleInWindowEnd,
+            [usageBottomThreshold](double value){ return value < usageBottomThreshold; }
+        );
+
+        if (needScaleIn) {
+            recommendedNodes = CalculateRecommendedNodes(
+                scaleInWindowBegin,
+                scaleInWindowEnd,
+                readyNodesCount,
+                TARGET_AVG_CPU_USAGE_PERCENT
+            );
+            BLOG_D("[MSR] Need scale in: " << readyNodesCount << " -> " << recommendedNodes);
+        }
+    }
+
+    if (avgCpuUsageHistory.size() >= SCALE_OUT_WINDOW_SIZE) {
+        auto scaleOutWindowBegin = avgCpuUsageHistory.end() - SCALE_OUT_WINDOW_SIZE;
+        auto scaleOutWindowEnd = avgCpuUsageHistory.end();
+
+        bool needScaleOut = std::all_of(
+            scaleOutWindowBegin,
+            scaleOutWindowEnd,
+            [](double value){ return value > TARGET_AVG_CPU_USAGE_PERCENT; }
+        );
+
+        if (needScaleOut) {
+            recommendedNodes = CalculateRecommendedNodes(
+                scaleOutWindowBegin,
+                scaleOutWindowEnd,
+                readyNodesCount,
+                TARGET_AVG_CPU_USAGE_PERCENT
+            );
+            BLOG_D("[MSR] Need scale out: " << readyNodesCount << " -> " << recommendedNodes);
+        }
+    }
+
+    if (recommendedNodes != 0) {
+        domain.LastScaleRecommendation = TScaleRecommendation{
+            .Nodes = recommendedNodes,
+            .Timestamp = TActivationContext::Now()
+        };
+    }
 
     Schedule(GetScaleRecommendationRefreshFrequency(), new TEvPrivate::TEvRefreshScaleRecommendation());
 }

ydb/core/mind/hive/hive_impl.h

@@ -1057,6 +1057,8 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId
     TString GetDomainName(TSubDomainKey domain);
     TSubDomainKey GetMySubDomainKey() const;
 
+    template<typename It>
+    static ui32 CalculateRecommendedNodes(It windowBegin, It windowEnd, size_t readyNodes, double target);
     void MakeScaleRecommendation();
 };
 

ydb/core/mind/hive/hive_ut.cpp

@@ -6726,36 +6726,6 @@ Y_UNIT_TEST_SUITE(THiveTest) {
             MakeSureTabletIsUp(runtime, tabletId, 0);
         }
     }
-
-    void RefreshScaleRecommendation(TTestBasicRuntime& runtime) {
-        runtime.AdvanceCurrentTime(TDuration::Minutes(1));
-        TDispatchOptions options;
-        options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(NHive::TEvPrivate::EvRefreshScaleRecommendation));
-        runtime.DispatchEvents(options);
-    }
-
-    Y_UNIT_TEST(TestScaleRecommendation) {
-        TTestBasicRuntime runtime(1, false);
-        Setup(runtime, true);
-        TVector<ui64> tabletIds;
-        TActorId sender = runtime.AllocateEdgeActor();
-        const ui64 hiveTablet = MakeDefaultHiveID();
-        CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::Hive), &CreateDefaultHive);
-
-        RefreshScaleRecommendation(runtime);
-
-        {
-            TDispatchOptions options;
-            options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvLocal::EvTabletMetricsAck));
-            runtime.DispatchEvents(options);
-        }
-
-        runtime.SendToPipe(hiveTablet, sender, new TEvInterconnect::TEvNodeDisconnected(runtime.GetNodeId(0)));
-        SendKillLocal(runtime, 0);
-        runtime.Register(CreateTabletKiller(hiveTablet));
-
-        CreateLocal(runtime, 0);
-    }
 }
 
 Y_UNIT_TEST_SUITE(TStorageBalanceTest) {
@@ -7025,4 +6995,87 @@ Y_UNIT_TEST_SUITE(TStorageBalanceTest) {
         UNIT_ASSERT_LE(bsc.GetOccupancyStDev("def1"), 0.1);
     }
 }
+
+Y_UNIT_TEST_SUITE(TScaleRecommenderTest) {
+    using namespace NTestSuiteTHiveTest;
+
+    void AssertScaleRecommencation(TTestBasicRuntime& runtime, NKikimrProto::EReplyStatus expectedStatus, ui32 expectedNodes = 0) {
+        const auto sender = runtime.AllocateEdgeActor();
+
+        const auto hiveId = MakeDefaultHiveID();
+        const TSubDomainKey subdomainKey(TTestTxConfig::SchemeShard, 1);
+        runtime.SendToPipe(hiveId, sender, new TEvHive::TEvRequestScaleRecommendation(subdomainKey));
+
+        TAutoPtr<IEventHandle> handle;
+        const auto* response = runtime.GrabEdgeEventRethrow<TEvHive::TEvResponseScaleRecommendation>(handle);
+        UNIT_ASSERT_VALUES_EQUAL(response->Record.GetStatus(), expectedStatus);
+        if (expectedNodes) {
+            UNIT_ASSERT_VALUES_EQUAL(response->Record.GetRecommendedNodes(), expectedNodes);
+        }
+    }
+
+    void WaitRefreshScaleRecommendation(TTestBasicRuntime& runtime, size_t count = 1) {
+        for (size_t i = 0; i < count; ++i) {
+            runtime.AdvanceCurrentTime(TDuration::Minutes(1));
+            TDispatchOptions options;
+            options.FinalEvents.emplace_back(NHive::TEvPrivate::EvRefreshScaleRecommendation);
+            runtime.DispatchEvents(options);
+        }
+    }
+
+    TTestActorRuntime::TEventObserver SetCpuUsageOnNode(TTestBasicRuntime& runtime, double cpuUsage) {
+        return runtime.SetObserverFunc([cpuUsage](TAutoPtr<IEventHandle>& event) {
+            if (event->GetTypeRewrite() == TEvHive::EvTabletMetrics) {
+                auto& record = event->Get<TEvHive::TEvTabletMetrics>()->Record;
+                record.SetTotalNodeCpuUsage(cpuUsage);
+            }
+            return TTestActorRuntime::EEventAction::PROCESS;
+        });
+    }
+
+    constexpr double LOW_CPU_USAGE = 0.2;
+    constexpr double HIGH_CPU_USAGE = 0.95;
+
+    Y_UNIT_TEST(BasicTest) {
+        // Setup test runtime with single node
+        TTestBasicRuntime runtime(1, false);
+        Setup(runtime, true);
+
+        // Setup hive
+        const auto hiveTablet = MakeDefaultHiveID();
+        CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::Hive), &CreateDefaultHive);
+        MakeSureTabletIsUp(runtime, hiveTablet, 0);
+        CreateLocal(runtime, 0);
+
+        // No data yet
+        AssertScaleRecommencation(runtime, NKikimrProto::NOTREADY);
+
+        // Set low CPU usage on Node
+        SetCpuUsageOnNode(runtime, LOW_CPU_USAGE);
+
+        // Need time to fill window with data
+        WaitRefreshScaleRecommendation(runtime, 15);
+
+        // Check scale recommendation for low CPU usage
+        AssertScaleRecommencation(runtime, NKikimrProto::OK, 1);
+
+        // Set high CPU usage on Node
+        SetCpuUsageOnNode(runtime, HIGH_CPU_USAGE);
+
+        // Need time to fill window with new data
+        WaitRefreshScaleRecommendation(runtime, 15);
+
+        // Check scale recommendation for high CPU usage
+        AssertScaleRecommencation(runtime, NKikimrProto::OK, 2);
+    }
+
+    Y_UNIT_TEST(NodeTest) {
+        // runtime.SendToPipe(hiveTablet, sender, new TEvInterconnect::TEvNodeDisconnected(runtime.GetNodeId(0)));
+        // SendKillLocal(runtime, 0);
+        // runtime.Register(CreateTabletKiller(hiveTablet));
+
+        // CreateLocal(runtime, 0);
+    }
+}
+
 }