Start tablets in object domain KIKIMR-20271 (ydb-platform#705)

* Start tablets in object domain KIKIMR-20271

ydb/core/base/hive.h

@@ -47,6 +47,7 @@ namespace NKikimr {
             EvRequestTabletOwners,
             EvReassignOnDecommitGroup,
             EvUpdateTabletsObject,
+            EvUpdateDomain,
 
             // replies
             EvBootTabletReply = EvBootTablet + 512,
@@ -80,6 +81,7 @@ namespace NKikimr {
             EvInvalidateStoragePoolsReply,
             EvReassignOnDecommitGroupReply,
             EvUpdateTabletsObjectReply,
+            EvUpdateDomainReply,
 
             EvEnd
         };
@@ -862,6 +864,10 @@ namespace NKikimr {
                 Record.SetStatus(status);
             }
         };
+
+        struct TEvUpdateDomain : TEventPB<TEvUpdateDomain, NKikimrHive::TEvUpdateDomain, EvUpdateDomain> {};
+
+        struct TEvUpdateDomainReply : TEventPB<TEvUpdateDomainReply, NKikimrHive::TEvUpdateDomainReply, EvUpdateDomainReply> {};
     };
 
     IActor* CreateDefaultHive(const TActorId &tablet, TTabletStorageInfo *info);

ydb/core/base/subdomain.h

@@ -2,6 +2,7 @@
 
 #include <ydb/core/protos/subdomains.pb.h>
 
+#include <util/generic/maybe.h>
 #include <util/system/types.h>
 #include <utility>
 
@@ -29,6 +30,8 @@ struct TSubDomainKey : public std::pair<ui64, ui64> {
 };
 
 static const TSubDomainKey InvalidSubDomainKey = TSubDomainKey();
+
+using TMaybeServerlessComputeResourcesMode = TMaybe<NKikimrSubDomains::EServerlessComputeResourcesMode, NMaybe::TPolicyUndefinedFail>;
 }
 
 template <>

ydb/core/mind/hive/domain_info.cpp

@@ -0,0 +1,22 @@
+#include "domain_info.h"
+
+namespace NKikimr {
+namespace NHive {
+
+ENodeSelectionPolicy TDomainInfo::GetNodeSelectionPolicy() const {
+    if (!ServerlessComputeResourcesMode) {
+        return ENodeSelectionPolicy::Default;
+    }
+
+    switch (*ServerlessComputeResourcesMode) {
+        case NKikimrSubDomains::SERVERLESS_COMPUTE_RESOURCES_MODE_DEDICATED:
+            return ENodeSelectionPolicy::PreferObjectDomain;
+        case NKikimrSubDomains::SERVERLESS_COMPUTE_RESOURCES_MODE_SHARED:
+            return ENodeSelectionPolicy::Default;
+        default:
+            return ENodeSelectionPolicy::Default;
+    }
+}
+
+} // NHive
+} // NKikimr

ydb/core/mind/hive/domain_info.h

@@ -5,10 +5,18 @@
 namespace NKikimr {
 namespace NHive {
 
+enum class ENodeSelectionPolicy : ui32 {
+    Default,
+    PreferObjectDomain,
+};
+
 struct TDomainInfo {
     TString Path;
     TTabletId HiveId = 0;
+    TMaybeServerlessComputeResourcesMode ServerlessComputeResourcesMode;
+
+    ENodeSelectionPolicy GetNodeSelectionPolicy() const;
 };
 
-}
-}
+} // NHive
+} // NKikimr

ydb/core/mind/hive/follower_group.h

@@ -17,7 +17,10 @@ struct TFollowerGroup {
     bool RequireDifferentNodes = false; // do not run followers on same nodes as another followers of the same leader
     bool FollowerCountPerDataCenter = false; // PER_AZ KIKIMR-10443
 
-    TFollowerGroup() = default;
+    explicit TFollowerGroup(const THive& hive) 
+        : NodeFilter(hive)
+    {}
+
     TFollowerGroup(const TFollowerGroup&) = delete;
     TFollowerGroup(TFollowerGroup&&) = delete;
     TFollowerGroup& operator =(const TFollowerGroup&) = delete;

ydb/core/mind/hive/hive.cpp

@@ -1,4 +1,7 @@
+#include "domain_info.h"
 #include "hive.h"
+#include "hive_impl.h"
+#include "leader_tablet_info.h"
 
 #include <ydb/core/util/tuples.h>
 
@@ -76,5 +79,25 @@ NMetrics::EResource GetDominantResourceType(const TResourceNormalizedValues& nor
     }
     return dominant;
 }
+
+TNodeFilter::TNodeFilter(const THive& hive) 
+    : Hive(hive) 
+{}
+
+TArrayRef<const TSubDomainKey> TNodeFilter::GetEffectiveAllowedDomains() const {
+    const auto* objectDomainInfo = Hive.FindDomain(ObjectDomain);
+
+    if (!objectDomainInfo) {
+        return {AllowedDomains.begin(), AllowedDomains.end()};
+    }
+
+    switch (objectDomainInfo->GetNodeSelectionPolicy()) {
+        case ENodeSelectionPolicy::Default:
+            return {AllowedDomains.begin(), AllowedDomains.end()};
+        case ENodeSelectionPolicy::PreferObjectDomain:
+            return {&ObjectDomain, 1};
+    }
 }
-}
+
+} // NHive
+} // NKikimr

ydb/core/mind/hive/hive.h

@@ -275,6 +275,13 @@ struct TNodeFilter {
     TVector<TSubDomainKey> AllowedDomains;
     TVector<TNodeId> AllowedNodes;
     TVector<TDataCenterId> AllowedDataCenters;
+    TSubDomainKey ObjectDomain;
+
+    const THive& Hive;
+
+    explicit TNodeFilter(const THive& hive);
+
+    TArrayRef<const TSubDomainKey> GetEffectiveAllowedDomains() const;
 };
 
 } // NHive

ydb/core/mind/hive/hive_domains.cpp

@@ -62,6 +62,18 @@ void THive::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) {
     }
 }
 
+void THive::Handle(TEvHive::TEvUpdateDomain::TPtr& ev) {
+    BLOG_D("Handle TEvHive::TEvUpdateDomain(" << ev->Get()->Record.ShortDebugString() << ")");
+    const TSubDomainKey subdomainKey(ev->Get()->Record.GetDomainKey());
+    TDomainInfo& domainInfo = Domains[subdomainKey];
+    if (ev->Get()->Record.GetServerlessComputeResourcesMode() != NKikimrSubDomains::SERVERLESS_COMPUTE_RESOURCES_MODE_UNSPECIFIED) {
+        domainInfo.ServerlessComputeResourcesMode = ev->Get()->Record.GetServerlessComputeResourcesMode();
+    } else {
+        domainInfo.ServerlessComputeResourcesMode.Clear();
+    }
+    Execute(CreateUpdateDomain(subdomainKey, std::move(ev)));
+}
+
 TString THive::GetDomainName(TSubDomainKey domain) {
     auto itDomain = Domains.find(domain);
     if (itDomain != Domains.end()) {

ydb/core/mind/hive/hive_impl.cpp

@@ -10,16 +10,18 @@
 #include <library/cpp/time_provider/time_provider.h>
 #include <util/generic/array_ref.h>
 
-template <>
-inline IOutputStream& operator <<(IOutputStream& out, const TArrayRef<const NKikimrHive::TDataCentersGroup*>& vec) {
+Y_DECLARE_OUT_SPEC(inline, TArrayRef<const NKikimrHive::TDataCentersGroup*>, out, vec) {
     out << '[';
     for (auto it = vec.begin(); it != vec.end(); ++it) {
         if (it != vec.begin())
             out << ';';
         out << (*it)->ShortDebugString();
     }
     out << ']';
-    return out;
+}
+
+Y_DECLARE_OUT_SPEC(inline, TArrayRef<const NKikimr::TSubDomainKey>, out, vec) {
+    out << '[' << JoinSeq(',', vec) << ']';
 }
 
 namespace NKikimr {
@@ -1268,7 +1270,8 @@ THive::TBestNodeResult THive::FindBestNode(const TTabletInfo& tablet) {
                             << " to run the tablet " << tablet.ToString()
                             << " node domains " << nodeInfo.ServicedDomains
                             << " tablet object domain " << tablet.GetLeader().ObjectDomain
-                            << " tablet allowed domains " << tablet.GetNodeFilter().AllowedDomains);
+                            << " tablet allowed domains " << tablet.GetNodeFilter().AllowedDomains
+                            << " tablet effective allowed domains " << tablet.GetNodeFilter().GetEffectiveAllowedDomains());
             }
         }
         if (!selectedNodes.empty()) {
@@ -1284,6 +1287,7 @@ THive::TBestNodeResult THive::FindBestNode(const TTabletInfo& tablet) {
     TNodeInfo* selectedNode = nullptr;
     if (!selectedNodes.empty()) {
         selectedNodes = SelectMaxPriorityNodes(std::move(selectedNodes), tablet);
+        BLOG_TRACE("[FBN] Tablet " << tablet.ToString() << " selected max priority nodes count " << selectedNodes.size());
 
         switch (GetNodeSelectStrategy()) {
             case NKikimrConfig::THiveConfig::HIVE_NODE_SELECT_STRATEGY_WEIGHTED_RANDOM:
@@ -1336,7 +1340,7 @@ THive::TBestNodeResult THive::FindBestNode(const TTabletInfo& tablet) {
         }
         nodesLeft -= debugState.NodesWithSomeoneFromOurFamily;
         if (debugState.NodesWithoutDomain == nodesLeft) {
-            tablet.BootState = TStringBuilder() << "Can't find domain " << tablet.GetNodeFilter().AllowedDomains;
+            tablet.BootState = TStringBuilder() << "Can't find domain " << tablet.GetNodeFilter().GetEffectiveAllowedDomains();
             return TBestNodeResult(true);
         }
         nodesLeft -= debugState.NodesWithoutDomain;
@@ -1486,6 +1490,14 @@ TDomainInfo* THive::FindDomain(TSubDomainKey key) {
     return &it->second;
 }
 
+const TDomainInfo* THive::FindDomain(TSubDomainKey key) const {
+    auto it = Domains.find(key);
+    if (it == Domains.end()) {
+        return nullptr;
+    }
+    return &it->second;
+}
+
 void THive::DeleteTablet(TTabletId tabletId) {
     auto it = Tablets.find(tabletId);
     if (it != Tablets.end()) {
@@ -2813,6 +2825,7 @@ void THive::ProcessEvent(std::unique_ptr<IEventHandle> event) {
         hFunc(TEvHive::TEvUpdateTabletsObject, Handle);
         hFunc(TEvPrivate::TEvRefreshStorageInfo, Handle);
         hFunc(TEvPrivate::TEvLogTabletMoves, Handle);
+        hFunc(TEvHive::TEvUpdateDomain, Handle);
     }
 }
 
@@ -2910,6 +2923,7 @@ STFUNC(THive::StateWork) {
         fFunc(TEvHive::TEvUpdateTabletsObject::EventType, EnqueueIncomingEvent);
         fFunc(TEvPrivate::TEvRefreshStorageInfo::EventType, EnqueueIncomingEvent);
         fFunc(TEvPrivate::TEvLogTabletMoves::EventType, EnqueueIncomingEvent);
+        fFunc(TEvHive::TEvUpdateDomain::EventType, EnqueueIncomingEvent);
         hFunc(TEvPrivate::TEvProcessIncomingEvent, Handle);
     default:
         if (!HandleDefaultEvents(ev, SelfId())) {

ydb/core/mind/hive/hive_impl.h

@@ -279,7 +279,6 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
     ITransaction* CreateDisconnectNode(THolder<TEvInterconnect::TEvNodeDisconnected> event);
     ITransaction* CreateProcessPendingOperations();
     ITransaction* CreateProcessBootQueue();
-    ITransaction* CreateUpdateDomain(TSubDomainKey subdomainKey);
     ITransaction* CreateSeizeTablets(TEvHive::TEvSeizeTablets::TPtr event);
     ITransaction* CreateSeizeTabletsReply(TEvHive::TEvSeizeTabletsReply::TPtr event);
     ITransaction* CreateReleaseTablets(TEvHive::TEvReleaseTablets::TPtr event);
@@ -290,6 +289,7 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
     ITransaction* CreateTabletOwnersReply(TEvHive::TEvTabletOwnersReply::TPtr event);
     ITransaction* CreateRequestTabletOwners(TEvHive::TEvRequestTabletOwners::TPtr event);
     ITransaction* CreateUpdateTabletsObject(TEvHive::TEvUpdateTabletsObject::TPtr event);
+    ITransaction* CreateUpdateDomain(TSubDomainKey subdomainKey, TEvHive::TEvUpdateDomain::TPtr event = {});
 
 public:
     TDomainsView DomainsView;
@@ -549,6 +549,7 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
     void Handle(TEvPrivate::TEvRefreshStorageInfo::TPtr& ev);
     void Handle(TEvPrivate::TEvLogTabletMoves::TPtr& ev);
     void Handle(TEvPrivate::TEvProcessIncomingEvent::TPtr& ev);
+    void Handle(TEvHive::TEvUpdateDomain::TPtr& ev);
 
 protected:
     void RestartPipeTx(ui64 tabletId);
@@ -618,6 +619,7 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
     TStoragePoolInfo& GetStoragePool(const TString& name);
     TStoragePoolInfo* FindStoragePool(const TString& name);
     TDomainInfo* FindDomain(TSubDomainKey key);
+    const TDomainInfo* FindDomain(TSubDomainKey key) const;
     const TNodeLocation& GetNodeLocation(TNodeId nodeId) const;
     void DeleteTablet(TTabletId tabletId);
     void DeleteNode(TNodeId nodeId);

ydb/core/mind/hive/hive_schema.h

@@ -267,9 +267,10 @@ struct Schema : NIceDb::Schema {
         struct Path : Column<3, NScheme::NTypeIds::Utf8> {};
         struct Primary : Column<4, NScheme::NTypeIds::Bool> {};
         struct HiveId : Column<5, NScheme::NTypeIds::Uint64> {};
+        struct ServerlessComputeResourcesMode : Column<6, NScheme::NTypeIds::Uint32> { using Type = NKikimrSubDomains::EServerlessComputeResourcesMode; };
 
         using TKey = TableKey<SchemeshardId, PathId>;
-        using TColumns = TableColumns<SchemeshardId, PathId, Path, Primary, HiveId>;
+        using TColumns = TableColumns<SchemeshardId, PathId, Path, Primary, HiveId, ServerlessComputeResourcesMode>;
     };
 
     struct BlockedOwner : Table<18> {