Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

deprioritise system tablets in balancer (#6840) #7047

Merged
merged 1 commit into from
Jul 25, 2024
Merged

deprioritise system tablets in balancer (#6840) #7047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 32 additions & 19 deletions ydb/core/mind/hive/balancer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,17 +62,17 @@ void BalanceNodes<NKikimrConfig::THiveConfig::HIVE_NODE_BALANCE_STRATEGY_RANDOM>
}

template<>
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(std::vector<TTabletInfo*>& tablets, EResourceToBalance resourceToBalance) {
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance resourceToBalance) {
auto& randGen = *TAppData::RandomProvider.Get();
// weighted random shuffle
std::vector<double> weights;
weights.reserve(tablets.size());
for (auto it = tablets.begin(); it != tablets.end(); ++it) {
weights.reserve(last - first);
for (auto it = first; it != last; ++it) {
weights.emplace_back((*it)->GetWeight(resourceToBalance));
}
auto itT = tablets.begin();
auto itT = first;
auto itW = weights.begin();
while (itT != tablets.end() && itW != weights.end()) {
while (itT != last && itW != weights.end()) {
auto idx = std::discrete_distribution(itW, weights.end())(randGen);
if (idx != 0) {
std::iter_swap(itT, std::next(itT, idx));
Expand All @@ -84,32 +84,32 @@ void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD
}

template<>
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(std::vector<TTabletInfo*>& tablets, EResourceToBalance resourceToBalance) {
std::sort(tablets.begin(), tablets.end(), [resourceToBalance](const TTabletInfo* a, const TTabletInfo* b) -> bool {
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance resourceToBalance) {
std::sort(first, last, [resourceToBalance](const TTabletInfo* a, const TTabletInfo* b) -> bool {
return a->GetWeight(resourceToBalance) > b->GetWeight(resourceToBalance);
});
}

template<>
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(std::vector<TTabletInfo*>& tablets, EResourceToBalance) {
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance) {
auto& randGen = *TAppData::RandomProvider.Get();
std::shuffle(tablets.begin(), tablets.end(), randGen);
std::shuffle(first, last, randGen);
}

template<>
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(std::vector<TTabletInfo*>& tablets, EResourceToBalance resourceToBalance) {
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance resourceToBalance) {
auto& randGen = *TAppData::RandomProvider.Get();
std::vector<std::pair<double, TTabletInfo*>> weights;
weights.reserve(tablets.size());
for (TTabletInfo* tablet : tablets) {
double weight = tablet->GetWeight(resourceToBalance);
weights.emplace_back(weight * randGen(), tablet);
weights.reserve(last - first);
for (auto it = first; it != last; ++it) {
double weight = (*it)->GetWeight(resourceToBalance);
weights.emplace_back(weight * randGen(), *it);
}
std::sort(weights.begin(), weights.end(), [](const auto& a, const auto& b) -> bool {
return a.first > b.first;
});
for (size_t n = 0; n < weights.size(); ++n) {
tablets[n] = weights[n].second;
first[n] = weights[n].second;
}
}

Expand Down Expand Up @@ -252,18 +252,31 @@ class THiveBalancer : public NActors::TActorBootstrapped<THiveBalancer>, public
}
BLOG_TRACE("Balancer on node " << node->Id << ": " << tablets.size() << "/" << nodeTablets.size() << " tablets are suitable for balancing");
if (!tablets.empty()) {
// avoid moving system tablets if possible
std::vector<TTabletInfo*>::iterator partitionIt;
if (Hive->GetLessSystemTabletsMoves()) {
partitionIt = std::partition(tablets.begin(), tablets.end(), [](TTabletInfo* tablet) {
return !THive::IsSystemTablet(tablet->GetTabletType());
});
} else {
partitionIt = tablets.end();
}
switch (Hive->GetTabletBalanceStrategy()) {
case NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM:
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(tablets, Settings.ResourceToBalance);
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(tablets.begin(), partitionIt, Settings.ResourceToBalance);
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(partitionIt, tablets.end(), Settings.ResourceToBalance);
break;
case NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM:
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(tablets, Settings.ResourceToBalance);
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(tablets.begin(), partitionIt, Settings.ResourceToBalance);
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(partitionIt, tablets.end(), Settings.ResourceToBalance);
break;
case NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST:
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(tablets, Settings.ResourceToBalance);
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(tablets.begin(), partitionIt, Settings.ResourceToBalance);
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(partitionIt, tablets.end(), Settings.ResourceToBalance);
break;
case NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM:
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(tablets, Settings.ResourceToBalance);
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(tablets.begin(), partitionIt, Settings.ResourceToBalance);
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(partitionIt, tablets.end(), Settings.ResourceToBalance);
break;
}
Tablets.clear();
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/mind/hive/balancer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ template<NKikimrConfig::THiveConfig::EHiveNodeBalanceStrategy EHiveNodeBalanceSt
void BalanceNodes(std::vector<TNodeInfo*>& nodes, EResourceToBalance resourceTobalance);

template<NKikimrConfig::THiveConfig::EHiveTabletBalanceStrategy EHiveTabletBalanceStrategy>
void BalanceTablets(std::vector<TTabletInfo*>& tablets, EResourceToBalance resourceToBalance);
void BalanceTablets(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance resourceToBalance);

template <NKikimrConfig::THiveConfig::EHiveChannelBalanceStrategy>
void BalanceChannels(std::vector<TLeaderTabletInfo::TChannel>& channels, NKikimrConfig::THiveConfig::EHiveStorageBalanceStrategy metricToBalance);
Expand Down
4 changes: 4 additions & 0 deletions ydb/core/mind/hive/hive_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -938,6 +938,10 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId
return CurrentConfig.GetNodeUsageRangeToKick();
}

bool GetLessSystemTabletsMoves() const {
return CurrentConfig.GetLessSystemTabletsMoves();
}

static void ActualizeRestartStatistics(google::protobuf::RepeatedField<google::protobuf::uint64>& restartTimestamps, ui64 barrier);
static ui64 GetRestartsPerPeriod(const google::protobuf::RepeatedField<google::protobuf::uint64>& restartTimestamps, ui64 barrier);
static bool IsSystemTablet(TTabletTypes::EType type);
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/mind/hive/hive_impl_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ Y_UNIT_TEST_SUITE(THiveImplTest) {

auto CheckSpeedAndDistribution = [](
std::unordered_map<ui64, TLeaderTabletInfo>& allTablets,
std::function<void(std::vector<TTabletInfo*>&, EResourceToBalance)> func,
std::function<void(std::vector<TTabletInfo*>::iterator, std::vector<TTabletInfo*>::iterator, EResourceToBalance)> func,
EResourceToBalance resource) -> void {

std::vector<TTabletInfo*> tablets;
Expand All @@ -119,7 +119,7 @@ Y_UNIT_TEST_SUITE(THiveImplTest) {

TProfileTimer timer;

func(tablets, resource);
func(tablets.begin(), tablets.end(), resource);

double passed = timer.Get().SecondsFloat();

Expand Down
2 changes: 2 additions & 0 deletions ydb/core/mind/hive/monitoring.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -836,6 +836,7 @@ class TTxMonEvent_Settings : public TTransactionBase<THive>, public TLoggedMonTr
UpdateConfig(db, "MinStorageScatterToBalance", configUpdates);
UpdateConfig(db, "MinGroupUsageToBalance", configUpdates);
UpdateConfig(db, "StorageBalancerInflight", configUpdates);
UpdateConfig(db, "LessSystemTabletsMoves", configUpdates);

if (params.contains("BalancerIgnoreTabletTypes")) {
auto value = params.Get("BalancerIgnoreTabletTypes");
Expand Down Expand Up @@ -1182,6 +1183,7 @@ class TTxMonEvent_Settings : public TTransactionBase<THive>, public TLoggedMonTr
ShowConfig(out, "MinStorageScatterToBalance");
ShowConfig(out, "MinGroupUsageToBalance");
ShowConfig(out, "StorageBalancerInflight");
ShowConfig(out, "LessSystemTabletsMoves");
ShowConfigForBalancerIgnoreTabletTypes(out);

out << "<div class='row' style='margin-top:40px'>";
Expand Down
1 change: 1 addition & 0 deletions ydb/core/protos/config.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1470,6 +1470,7 @@ message THiveConfig {
optional uint64 StorageBalancerInflight = 73 [default = 1];
optional bool EnableDestroyOperations = 74 [default = false];
optional double NodeUsageRangeToKick = 75 [default = 0.2];
optional bool LessSystemTabletsMoves = 77 [default = true];
}

message TBlobCacheConfig {
Expand Down
Loading