Skip to content

Commit 4ba1b11

Browse files
vporyadkevporyadke
authored
deprioritise system tablets in balancer (#6840) (#7047)
1 parent 66d836f commit 4ba1b11

File tree

6 files changed

+42
-22
lines changed

6 files changed

+42
-22
lines changed

‎ydb/core/mind/hive/balancer.cpp

Lines changed: 32 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -62,17 +62,17 @@ void BalanceNodes<NKikimrConfig::THiveConfig::HIVE_NODE_BALANCE_STRATEGY_RANDOM>
6262
}
6363

6464
template<>
65-
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(std::vector<TTabletInfo*>& tablets, EResourceToBalance resourceToBalance) {
65+
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance resourceToBalance) {
6666
auto& randGen = *TAppData::RandomProvider.Get();
6767
// weighted random shuffle
6868
std::vector<double> weights;
69-
weights.reserve(tablets.size());
70-
for (auto it = tablets.begin(); it != tablets.end(); ++it) {
69+
weights.reserve(last - first);
70+
for (auto it = first; it != last; ++it) {
7171
weights.emplace_back((*it)->GetWeight(resourceToBalance));
7272
}
73-
auto itT = tablets.begin();
73+
auto itT = first;
7474
auto itW = weights.begin();
75-
while (itT != tablets.end() && itW != weights.end()) {
75+
while (itT != last && itW != weights.end()) {
7676
auto idx = std::discrete_distribution(itW, weights.end())(randGen);
7777
if (idx != 0) {
7878
std::iter_swap(itT, std::next(itT, idx));
@@ -84,32 +84,32 @@ void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD
8484
}
8585

8686
template<>
87-
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(std::vector<TTabletInfo*>& tablets, EResourceToBalance resourceToBalance) {
88-
std::sort(tablets.begin(), tablets.end(), [resourceToBalance](const TTabletInfo* a, const TTabletInfo* b) -> bool {
87+
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance resourceToBalance) {
88+
std::sort(first, last, [resourceToBalance](const TTabletInfo* a, const TTabletInfo* b) -> bool {
8989
return a->GetWeight(resourceToBalance) > b->GetWeight(resourceToBalance);
9090
});
9191
}
9292

9393
template<>
94-
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(std::vector<TTabletInfo*>& tablets, EResourceToBalance) {
94+
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance) {
9595
auto& randGen = *TAppData::RandomProvider.Get();
96-
std::shuffle(tablets.begin(), tablets.end(), randGen);
96+
std::shuffle(first, last, randGen);
9797
}
9898

9999
template<>
100-
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(std::vector<TTabletInfo*>& tablets, EResourceToBalance resourceToBalance) {
100+
void BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance resourceToBalance) {
101101
auto& randGen = *TAppData::RandomProvider.Get();
102102
std::vector<std::pair<double, TTabletInfo*>> weights;
103-
weights.reserve(tablets.size());
104-
for (TTabletInfo* tablet : tablets) {
105-
double weight = tablet->GetWeight(resourceToBalance);
106-
weights.emplace_back(weight * randGen(), tablet);
103+
weights.reserve(last - first);
104+
for (auto it = first; it != last; ++it) {
105+
double weight = (*it)->GetWeight(resourceToBalance);
106+
weights.emplace_back(weight * randGen(), *it);
107107
}
108108
std::sort(weights.begin(), weights.end(), [](const auto& a, const auto& b) -> bool {
109109
return a.first > b.first;
110110
});
111111
for (size_t n = 0; n < weights.size(); ++n) {
112-
tablets[n] = weights[n].second;
112+
first[n] = weights[n].second;
113113
}
114114
}
115115

@@ -252,18 +252,31 @@ class THiveBalancer : public NActors::TActorBootstrapped<THiveBalancer>, public
252252
}
253253
BLOG_TRACE("Balancer on node " << node->Id << ": " << tablets.size() << "/" << nodeTablets.size() << " tablets are suitable for balancing");
254254
if (!tablets.empty()) {
255+
// avoid moving system tablets if possible
256+
std::vector<TTabletInfo*>::iterator partitionIt;
257+
if (Hive->GetLessSystemTabletsMoves()) {
258+
partitionIt = std::partition(tablets.begin(), tablets.end(), [](TTabletInfo* tablet) {
259+
return !THive::IsSystemTablet(tablet->GetTabletType());
260+
});
261+
} else {
262+
partitionIt = tablets.end();
263+
}
255264
switch (Hive->GetTabletBalanceStrategy()) {
256265
case NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM:
257-
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(tablets, Settings.ResourceToBalance);
266+
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(tablets.begin(), partitionIt, Settings.ResourceToBalance);
267+
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_OLD_WEIGHTED_RANDOM>(partitionIt, tablets.end(), Settings.ResourceToBalance);
258268
break;
259269
case NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM:
260-
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(tablets, Settings.ResourceToBalance);
270+
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(tablets.begin(), partitionIt, Settings.ResourceToBalance);
271+
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_WEIGHTED_RANDOM>(partitionIt, tablets.end(), Settings.ResourceToBalance);
261272
break;
262273
case NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST:
263-
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(tablets, Settings.ResourceToBalance);
274+
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(tablets.begin(), partitionIt, Settings.ResourceToBalance);
275+
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_HEAVIEST>(partitionIt, tablets.end(), Settings.ResourceToBalance);
264276
break;
265277
case NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM:
266-
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(tablets, Settings.ResourceToBalance);
278+
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(tablets.begin(), partitionIt, Settings.ResourceToBalance);
279+
BalanceTablets<NKikimrConfig::THiveConfig::HIVE_TABLET_BALANCE_STRATEGY_RANDOM>(partitionIt, tablets.end(), Settings.ResourceToBalance);
267280
break;
268281
}
269282
Tablets.clear();

‎ydb/core/mind/hive/balancer.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ template<NKikimrConfig::THiveConfig::EHiveNodeBalanceStrategy EHiveNodeBalanceSt
1010
void BalanceNodes(std::vector<TNodeInfo*>& nodes, EResourceToBalance resourceTobalance);
1111

1212
template<NKikimrConfig::THiveConfig::EHiveTabletBalanceStrategy EHiveTabletBalanceStrategy>
13-
void BalanceTablets(std::vector<TTabletInfo*>& tablets, EResourceToBalance resourceToBalance);
13+
void BalanceTablets(std::vector<TTabletInfo*>::iterator first, std::vector<TTabletInfo*>::iterator last, EResourceToBalance resourceToBalance);
1414

1515
template <NKikimrConfig::THiveConfig::EHiveChannelBalanceStrategy>
1616
void BalanceChannels(std::vector<TLeaderTabletInfo::TChannel>& channels, NKikimrConfig::THiveConfig::EHiveStorageBalanceStrategy metricToBalance);

‎ydb/core/mind/hive/hive_impl.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -938,6 +938,10 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId
938938
return CurrentConfig.GetNodeUsageRangeToKick();
939939
}
940940

941+
bool GetLessSystemTabletsMoves() const {
942+
return CurrentConfig.GetLessSystemTabletsMoves();
943+
}
944+
941945
static void ActualizeRestartStatistics(google::protobuf::RepeatedField<google::protobuf::uint64>& restartTimestamps, ui64 barrier);
942946
static ui64 GetRestartsPerPeriod(const google::protobuf::RepeatedField<google::protobuf::uint64>& restartTimestamps, ui64 barrier);
943947
static bool IsSystemTablet(TTabletTypes::EType type);

‎ydb/core/mind/hive/hive_impl_ut.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ Y_UNIT_TEST_SUITE(THiveImplTest) {
109109

110110
auto CheckSpeedAndDistribution = [](
111111
std::unordered_map<ui64, TLeaderTabletInfo>& allTablets,
112-
std::function<void(std::vector<TTabletInfo*>&, EResourceToBalance)> func,
112+
std::function<void(std::vector<TTabletInfo*>::iterator, std::vector<TTabletInfo*>::iterator, EResourceToBalance)> func,
113113
EResourceToBalance resource) -> void {
114114

115115
std::vector<TTabletInfo*> tablets;
@@ -119,7 +119,7 @@ Y_UNIT_TEST_SUITE(THiveImplTest) {
119119

120120
TProfileTimer timer;
121121

122-
func(tablets, resource);
122+
func(tablets.begin(), tablets.end(), resource);
123123

124124
double passed = timer.Get().SecondsFloat();
125125

‎ydb/core/mind/hive/monitoring.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -836,6 +836,7 @@ class TTxMonEvent_Settings : public TTransactionBase<THive>, public TLoggedMonTr
836836
UpdateConfig(db, "MinStorageScatterToBalance", configUpdates);
837837
UpdateConfig(db, "MinGroupUsageToBalance", configUpdates);
838838
UpdateConfig(db, "StorageBalancerInflight", configUpdates);
839+
UpdateConfig(db, "LessSystemTabletsMoves", configUpdates);
839840

840841
if (params.contains("BalancerIgnoreTabletTypes")) {
841842
auto value = params.Get("BalancerIgnoreTabletTypes");
@@ -1182,6 +1183,7 @@ class TTxMonEvent_Settings : public TTransactionBase<THive>, public TLoggedMonTr
11821183
ShowConfig(out, "MinStorageScatterToBalance");
11831184
ShowConfig(out, "MinGroupUsageToBalance");
11841185
ShowConfig(out, "StorageBalancerInflight");
1186+
ShowConfig(out, "LessSystemTabletsMoves");
11851187
ShowConfigForBalancerIgnoreTabletTypes(out);
11861188

11871189
out << "<div class='row' style='margin-top:40px'>";

‎ydb/core/protos/config.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1470,6 +1470,7 @@ message THiveConfig {
14701470
optional uint64 StorageBalancerInflight = 73 [default = 1];
14711471
optional bool EnableDestroyOperations = 74 [default = false];
14721472
optional double NodeUsageRangeToKick = 75 [default = 0.2];
1473+
optional bool LessSystemTabletsMoves = 77 [default = true];
14731474
}
14741475

14751476
message TBlobCacheConfig {

0 commit comments

Comments
 (0)