Skip to content

Commit df406e3

Browse files
vporyadkevporyadke
authored
process follower updates in chunks (#13023)
1 parent 8c80706 commit df406e3

File tree

6 files changed

+162
-52
lines changed

6 files changed

+162
-52
lines changed

‎ydb/core/mind/hive/hive.h

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,46 @@ struct TNodeFilter {
332332
bool IsAllowedDataCenter(TDataCenterId dc) const;
333333
};
334334

335+
struct TFollowerUpdates {
336+
enum class EAction {
337+
Create,
338+
Update,
339+
Delete,
340+
};
341+
342+
struct TUpdate {
343+
EAction Action;
344+
TFullTabletId TabletId;
345+
TFollowerGroupId GroupId;
346+
TDataCenterId DataCenter;
347+
};
348+
349+
std::deque<TUpdate> Updates;
350+
351+
bool Empty() const {
352+
return Updates.empty();
353+
}
354+
355+
void Create(TFullTabletId leaderTablet, TFollowerGroupId group, TDataCenterId dc) {
356+
Updates.emplace_back(EAction::Create, leaderTablet, group, dc);
357+
}
358+
359+
void Update(TFullTabletId tablet, TDataCenterId dc) {
360+
Updates.emplace_back(EAction::Update, tablet, 0, dc);
361+
}
362+
363+
void Delete(TFullTabletId tablet, TFollowerGroupId group, TDataCenterId dc) {
364+
Updates.emplace_back(EAction::Delete, tablet, group, dc);
365+
}
366+
367+
TUpdate Pop() {
368+
TUpdate update = Updates.front();
369+
Updates.pop_front();
370+
return update;
371+
}
372+
};
373+
374+
335375
} // NHive
336376
} // NKikimr
337377

‎ydb/core/mind/hive/hive_events.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ struct TEvPrivate {
3636
EvUpdateDataCenterFollowers,
3737
EvGenerateTestData,
3838
EvRefreshScaleRecommendation,
39+
EvUpdateFollowers,
3940
EvEnd
4041
};
4142

@@ -133,6 +134,9 @@ struct TEvPrivate {
133134
struct TEvGenerateTestData : TEventLocal<TEvGenerateTestData, EvGenerateTestData> {};
134135

135136
struct TEvRefreshScaleRecommendation : TEventLocal<TEvRefreshScaleRecommendation, EvRefreshScaleRecommendation> {};
137+
138+
struct TEvUpdateFollowers : TEventLocal<TEvUpdateFollowers, EvUpdateFollowers> {
139+
};
136140
};
137141

138142
} // NHive

‎ydb/core/mind/hive/hive_impl.cpp

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3072,6 +3072,7 @@ void THive::ProcessEvent(std::unique_ptr<IEventHandle> event) {
30723072
hFunc(TEvPrivate::TEvGenerateTestData, Handle);
30733073
hFunc(TEvPrivate::TEvRefreshScaleRecommendation, Handle);
30743074
hFunc(TEvHive::TEvConfigureScaleRecommender, Handle);
3075+
hFunc(TEvPrivate::TEvUpdateFollowers, Handle);
30753076
}
30763077
}
30773078

@@ -3179,6 +3180,7 @@ STFUNC(THive::StateWork) {
31793180
fFunc(TEvPrivate::TEvGenerateTestData::EventType, EnqueueIncomingEvent);
31803181
fFunc(TEvPrivate::TEvRefreshScaleRecommendation::EventType, EnqueueIncomingEvent);
31813182
fFunc(TEvHive::TEvConfigureScaleRecommender::EventType, EnqueueIncomingEvent);
3183+
fFunc(TEvPrivate::TEvUpdateFollowers::EventType, EnqueueIncomingEvent);
31823184
hFunc(TEvPrivate::TEvProcessIncomingEvent, Handle);
31833185
default:
31843186
if (!HandleDefaultEvents(ev, SelfId())) {
@@ -3480,7 +3482,40 @@ void THive::Handle(TEvHive::TEvRequestTabletDistribution::TPtr& ev) {
34803482
}
34813483

34823484
void THive::Handle(TEvPrivate::TEvUpdateDataCenterFollowers::TPtr& ev) {
3483-
Execute(CreateUpdateDcFollowers(ev->Get()->DataCenter));
3485+
auto dataCenterId = ev->Get()->DataCenter;
3486+
auto& dataCenter = DataCenters[dataCenterId];
3487+
if (!dataCenter.UpdateScheduled) {
3488+
return;
3489+
}
3490+
dataCenter.UpdateScheduled = false;
3491+
if (dataCenter.IsRegistered()) {
3492+
for (auto& [tabletId, tablet] : Tablets) {
3493+
for (auto& group : tablet.FollowerGroups) {
3494+
auto& followers = dataCenter.Followers[{tabletId, group.Id}];
3495+
i64 neededCount = group.GetFollowerCountForDataCenter(dataCenterId);
3496+
i64 delta = neededCount - std::ssize(followers);
3497+
for (i64 i = 0; i < delta; ++i) {
3498+
BLOG_TRACE("UpdateDataCenterFollowers: Pending create follower for " << tabletId);
3499+
PendingFollowerUpdates.Create(tablet.GetFullTabletId(), group.Id, dataCenterId);
3500+
}
3501+
}
3502+
}
3503+
} else {
3504+
for (auto& [group, followers] : dataCenter.Followers) {
3505+
for (auto follower : followers) {
3506+
BLOG_TRACE("UpdateDataCenterFollowers: Pending delete follower for " << follower->GetFullTabletId());
3507+
PendingFollowerUpdates.Delete(follower->GetFullTabletId(), group.second, dataCenterId);
3508+
}
3509+
}
3510+
}
3511+
if (!PendingFollowerUpdates.Empty() && !ProcessFollowerUpdatesScheduled) {
3512+
Send(SelfId(), new TEvPrivate::TEvUpdateFollowers);
3513+
ProcessFollowerUpdatesScheduled = true;
3514+
}
3515+
}
3516+
3517+
void THive::Handle(TEvPrivate::TEvUpdateFollowers::TPtr&) {
3518+
Execute(CreateProcessUpdateFollowers());
34843519
}
34853520

34863521
void THive::MakeScaleRecommendation() {

‎ydb/core/mind/hive/hive_impl.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
240240
friend class TTxUpdateTabletGroups;
241241
friend class TTxMonEvent_TabletAvailability;
242242
friend class TLoggedMonTransaction;
243-
friend class TTxUpdateDcFollowers;
243+
friend class TTxProcessUpdateFollowers;
244244

245245
friend class TDeleteTabletActor;
246246

@@ -303,7 +303,7 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
303303
ITransaction* CreateRequestTabletOwners(TEvHive::TEvRequestTabletOwners::TPtr event);
304304
ITransaction* CreateUpdateTabletsObject(TEvHive::TEvUpdateTabletsObject::TPtr event);
305305
ITransaction* CreateUpdateDomain(TSubDomainKey subdomainKey, TEvHive::TEvUpdateDomain::TPtr event = {});
306-
ITransaction* CreateUpdateDcFollowers(const TDataCenterId& dc);
306+
ITransaction* CreateProcessUpdateFollowers();
307307
ITransaction* CreateGenerateTestData(uint64_t seed);
308308
ITransaction* CreateDeleteNode(TNodeId nodeId);
309309
ITransaction* CreateConfigureScaleRecommender(TEvHive::TEvConfigureScaleRecommender::TPtr event);
@@ -405,6 +405,7 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
405405
bool ProcessPendingOperationsScheduled = false;
406406
bool LogTabletMovesScheduled = false;
407407
bool ProcessStorageBalancerScheduled = false;
408+
bool ProcessFollowerUpdatesScheduled = false;
408409
TResourceRawValues TotalRawResourceValues = {};
409410
TResourceNormalizedValues TotalNormalizedResourceValues = {};
410411
TInstant LastResourceChangeReaction;
@@ -422,6 +423,7 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
422423
std::vector<TActorId> ActorsWaitingToMoveTablets;
423424
std::queue<TActorId> NodePingQueue;
424425
std::unordered_set<TNodeId> NodePingsInProgress;
426+
TFollowerUpdates PendingFollowerUpdates;
425427

426428
struct TPendingCreateTablet {
427429
NKikimrHive::TEvCreateTablet CreateTablet;
@@ -586,6 +588,7 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
586588
void Handle(TEvPrivate::TEvGenerateTestData::TPtr& ev);
587589
void Handle(TEvPrivate::TEvRefreshScaleRecommendation::TPtr& ev);
588590
void Handle(TEvHive::TEvConfigureScaleRecommender::TPtr& ev);
591+
void Handle(TEvPrivate::TEvUpdateFollowers::TPtr& ev);
589592

590593
protected:
591594
void RestartPipeTx(ui64 tabletId);

‎ydb/core/mind/hive/tx__load_everything.cpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -674,7 +674,7 @@ class TTxLoadEverything : public TTransactionBase<THive> {
674674
--cnt;
675675
allowedDc = {dc};
676676
Self->DataCenters[dc].Followers[{tabletId, groupId}].push_back(followerIt);
677-
db.Table<Schema::TabletFollowerTablet>().Key(tabletId, followerIt->Id).Update<Schema::TabletFollowerTablet::DataCenter>(dc);
677+
Self->PendingFollowerUpdates.Update({tabletId, followerIt->Id}, dc);
678678
ok = true;
679679
}
680680
}
@@ -692,7 +692,7 @@ class TTxLoadEverything : public TTransactionBase<THive> {
692692
}
693693
follower->NodeFilter.AllowedDataCenters = {dcIt->first};
694694
Self->DataCenters[dcIt->first].Followers[{tabletId, groupId}].push_back(follower);
695-
db.Table<Schema::TabletFollowerTablet>().Key(follower->GetFullTabletId()).Update<Schema::TabletFollowerTablet::DataCenter>(dcIt->first);
695+
Self->PendingFollowerUpdates.Update(follower->GetFullTabletId(), dcIt->first);
696696
--dcIt->second;
697697
}
698698
}
@@ -859,6 +859,11 @@ class TTxLoadEverything : public TTransactionBase<THive> {
859859
Self->MigrationState = NKikimrHive::EMigrationState::MIGRATION_READY;
860860
ctx.Send(Self->SelfId(), new TEvPrivate::TEvBootTablets());
861861

862+
if (!Self->PendingFollowerUpdates.Empty()) {
863+
ctx.Send(Self->SelfId(), new TEvPrivate::TEvUpdateFollowers);
864+
Self->ProcessFollowerUpdatesScheduled = true;
865+
}
866+
862867
for (auto it = Self->Nodes.begin(); it != Self->Nodes.end(); ++it) {
863868
Self->ScheduleUnlockTabletExecution(it->second);
864869
}

‎ydb/core/mind/hive/tx__update_dc_followers.cpp

Lines changed: 70 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -4,64 +4,87 @@
44
namespace NKikimr {
55
namespace NHive {
66

7-
class TTxUpdateDcFollowers : public TTransactionBase<THive> {
8-
TDataCenterId DataCenterId;
7+
class TTxProcessUpdateFollowers : public TTransactionBase<THive> {
98
TSideEffects SideEffects;
9+
10+
static constexpr size_t MAX_UPDATES_PROCESSED = 1000;
1011
public:
11-
TTxUpdateDcFollowers(const TDataCenterId& dataCenter, THive* hive)
12+
TTxProcessUpdateFollowers(THive* hive)
1213
: TBase(hive)
13-
, DataCenterId(dataCenter)
1414
{}
1515

1616
TTxType GetTxType() const override { return NHive::TXTYPE_UPDATE_DC_FOLLOWERS; }
1717

1818
bool Execute(TTransactionContext& txc, const TActorContext&) override {
19-
BLOG_D("THive::TTxUpdateDcFollowers::Execute(" << DataCenterId << ")");
20-
SideEffects.Reset(Self->SelfId());
19+
BLOG_D("TTxProcessUpdateFollowers::Execute()");
2120
NIceDb::TNiceDb db(txc.DB);
22-
auto& dataCenter = Self->DataCenters[DataCenterId];
23-
if (!dataCenter.UpdateScheduled) {
24-
return true;
25-
}
26-
dataCenter.UpdateScheduled = false;
27-
if (dataCenter.IsRegistered()) {
28-
for (auto& [tabletId, tablet] : Self->Tablets) {
29-
for (auto& group : tablet.FollowerGroups) {
30-
auto& followers = dataCenter.Followers[{tabletId, group.Id}];
31-
auto neededCount = group.GetFollowerCountForDataCenter(DataCenterId);
32-
while (followers.size() < neededCount) {
33-
TFollowerTabletInfo& follower = tablet.AddFollower(group);
34-
follower.NodeFilter.AllowedDataCenters = {DataCenterId};
35-
follower.Statistics.SetLastAliveTimestamp(TlsActivationContext->Now().MilliSeconds());
36-
db.Table<Schema::TabletFollowerTablet>().Key(tabletId, follower.Id).Update(
37-
NIceDb::TUpdate<Schema::TabletFollowerTablet::GroupID>(follower.FollowerGroup.Id),
38-
NIceDb::TUpdate<Schema::TabletFollowerTablet::FollowerNode>(0),
39-
NIceDb::TUpdate<Schema::TabletFollowerTablet::Statistics>(follower.Statistics),
40-
NIceDb::TUpdate<Schema::TabletFollowerTablet::DataCenter>(DataCenterId));
41-
follower.InitTabletMetrics();
42-
follower.BecomeStopped();
43-
follower.InitiateBoot();
44-
followers.push_back(std::prev(tablet.Followers.end()));
45-
BLOG_D("THive::TTxUpdateDcFollowers::Execute(" << DataCenterId << "): created follower " << follower.GetFullTabletId());
21+
SideEffects.Reset(Self->SelfId());
22+
for (size_t i = 0; !Self->PendingFollowerUpdates.Empty() && i < MAX_UPDATES_PROCESSED; ++i) {
23+
auto op = Self->PendingFollowerUpdates.Pop();
24+
TTabletInfo* tablet = Self->FindTablet(op.TabletId);
25+
auto& dc = Self->DataCenters[op.DataCenter];
26+
if (tablet == nullptr) {
27+
continue;
28+
}
29+
switch (op.Action) {
30+
case TFollowerUpdates::EAction::Create:
31+
{
32+
if (!dc.IsRegistered()) {
33+
continue;
34+
}
35+
TFollowerGroup& group = tablet->AsLeader().GetFollowerGroup(op.GroupId);
36+
auto& followers = dc.Followers[{op.TabletId.first, op.GroupId}];
37+
if (group.GetFollowerCountForDataCenter(op.DataCenter) <= followers.size()) {
38+
continue;
4639
}
40+
TFollowerTabletInfo& follower = tablet->AsLeader().AddFollower(group);
41+
follower.NodeFilter.AllowedDataCenters = {op.DataCenter};
42+
follower.Statistics.SetLastAliveTimestamp(TlsActivationContext->Now().MilliSeconds());
43+
db.Table<Schema::TabletFollowerTablet>().Key(op.TabletId.first, follower.Id).Update(
44+
NIceDb::TUpdate<Schema::TabletFollowerTablet::GroupID>(follower.FollowerGroup.Id),
45+
NIceDb::TUpdate<Schema::TabletFollowerTablet::FollowerNode>(0),
46+
NIceDb::TUpdate<Schema::TabletFollowerTablet::Statistics>(follower.Statistics),
47+
NIceDb::TUpdate<Schema::TabletFollowerTablet::DataCenter>(op.DataCenter));
48+
follower.InitTabletMetrics();
49+
follower.BecomeStopped();
50+
follower.InitiateBoot();
51+
followers.push_back(std::prev(tablet->AsLeader().Followers.end()));
52+
BLOG_D("THive::TTxProcessUpdateFollowers::Execute(): created follower " << follower.GetFullTabletId());
53+
break;
4754
}
48-
}
49-
} else {
50-
// deleting followers
51-
i64 deletedFollowers = 0;
52-
for (auto& [_, followers] : dataCenter.Followers) {
53-
for (auto follower : followers) {
54-
db.Table<Schema::TabletFollowerTablet>().Key(follower->GetFullTabletId()).Delete();
55-
db.Table<Schema::Metrics>().Key(follower->GetFullTabletId()).Delete();
56-
follower->InitiateStop(SideEffects);
57-
auto& leader = follower->GetLeader();
58-
leader.Followers.erase(follower);
59-
++deletedFollowers;
55+
case TFollowerUpdates::EAction::Update:
56+
{
57+
// This is updated in memory in LoadEverything
58+
bool exists = db.Table<Schema::TabletFollowerTablet>().Key(op.TabletId).Select().IsValid();
59+
Y_ABORT_UNLESS(exists, "%s", (TStringBuilder() << "trying to update tablet " << op.TabletId).data());
60+
db.Table<Schema::TabletFollowerTablet>().Key(op.TabletId).Update<Schema::TabletFollowerTablet::DataCenter>(op.DataCenter);
61+
break;
62+
}
63+
case TFollowerUpdates::EAction::Delete:
64+
{
65+
if (dc.IsRegistered()) {
66+
continue;
67+
}
68+
db.Table<Schema::TabletFollowerTablet>().Key(op.TabletId).Delete();
69+
db.Table<Schema::Metrics>().Key(op.TabletId).Delete();
70+
tablet->InitiateStop(SideEffects);
71+
auto& followers = dc.Followers[{op.TabletId.first, op.GroupId}]; // Note: there are at most 3 followers here, see TPartitionConfigMerger
72+
auto iter = std::find_if(followers.begin(), followers.end(), [tabletId = op.TabletId](const auto& fw) {
73+
return fw->GetFullTabletId() == tabletId;
74+
});
75+
Y_ABORT_UNLESS(iter != followers.end());
76+
auto& leader = tablet->GetLeader();
77+
leader.Followers.erase(*iter);
78+
followers.erase(iter);
79+
Self->UpdateCounterTabletsTotal(-1);
80+
break;
6081
}
6182
}
62-
BLOG_D("THive::TTxUpdateDcFollowers::Execute(" << DataCenterId << "): deleted " << deletedFollowers << " followers");
63-
Self->UpdateCounterTabletsTotal(-deletedFollowers);
64-
dataCenter.Followers.clear();
83+
}
84+
if (Self->PendingFollowerUpdates.Empty()) {
85+
Self->ProcessFollowerUpdatesScheduled = false;
86+
} else {
87+
SideEffects.Send(Self->SelfId(), new TEvPrivate::TEvUpdateFollowers);
6588
}
6689
return true;
6790
}
@@ -71,8 +94,8 @@ class TTxUpdateDcFollowers : public TTransactionBase<THive> {
7194
}
7295
};
7396

74-
ITransaction* THive::CreateUpdateDcFollowers(const TDataCenterId& dc) {
75-
return new TTxUpdateDcFollowers(dc, this);
97+
ITransaction* THive::CreateProcessUpdateFollowers() {
98+
return new TTxProcessUpdateFollowers(this);
7699
}
77100

78101
} // NHive

0 commit comments

Comments
 (0)