Skip to content

Commit 892d691

Browse files
vporyadkevporyadke
authored
improve failsafe on cuttting history (#6918)
1 parent a55ca2e commit 892d691

File tree

13 files changed

+157
-30
lines changed

13 files changed

+157
-30
lines changed

‎ydb/core/mind/hive/hive_impl.cpp

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -631,11 +631,8 @@ void THive::BuildLocalConfig() {
631631
}
632632

633633
void THive::BuildCurrentConfig() {
634-
BLOG_D("THive::BuildCurrentConfig ClusterConfig = " << ClusterConfig.ShortDebugString());
635634
CurrentConfig = ClusterConfig;
636-
BLOG_D("THive::BuildCurrentConfig DatabaseConfig = " << DatabaseConfig.ShortDebugString());
637635
CurrentConfig.MergeFrom(DatabaseConfig);
638-
BLOG_D("THive::BuildCurrentConfig CurrentConfig = " << CurrentConfig.ShortDebugString());
639636
TabletLimit.clear();
640637
for (const auto& tabletLimit : CurrentConfig.GetDefaultTabletLimit()) {
641638
TabletLimit.insert_or_assign(tabletLimit.GetType(), tabletLimit);
@@ -652,6 +649,22 @@ void THive::BuildCurrentConfig() {
652649
}
653650
}
654651
MakeTabletTypeSet(BalancerIgnoreTabletTypes);
652+
CutHistoryDenyList.clear();
653+
for (auto name : SplitString(CurrentConfig.GetCutHistoryDenyList(), ",")) {
654+
TTabletTypes::EType type = TTabletTypes::StrToType(Strip(name));
655+
if (IsValidTabletType(type)) {
656+
CutHistoryDenyList.emplace_back(type);
657+
}
658+
}
659+
MakeTabletTypeSet(CutHistoryDenyList);
660+
CutHistoryAllowList.clear();
661+
for (auto name : SplitString(CurrentConfig.GetCutHistoryAllowList(), ",")) {
662+
TTabletTypes::EType type = TTabletTypes::StrToType(Strip(name));
663+
if (IsValidTabletType(type)) {
664+
CutHistoryAllowList.emplace_back(type);
665+
}
666+
}
667+
MakeTabletTypeSet(CutHistoryAllowList);
655668
if (!CurrentConfig.GetSpreadNeighbours()) {
656669
// SpreadNeighbours can be turned off anytime, but
657670
// cannot be safely turned on without Hive restart
@@ -2223,9 +2236,8 @@ void THive::Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev)
22232236
const NKikimrConsole::TConfigNotificationRequest& record = ev->Get()->Record;
22242237
ClusterConfig = record.GetConfig().GetHiveConfig();
22252238
NodeBrokerEpoch = TDuration::MicroSeconds(record.GetConfig().GetNodeBrokerConfig().GetEpochDuration());
2226-
BLOG_D("Received TEvConsole::TEvConfigNotificationRequest with update of cluster config: " << ClusterConfig.ShortDebugString()
2227-
<< "; " << record.GetConfig().GetNodeBrokerConfig().ShortDebugString());
22282239
BuildCurrentConfig();
2240+
BLOG_D("Merged config: " << CurrentConfig);
22292241
Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(record), 0, ev->Cookie);
22302242
}
22312243

‎ydb/core/mind/hive/hive_impl.h

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,8 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
453453

454454
// normalized to be sorted list of unique values
455455
std::vector<TTabletTypes::EType> BalancerIgnoreTabletTypes; // built from CurrentConfig
456+
std::vector<TTabletTypes::EType> CutHistoryDenyList; // built from CurrentConfig
457+
std::vector<TTabletTypes::EType> CutHistoryAllowList; // built from CurrentConfig
456458

457459
struct TTabletMoveInfo {
458460
TInstant Timestamp;
@@ -902,6 +904,21 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId
902904
return (found != ignoreList.end());
903905
}
904906

907+
bool IsCutHistoryAllowed(TTabletTypes::EType type) const {
908+
bool allowed = true;
909+
const auto& denyList = CutHistoryDenyList;
910+
if (!denyList.empty()) {
911+
bool found = std::find(denyList.begin(), denyList.end(), type) != denyList.end();
912+
allowed &= !found;
913+
}
914+
const auto& allowList = CutHistoryAllowList;
915+
if (!allowList.empty()) {
916+
bool found = std::find(allowList.begin(), allowList.end(), type) != allowList.end();
917+
allowed &= found;
918+
}
919+
return allowed;
920+
}
921+
905922
double GetSpaceUsagePenaltyThreshold() {
906923
return CurrentConfig.GetSpaceUsagePenaltyThreshold();
907924
}

‎ydb/core/mind/hive/hive_impl_ut.cpp

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,3 +218,78 @@ Y_UNIT_TEST_SUITE(THiveImplTest) {
218218
UNIT_ASSERT_VALUES_EQUAL(stDev1, stDev2);
219219
}
220220
}
221+
222+
Y_UNIT_TEST_SUITE(TCutHistoryRestrictions) {
223+
class TTestHive : public THive {
224+
public:
225+
TTestHive(TTabletStorageInfo *info, const TActorId &tablet) : THive(info, tablet) {}
226+
227+
template<typename F>
228+
void UpdateConfig(F func) {
229+
func(ClusterConfig);
230+
BuildCurrentConfig();
231+
}
232+
};
233+
234+
Y_UNIT_TEST(BasicTest) {
235+
TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
236+
hiveStorage->TabletType = TTabletTypes::Hive;
237+
TTestHive hive(hiveStorage.Get(), TActorId());
238+
hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
239+
config.SetCutHistoryAllowList("DataShard,Coordinator");
240+
config.SetCutHistoryDenyList("GraphShard");
241+
});
242+
UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::DataShard));
243+
UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::GraphShard));
244+
UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::Hive));
245+
}
246+
247+
Y_UNIT_TEST(EmptyAllowList) {
248+
TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
249+
hiveStorage->TabletType = TTabletTypes::Hive;
250+
TTestHive hive(hiveStorage.Get(), TActorId());
251+
hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
252+
config.SetCutHistoryAllowList("");
253+
config.SetCutHistoryDenyList("GraphShard");
254+
});
255+
UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::GraphShard));
256+
UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::Hive));
257+
}
258+
259+
Y_UNIT_TEST(EmptyDenyList) {
260+
TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
261+
hiveStorage->TabletType = TTabletTypes::Hive;
262+
TTestHive hive(hiveStorage.Get(), TActorId());
263+
hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
264+
config.SetCutHistoryAllowList("DataShard,Coordinator");
265+
config.SetCutHistoryDenyList("");
266+
});
267+
UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::DataShard));
268+
UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::GraphShard));
269+
}
270+
271+
Y_UNIT_TEST(SameTabletInBothLists) {
272+
TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
273+
hiveStorage->TabletType = TTabletTypes::Hive;
274+
TTestHive hive(hiveStorage.Get(), TActorId());
275+
hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
276+
config.SetCutHistoryAllowList("DataShard,Coordinator");
277+
config.SetCutHistoryDenyList("SchemeShard,DataShard");
278+
});
279+
UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::DataShard));
280+
UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::SchemeShard));
281+
UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::Hive));
282+
UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::Coordinator));
283+
}
284+
285+
Y_UNIT_TEST(BothListsEmpty) {
286+
TIntrusivePtr<TTabletStorageInfo> hiveStorage = new TTabletStorageInfo;
287+
hiveStorage->TabletType = TTabletTypes::Hive;
288+
TTestHive hive(hiveStorage.Get(), TActorId());
289+
hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) {
290+
config.SetCutHistoryAllowList("");
291+
config.SetCutHistoryDenyList("");
292+
});
293+
UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::DataShard));
294+
}
295+
}

‎ydb/core/mind/hive/hive_schema.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,9 +177,10 @@ struct Schema : NIceDb::Schema {
177177
struct Group : Column<3, NScheme::NTypeIds::Uint64> {};
178178
struct Version : Column<4, NScheme::NTypeIds::Uint64> {};
179179
struct Timestamp : Column<5, NScheme::NTypeIds::Uint64> {};
180+
struct DeletedAtGeneration : Column<6, NScheme::NTypeIds::Uint64> { static constexpr uint64_t Default = 0; };
180181

181182
using TKey = TableKey<Tablet, Channel, Generation>;
182-
using TColumns = TableColumns<Tablet, Channel, Generation, Group, Version, Timestamp>;
183+
using TColumns = TableColumns<Tablet, Channel, Generation, Group, Version, Timestamp, DeletedAtGeneration>;
183184
};
184185

185186
struct Node : Table<4> {

‎ydb/core/mind/hive/leader_tablet_info.cpp

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -367,12 +367,16 @@ void TLeaderTabletInfo::ActualizeTabletStatistics(TInstant now) {
367367
}
368368
}
369369

370-
void TLeaderTabletInfo::RestoreDeletedHistory() {
371-
for (const auto& entry : DeletedHistory) {
370+
void TLeaderTabletInfo::RestoreDeletedHistory(TTransactionContext& txc) {
371+
NIceDb::TNiceDb db(txc.DB);
372+
while (!DeletedHistory.empty()) {
373+
const auto& entry = DeletedHistory.front();
372374
if (entry.Channel >= TabletStorageInfo->Channels.size()) {
373375
continue;
374376
}
375377
TabletStorageInfo->Channels[entry.Channel].History.push_back(entry.Entry);
378+
db.Table<Schema::TabletChannelGen>().Key(Id, entry.Channel, entry.Entry.FromGeneration).Update<Schema::TabletChannelGen::DeletedAtGeneration>(0);
379+
DeletedHistory.pop();
376380
}
377381

378382
for (auto& channel : TabletStorageInfo->Channels) {
@@ -381,8 +385,6 @@ void TLeaderTabletInfo::RestoreDeletedHistory() {
381385
return lhs.FromGeneration < rhs.FromGeneration;
382386
});
383387
}
384-
385-
DeletedHistory.clear();
386388
}
387389

388390
void TLeaderTabletInfo::SetType(TTabletTypes::EType type) {

‎ydb/core/mind/hive/leader_tablet_info.h

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,12 @@ struct TLeaderTabletInfo : TTabletInfo {
5353
struct TChannelHistoryEntry {
5454
ui32 Channel;
5555
TTabletChannelInfo::THistoryEntry Entry;
56+
ui32 DeletedAtGeneration;
5657

57-
TChannelHistoryEntry(ui32 channel, const TTabletChannelInfo::THistoryEntry& entry)
58+
TChannelHistoryEntry(ui32 channel, const TTabletChannelInfo::THistoryEntry& entry, ui32 deletedAtGeneration)
5859
: Channel(channel)
5960
, Entry(entry)
61+
, DeletedAtGeneration(deletedAtGeneration)
6062
{
6163
}
6264
};
@@ -71,8 +73,8 @@ struct TLeaderTabletInfo : TTabletInfo {
7173
TIntrusivePtr<TTabletStorageInfo> TabletStorageInfo;
7274
TChannelsBindings BoundChannels;
7375
std::bitset<MAX_TABLET_CHANNELS> ChannelProfileNewGroup;
74-
std::vector<TChannelHistoryEntry> DeletedHistory;
75-
bool WasAliveSinceCutHistory = false;
76+
std::queue<TChannelHistoryEntry> DeletedHistory;
77+
bool WasAliveSinceCutHistory = true;
7678
NKikimrHive::TEvReassignTablet::EHiveReassignReason ChannelProfileReassignReason;
7779
ui32 KnownGeneration;
7880
TTabletCategoryInfo* Category;
@@ -352,7 +354,7 @@ struct TLeaderTabletInfo : TTabletInfo {
352354
TString GetChannelStoragePoolName(const TChannelProfiles::TProfile::TChannel& channel) const;
353355
TString GetChannelStoragePoolName(ui32 channelId) const;
354356
TStoragePoolInfo& GetStoragePool(ui32 channelId) const;
355-
void RestoreDeletedHistory();
357+
void RestoreDeletedHistory(TTransactionContext& txc);
356358

357359
void SetType(TTabletTypes::EType type);
358360
};

‎ydb/core/mind/hive/tx__cut_tablet_history.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ class TTxCutTabletHistory : public TTransactionBase<THive> {
1414

1515
TTxType GetTxType() const override { return NHive::TXTYPE_CUT_TABLET_HISTORY; }
1616

17-
bool Execute(TTransactionContext&, const TActorContext&) override {
17+
bool Execute(TTransactionContext& txc, const TActorContext&) override {
1818
TEvHive::TEvCutTabletHistory* msg = Event->Get();
1919
auto tabletId = msg->Record.GetTabletID();
2020
BLOG_D("THive::TTxCutTabletHistory::Execute(" << tabletId << ")");
2121
TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(tabletId);
22-
if (tablet != nullptr && tablet->IsReadyToReassignTablet()) {
22+
if (tablet != nullptr && tablet->IsReadyToReassignTablet() && Self->IsCutHistoryAllowed(tablet->Type)) {
2323
auto channel = msg->Record.GetChannel();
2424
Y_ABORT_UNLESS(channel < tablet->TabletStorageInfo->Channels.size());
2525
TTabletChannelInfo& channelInfo = tablet->TabletStorageInfo->Channels[channel];
@@ -30,11 +30,11 @@ class TTxCutTabletHistory : public TTransactionBase<THive> {
3030
channelInfo.History.end(),
3131
TTabletChannelInfo::THistoryEntry(fromGeneration, groupId));
3232
if (it != channelInfo.History.end()) {
33-
tablet->DeletedHistory.emplace_back(channel, *it);
33+
Self->TabletCounters->Cumulative()[NHive::COUNTER_HISTORY_CUT].Increment(1);
34+
tablet->DeletedHistory.emplace(channel, *it, tablet->KnownGeneration);
3435
channelInfo.History.erase(it);
35-
/* to be safe, don't do it just yet
3636
NIceDb::TNiceDb db(txc.DB);
37-
db.Table<Schema::TabletChannelGen>().Key(tabletId, channel, fromGeneration).Delete();*/
37+
db.Table<Schema::TabletChannelGen>().Key(tabletId, channel, fromGeneration).Update<Schema::TabletChannelGen::DeletedAtGeneration>(tablet->KnownGeneration);
3838
}
3939
}
4040
return true;

‎ydb/core/mind/hive/tx__load_everything.cpp

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -533,8 +533,14 @@ class TTxLoadEverything : public TTransactionBase<THive> {
533533
tablet->TabletStorageInfo->Channels.emplace_back();
534534
tablet->TabletStorageInfo->Channels.back().Channel = tablet->TabletStorageInfo->Channels.size() - 1;
535535
}
536-
TTabletChannelInfo& channel = tablet->TabletStorageInfo->Channels[channelId];
537-
channel.History.emplace_back(generationId, groupId, timestamp);
536+
TTabletChannelInfo::THistoryEntry entry(generationId, groupId, timestamp);
537+
auto deletedAtGeneration = tabletChannelGenRowset.GetValueOrDefault<Schema::TabletChannelGen::DeletedAtGeneration>();
538+
if (deletedAtGeneration) {
539+
tablet->DeletedHistory.emplace(channelId, entry, deletedAtGeneration);
540+
} else {
541+
TTabletChannelInfo& channel = tablet->TabletStorageInfo->Channels[channelId];
542+
channel.History.push_back(entry);
543+
}
538544
} else {
539545
++numMissingTablets;
540546
}

‎ydb/core/mind/hive/tx__start_tablet.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ class TTxStartTablet : public TTransactionBase<THive> {
5555
if (!leader.DeletedHistory.empty()) {
5656
if (!leader.WasAliveSinceCutHistory) {
5757
BLOG_ERROR("THive::TTxStartTablet::Execute Tablet " << TabletId << " failed to start after cutting history - will restore history");
58-
leader.RestoreDeletedHistory();
58+
Self->TabletCounters->Cumulative()[NHive::COUNTER_HISTORY_RESTORED].Increment(leader.DeletedHistory.size());
59+
leader.RestoreDeletedHistory(txc);
5960
} else {
6061
leader.WasAliveSinceCutHistory = false;
6162
}

‎ydb/core/mind/hive/tx__update_tablet_status.cpp

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,14 @@ class TTxUpdateTabletStatus : public TTransactionBase<THive> {
124124
NIceDb::TUpdate<Schema::Tablet::KnownGeneration>(Generation),
125125
NIceDb::TUpdate<Schema::Tablet::Statistics>(tablet->Statistics));
126126
Self->UpdateTabletFollowersNumber(leader, db, SideEffects);
127+
128+
// tablet booted successfully, we may actually cut history now
129+
while (!leader.DeletedHistory.empty() && leader.DeletedHistory.front().DeletedAtGeneration < leader.KnownGeneration) {
130+
leader.WasAliveSinceCutHistory = true;
131+
const auto& entry = leader.DeletedHistory.front();
132+
db.Table<Schema::TabletChannelGen>().Key(TabletId, entry.Channel, entry.Entry.FromGeneration).Delete();
133+
leader.DeletedHistory.pop();
134+
}
127135
} else {
128136
db.Table<Schema::TabletFollowerTablet>().Key(TabletId, FollowerId).Update(
129137
NIceDb::TUpdate<Schema::TabletFollowerTablet::GroupID>(tablet->AsFollower().FollowerGroup.Id),
@@ -160,13 +168,6 @@ class TTxUpdateTabletStatus : public TTransactionBase<THive> {
160168
db.Table<Schema::Tablet>().Key(TabletId).Update(NIceDb::TUpdate<Schema::Tablet::LeaderNode>(0),
161169
NIceDb::TUpdate<Schema::Tablet::KnownGeneration>(leader.KnownGeneration),
162170
NIceDb::TUpdate<Schema::Tablet::Statistics>(tablet->Statistics));
163-
164-
// tablet booted successfully, we may actually cut history now
165-
leader.WasAliveSinceCutHistory = true;
166-
for (const auto& entry : leader.DeletedHistory) {
167-
db.Table<Schema::TabletChannelGen>().Key(TabletId, entry.Channel, entry.Entry.FromGeneration).Delete();
168-
}
169-
leader.DeletedHistory.clear();
170171
} else {
171172
db.Table<Schema::TabletFollowerTablet>().Key(TabletId, FollowerId).Update(
172173
NIceDb::TUpdate<Schema::TabletFollowerTablet::GroupID>(tablet->AsFollower().FollowerGroup.Id),

‎ydb/core/protos/config.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1504,6 +1504,8 @@ message THiveConfig {
15041504
optional double NodeUsageRangeToKick = 75 [default = 0.2];
15051505
optional bool LessSystemTabletsMoves = 77 [default = true];
15061506
optional uint64 MaxPingsInFlight = 78 [default = 1000];
1507+
optional string CutHistoryDenyList = 76 [default = "ColumnShard,KeyValue,PersQueue,BlobDepot"];
1508+
optional string CutHistoryAllowList = 79 [default = "DataShard"];
15071509
}
15081510

15091511
message TBlobCacheConfig {

‎ydb/core/protos/counters_hive.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ enum ECumulativeCounters {
4848
COUNTER_SUGGESTED_SCALE_DOWN = 11 [(CounterOpts) = {Name: "SuggestedScaleDown"}];
4949
COUNTER_STORAGE_BALANCER_EXECUTED = 12 [(CounterOpts) = {Name: "StorageBalancerExecuted"}];
5050
COUNTER_TABLETS_STORAGE_REASSIGNED = 13 [(CounterOpts) = {Name: "TabletsStorageReassigned"}];
51+
COUNTER_HISTORY_CUT = 14 [(CounterOpts) = {Name: "TabletHistoryCut"}];
52+
COUNTER_HISTORY_RESTORED = 15 [(CounterOpts) = {Name: "TabletHistoryRestored"}];
5153
}
5254

5355
enum EPercentileCounters {

‎ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -520,6 +520,11 @@
520520
"ColumnId": 5,
521521
"ColumnName": "Timestamp",
522522
"ColumnType": "Uint64"
523+
},
524+
{
525+
"ColumnId": 6,
526+
"ColumnName": "DeletedAtGeneration",
527+
"ColumnType": "Uint64"
523528
}
524529
],
525530
"ColumnsDropped": [],
@@ -531,7 +536,8 @@
531536
2,
532537
3,
533538
4,
534-
5
539+
5,
540+
6
535541
],
536542
"RoomID": 0,
537543
"Codec": 0,

0 commit comments

Comments
 (0)