Skip to content

Commit 6d047a4

Browse files
authored
always delete nodes persistently in hive (#11272)
1 parent b328a5f commit 6d047a4

File tree

9 files changed

+171
-10
lines changed

9 files changed

+171
-10
lines changed

ydb/core/mind/hive/hive_impl.cpp

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ void THive::RestartPipeTx(ui64 tabletId) {
9696
}
9797

9898
bool THive::TryToDeleteNode(TNodeInfo* node) {
99-
if (node->CanBeDeleted()) {
99+
if (node->CanBeDeleted(TActivationContext::Now())) {
100100
BLOG_I("TryToDeleteNode(" << node->Id << "): deleting");
101101
DeleteNode(node->Id);
102102
return true;
@@ -120,12 +120,15 @@ void THive::Handle(TEvTabletPipe::TEvServerConnected::TPtr& ev) {
120120
void THive::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev) {
121121
if (ev->Get()->TabletId == TabletID()) {
122122
BLOG_TRACE("Handle TEvTabletPipe::TEvServerDisconnected(" << ev->Get()->ClientId << ") " << ev->Get()->ServerId);
123-
TNodeInfo* node = FindNode(ev->Get()->ClientId.NodeId());
123+
auto nodeId = ev->Get()->ClientId.NodeId();
124+
TNodeInfo* node = FindNode(nodeId);
124125
if (node != nullptr) {
125126
Erase(node->PipeServers, ev->Get()->ServerId);
126127
if (node->PipeServers.empty() && node->IsUnknown()) {
127128
ObjectDistributions.RemoveNode(*node);
128-
TryToDeleteNode(node);
129+
if (TryToDeleteNode(node)) {
130+
Execute(CreateDeleteNode(nodeId));
131+
}
129132
}
130133
}
131134
}
@@ -3392,13 +3395,16 @@ void THive::Handle(TEvPrivate::TEvLogTabletMoves::TPtr&) {
33923395
}
33933396

33943397
void THive::Handle(TEvPrivate::TEvDeleteNode::TPtr& ev) {
3395-
auto node = FindNode(ev->Get()->NodeId);
3398+
auto nodeId = ev->Get()->NodeId;
3399+
auto node = FindNode(nodeId);
33963400
if (node == nullptr) {
33973401
return;
33983402
}
33993403
node->DeletionScheduled = false;
34003404
if (!node->IsAlive()) {
3401-
TryToDeleteNode(node);
3405+
if (TryToDeleteNode(node)) {
3406+
Execute(CreateDeleteNode(nodeId));
3407+
}
34023408
}
34033409
}
34043410

ydb/core/mind/hive/hive_impl.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,7 @@ class THive : public TActor<THive>, public TTabletExecutedFlat, public THiveShar
301301
ITransaction* CreateRequestTabletOwners(TEvHive::TEvRequestTabletOwners::TPtr event);
302302
ITransaction* CreateUpdateTabletsObject(TEvHive::TEvUpdateTabletsObject::TPtr event);
303303
ITransaction* CreateUpdateDomain(TSubDomainKey subdomainKey, TEvHive::TEvUpdateDomain::TPtr event = {});
304+
ITransaction* CreateDeleteNode(TNodeId nodeId);
304305

305306
public:
306307
TDomainsView DomainsView;

ydb/core/mind/hive/hive_ut.cpp

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,6 +1069,119 @@ Y_UNIT_TEST_SUITE(THiveTest) {
10691069
UNIT_ASSERT(!isNodeEmpty(nodeId));
10701070
}
10711071

1072+
Y_UNIT_TEST(DrainWithHiveRestart) {
1073+
// 1. Drain a node
1074+
// 2. Kill it & wait for hive to delete it
1075+
// 3. Start the node again
1076+
// 4. Restart hive
1077+
// 5. Ensure node is not down (by creating tablets)
1078+
const int NUM_NODES = 3;
1079+
const int NUM_TABLETS = 10;
1080+
TTestBasicRuntime runtime(NUM_NODES, false);
1081+
Setup(runtime, true, 2, [](TAppPrepare& app) {
1082+
app.HiveConfig.SetNodeDeletePeriod(1);
1083+
});
1084+
const ui64 hiveTablet = MakeDefaultHiveID();
1085+
const ui64 testerTablet = MakeTabletID(false, 1);
1086+
const TActorId hiveActor = CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::Hive), &CreateDefaultHive);
1087+
runtime.EnableScheduleForActor(hiveActor);
1088+
{
1089+
TDispatchOptions options;
1090+
options.FinalEvents.emplace_back(TEvLocal::EvStatus, NUM_NODES);
1091+
runtime.DispatchEvents(options);
1092+
}
1093+
TTabletTypes::EType tabletType = TTabletTypes::Dummy;
1094+
std::unordered_set<TTabletId> tablets;
1095+
TActorId senderA = runtime.AllocateEdgeActor(0);
1096+
auto createTablets = [&] {
1097+
for (int i = 0; i < NUM_TABLETS; ++i) {
1098+
THolder<TEvHive::TEvCreateTablet> ev(new TEvHive::TEvCreateTablet(testerTablet, 100500 + tablets.size(), tabletType, BINDED_CHANNELS));
1099+
runtime.SendToPipe(hiveTablet, senderA, ev.Release(), 0, GetPipeConfigWithRetries());
1100+
TAutoPtr<IEventHandle> handle;
1101+
auto createTabletReply = runtime.GrabEdgeEventRethrow<TEvHive::TEvCreateTabletReply>(handle);
1102+
ui64 tabletId = createTabletReply->Record.GetTabletID();
1103+
tablets.insert(tabletId);
1104+
}
1105+
NTabletPipe::TClientConfig pipeConfig;
1106+
pipeConfig.RetryPolicy = NTabletPipe::TClientRetryPolicy::WithRetries();
1107+
for (TTabletId tabletId : tablets) {
1108+
MakeSureTabletIsUp(runtime, tabletId, 0, &pipeConfig);
1109+
}
1110+
};
1111+
1112+
createTablets();
1113+
1114+
ui32 nodeIdx = 0;
1115+
ui32 nodeId = runtime.GetNodeId(nodeIdx);
1116+
{
1117+
Ctest << "1. Drain a node\n";
1118+
1119+
runtime.SendToPipe(hiveTablet, senderA, new TEvHive::TEvDrainNode(nodeId));
1120+
1121+
Ctest << "2. Kill it & wait for hive to delete it\n";
1122+
1123+
SendKillLocal(runtime, nodeIdx);
1124+
{
1125+
TDispatchOptions options;
1126+
options.FinalEvents.emplace_back(NHive::TEvPrivate::EvDeleteNode);
1127+
runtime.DispatchEvents(options);
1128+
runtime.AdvanceCurrentTime(TDuration::Seconds(2));
1129+
runtime.DispatchEvents(options);
1130+
}
1131+
}
1132+
1133+
auto isNodeEmpty = [&](ui32 nodeId) -> bool {
1134+
bool empty = true;
1135+
TAutoPtr<IEventHandle> handle;
1136+
TActorId whiteboard = NNodeWhiteboard::MakeNodeWhiteboardServiceId(nodeId);
1137+
runtime.Send(new IEventHandle(whiteboard, senderA, new NNodeWhiteboard::TEvWhiteboard::TEvTabletStateRequest()));
1138+
NNodeWhiteboard::TEvWhiteboard::TEvTabletStateResponse* wbResponse = runtime.GrabEdgeEventRethrow<NNodeWhiteboard::TEvWhiteboard::TEvTabletStateResponse>(handle);
1139+
for (const NKikimrWhiteboard::TTabletStateInfo& tabletInfo : wbResponse->Record.GetTabletStateInfo()) {
1140+
if (tablets.contains(tabletInfo.GetTabletId()) && tabletInfo.GetState() != NKikimrWhiteboard::TTabletStateInfo::Dead) {
1141+
Ctest << "Tablet " << tabletInfo.GetTabletId() << "." << tabletInfo.GetFollowerId()
1142+
<< " is not dead yet (" << NKikimrWhiteboard::TTabletStateInfo::ETabletState_Name(tabletInfo.GetState()) << ")" << Endl;
1143+
empty = false;
1144+
}
1145+
}
1146+
return empty;
1147+
};
1148+
1149+
Ctest << "3. Start the node again\n";
1150+
CreateLocal(runtime, nodeIdx);
1151+
1152+
{
1153+
TDispatchOptions options;
1154+
options.FinalEvents.emplace_back(TEvLocal::EvStatus);
1155+
runtime.DispatchEvents(options);
1156+
}
1157+
1158+
Ctest << "4. Restart hive\n";
1159+
1160+
runtime.Register(CreateTabletKiller(hiveTablet));
1161+
{
1162+
TDispatchOptions options;
1163+
std::unordered_set<ui32> nodesConnected;
1164+
auto observer = runtime.AddObserver<TEvLocal::TEvStatus>([&](auto&& ev) { nodesConnected.insert(ev->Sender.NodeId()); });
1165+
auto waitFor = [&](const auto& condition, const TString& description) {
1166+
while (!condition()) {
1167+
Ctest << "waiting for " << description << Endl;
1168+
TDispatchOptions options;
1169+
options.CustomFinalCondition = [&]() {
1170+
return condition();
1171+
};
1172+
runtime.DispatchEvents(options);
1173+
}
1174+
};
1175+
waitFor([&](){return nodesConnected.size() == NUM_NODES; }, "nodes to connect");
1176+
}
1177+
1178+
Ctest << "5. Ensure node is not down (by creating tablets)\n";
1179+
1180+
createTablets();
1181+
1182+
UNIT_ASSERT(!isNodeEmpty(nodeId));
1183+
}
1184+
10721185
Y_UNIT_TEST(TestCreateSubHiveCreateTablet) {
10731186
TTestBasicRuntime runtime(1, false);
10741187
Setup(runtime, true);

ydb/core/mind/hive/node_info.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -467,15 +467,15 @@ TResourceRawValues TNodeInfo::GetStDevResourceValues() {
467467
return GetStDev(values);
468468
}
469469

470-
bool TNodeInfo::CanBeDeleted() const {
470+
bool TNodeInfo::CanBeDeleted(TInstant now) const {
471471
TInstant lastAlive(TInstant::MilliSeconds(Statistics.GetLastAliveTimestamp()));
472472
if (lastAlive) {
473473
return (IsDisconnected() || IsUnknown())
474474
&& !Local
475475
&& GetTabletsTotal() == 0
476476
&& LockedTablets.empty()
477477
&& !Freeze
478-
&& (lastAlive + Hive.GetNodeDeletePeriod() < TInstant::Now());
478+
&& (lastAlive + Hive.GetNodeDeletePeriod() < now);
479479
} else {
480480
return (IsDisconnected() || IsUnknown()) && !Local && GetTabletsTotal() == 0 && LockedTablets.empty() && !Freeze;
481481
}

ydb/core/mind/hive/node_info.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ struct TNodeInfo {
231231
}
232232
}
233233

234-
bool CanBeDeleted() const;
234+
bool CanBeDeleted(TInstant now) const;
235235
void RegisterInDomains();
236236
void DeregisterInDomains();
237237
void Ping();
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#include "hive_impl.h"
2+
#include "hive_log.h"
3+
4+
namespace NKikimr {
5+
namespace NHive {
6+
7+
class TTxDeleteNode : public TTransactionBase<THive> {
8+
protected:
9+
TNodeId NodeId;
10+
public:
11+
TTxDeleteNode(TNodeId nodeId, THive *hive)
12+
: TBase(hive)
13+
, NodeId(nodeId)
14+
{}
15+
16+
bool Execute(TTransactionContext &txc, const TActorContext&) override {
17+
NIceDb::TNiceDb db(txc.DB);
18+
db.Table<Schema::Node>().Key(NodeId).Delete();
19+
auto restrictionsRowset = db.Table<Schema::TabletAvailabilityRestrictions>().Range(NodeId).Select();
20+
while (!restrictionsRowset.EndOfSet()) {
21+
db.Table<Schema::TabletAvailabilityRestrictions>().Key(restrictionsRowset.GetKey()).Delete();
22+
if (!restrictionsRowset.Next()) {
23+
return false;
24+
}
25+
}
26+
return true;
27+
}
28+
29+
void Complete(const TActorContext&) override {
30+
}
31+
};
32+
33+
ITransaction* THive::CreateDeleteNode(TNodeId nodeId) {
34+
return new TTxDeleteNode(nodeId, this);
35+
}
36+
37+
} // NHive
38+
} // NKikimr

ydb/core/mind/hive/tx__load_everything.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -688,8 +688,9 @@ class TTxLoadEverything : public TTransactionBase<THive> {
688688

689689
size_t numDeletedNodes = 0;
690690
size_t numDeletedRestrictions = 0;
691+
TInstant now = TActivationContext::Now();
691692
for (auto itNode = Self->Nodes.begin(); itNode != Self->Nodes.end();) {
692-
if (itNode->second.CanBeDeleted()) {
693+
if (itNode->second.CanBeDeleted(now)) {
693694
++numDeletedNodes;
694695
auto restrictionsRowset = db.Table<Schema::TabletAvailabilityRestrictions>().Range(itNode->first).Select();
695696
while (!restrictionsRowset.EndOfSet()) {

ydb/core/mind/hive/tx__register_node.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class TTxRegisterNode : public TTransactionBase<THive> {
2323
TNodeId nodeId = Local.NodeId();
2424
TNodeInfo& node = Self->GetNode(nodeId);
2525
if (node.Local != Local) {
26-
TInstant now = TInstant::Now();
26+
TInstant now = TActivationContext::Now();
2727
node.Statistics.AddRestartTimestamp(now.MilliSeconds());
2828
node.ActualizeNodeStatistics(now);
2929
for (const auto& t : node.Tablets) {
@@ -57,6 +57,7 @@ class TTxRegisterNode : public TTransactionBase<THive> {
5757
db.Table<Schema::Node>().Key(nodeId).Update<Schema::Node::Down, Schema::Node::Freeze>(false, false);
5858
}
5959
if (node.BecomeUpOnRestart) {
60+
BLOG_TRACE("THive::TTxRegisterNode(" << Local.NodeId() << ")::Execute - node became up on restart");
6061
node.SetDown(false);
6162
node.BecomeUpOnRestart = false;
6263
db.Table<Schema::Node>().Key(nodeId).Update<Schema::Node::Down, Schema::Node::BecomeUpOnRestart>(false, false);

ydb/core/mind/hive/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ SRCS(
4747
tx__configure_subdomain.cpp
4848
tx__create_tablet.cpp
4949
tx__cut_tablet_history.cpp
50+
tx__delete_node.cpp
5051
tx__delete_tablet.cpp
5152
tx__delete_tablet_result.cpp
5253
tx__disconnect_node.cpp

0 commit comments

Comments
 (0)