Skip to content

Commit bf2dd5d

Browse files
Transaction state after PQ tablet restart (#7015)
1 parent 55903d1 commit bf2dd5d

File tree

3 files changed

+192
-54
lines changed

3 files changed

+192
-54
lines changed

ydb/core/persqueue/pq_impl.cpp

Lines changed: 40 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -923,11 +923,9 @@ void TPersQueue::MoveTopTxToCalculating(TDistributedTransaction& tx,
923923
converterFactory,
924924
tx.TopicConverter,
925925
ctx);
926-
if (InitCompleted) {
927-
CreateNewPartitions(tx.TabletConfig,
928-
tx.TopicConverter,
929-
ctx);
930-
}
926+
CreateNewPartitions(tx.TabletConfig,
927+
tx.TopicConverter,
928+
ctx);
931929
SendEvProposePartitionConfig(ctx, tx);
932930
break;
933931
}
@@ -940,24 +938,6 @@ void TPersQueue::MoveTopTxToCalculating(TDistributedTransaction& tx,
940938
", NewState " << NKikimrPQ::TTransaction_EState_Name(tx.State));
941939
}
942940

943-
void TPersQueue::UpdateTopTxState(const TActorContext& ctx)
944-
{
945-
Y_ABORT_UNLESS(!InitCompleted);
946-
947-
if (TxQueue.empty()) {
948-
return;
949-
}
950-
951-
Y_ABORT_UNLESS(Txs.contains(TxQueue.front().second));
952-
auto& tx = Txs.at(TxQueue.front().second);
953-
954-
if (tx.State <= NKikimrPQ::TTransaction::PLANNED) {
955-
return;
956-
}
957-
958-
MoveTopTxToCalculating(tx, ctx);
959-
}
960-
961941
void TPersQueue::AddSupportivePartition(const TPartitionId& partitionId)
962942
{
963943
Partitions.emplace(partitionId,
@@ -1097,8 +1077,6 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult&
10971077
ctx);
10981078
}
10991079

1100-
UpdateTopTxState(ctx);
1101-
11021080
ConfigInited = true;
11031081

11041082
InitializeMeteringSink(ctx);
@@ -3961,7 +3939,9 @@ void TPersQueue::SendEvTxCommitToPartitions(const TActorContext& ctx,
39613939
auto event = std::make_unique<TEvPQ::TEvTxCommit>(tx.Step, tx.TxId);
39623940

39633941
auto p = Partitions.find(TPartitionId(partitionId));
3964-
Y_ABORT_UNLESS(p != Partitions.end());
3942+
Y_ABORT_UNLESS(p != Partitions.end(),
3943+
"Tablet %" PRIu64 ", Partition %" PRIu32 ", TxId %" PRIu64,
3944+
TabletID(), partitionId, tx.TxId);
39653945

39663946
ctx.Send(p->second.Actor, event.release());
39673947
}
@@ -4061,7 +4041,9 @@ void TPersQueue::CheckTxState(const TActorContext& ctx,
40614041

40624042
switch (tx.State) {
40634043
case NKikimrPQ::TTransaction::UNKNOWN:
4064-
Y_ABORT_UNLESS(tx.TxId != Max<ui64>());
4044+
Y_ABORT_UNLESS(tx.TxId != Max<ui64>(),
4045+
"PQ %" PRIu64 ", TxId %" PRIu64,
4046+
TabletID(), tx.TxId);
40654047

40664048
WriteTx(tx, NKikimrPQ::TTransaction::PREPARED);
40674049
ScheduleProposeTransactionResult(tx);
@@ -4073,7 +4055,9 @@ void TPersQueue::CheckTxState(const TActorContext& ctx,
40734055
break;
40744056

40754057
case NKikimrPQ::TTransaction::PREPARING:
4076-
Y_ABORT_UNLESS(tx.WriteInProgress);
4058+
Y_ABORT_UNLESS(tx.WriteInProgress,
4059+
"PQ %" PRIu64 ", TxId %" PRIu64,
4060+
TabletID(), tx.TxId);
40774061

40784062
tx.WriteInProgress = false;
40794063

@@ -4088,7 +4072,9 @@ void TPersQueue::CheckTxState(const TActorContext& ctx,
40884072
break;
40894073

40904074
case NKikimrPQ::TTransaction::PREPARED:
4091-
Y_ABORT_UNLESS(tx.Step != Max<ui64>());
4075+
Y_ABORT_UNLESS(tx.Step != Max<ui64>(),
4076+
"PQ %" PRIu64 ", TxId %" PRIu64,
4077+
TabletID(), tx.TxId);
40924078

40934079
WriteTx(tx, NKikimrPQ::TTransaction::PLANNED);
40944080

@@ -4099,7 +4085,9 @@ void TPersQueue::CheckTxState(const TActorContext& ctx,
40994085
break;
41004086

41014087
case NKikimrPQ::TTransaction::PLANNING:
4102-
Y_ABORT_UNLESS(tx.WriteInProgress);
4088+
Y_ABORT_UNLESS(tx.WriteInProgress,
4089+
"PQ %" PRIu64 ", TxId %" PRIu64,
4090+
TabletID(), tx.TxId);
41034091

41044092
tx.WriteInProgress = false;
41054093

@@ -4123,7 +4111,10 @@ void TPersQueue::CheckTxState(const TActorContext& ctx,
41234111
break;
41244112

41254113
case NKikimrPQ::TTransaction::CALCULATING:
4126-
Y_ABORT_UNLESS(tx.PartitionRepliesCount <= tx.PartitionRepliesExpected);
4114+
Y_ABORT_UNLESS(tx.PartitionRepliesCount <= tx.PartitionRepliesExpected,
4115+
"PQ %" PRIu64 ", TxId %" PRIu64 ", PartitionRepliesCount %" PRISZT ", PartitionRepliesExpected %" PRISZT,
4116+
TabletID(), tx.TxId,
4117+
tx.PartitionRepliesCount, tx.PartitionRepliesExpected);
41274118

41284119
PQ_LOG_D("Received " << tx.PartitionRepliesCount <<
41294120
", Expected " << tx.PartitionRepliesExpected);
@@ -4132,8 +4123,6 @@ void TPersQueue::CheckTxState(const TActorContext& ctx,
41324123
switch (tx.Kind) {
41334124
case NKikimrPQ::TTransaction::KIND_DATA:
41344125
case NKikimrPQ::TTransaction::KIND_CONFIG:
4135-
WriteTx(tx, NKikimrPQ::TTransaction::WAIT_RS);
4136-
41374126
tx.State = NKikimrPQ::TTransaction::CALCULATED;
41384127
PQ_LOG_D("TxId " << tx.TxId <<
41394128
", NewState " << NKikimrPQ::TTransaction_EState_Name(tx.State));
@@ -4143,14 +4132,16 @@ void TPersQueue::CheckTxState(const TActorContext& ctx,
41434132
case NKikimrPQ::TTransaction::KIND_UNKNOWN:
41444133
Y_ABORT_UNLESS(false);
41454134
}
4135+
} else {
4136+
break;
41464137
}
41474138

4148-
break;
4139+
[[fallthrough]];
41494140

41504141
case NKikimrPQ::TTransaction::CALCULATED:
4151-
Y_ABORT_UNLESS(tx.WriteInProgress);
4152-
4153-
tx.WriteInProgress = false;
4142+
Y_ABORT_UNLESS(!tx.WriteInProgress,
4143+
"PQ %" PRIu64 ", TxId %" PRIu64,
4144+
TabletID(), tx.TxId);
41544145

41554146
tx.State = NKikimrPQ::TTransaction::WAIT_RS;
41564147
PQ_LOG_D("TxId " << tx.TxId <<
@@ -4164,7 +4155,8 @@ void TPersQueue::CheckTxState(const TActorContext& ctx,
41644155
// from TEvProposeTransaction
41654156
//
41664157
Y_ABORT_UNLESS(tx.ReadSetAcks.size() <= tx.PredicatesReceived.size(),
4167-
"tx.ReadSetAcks.size=%" PRISZT ", tx.PredicatesReceived.size=%" PRISZT,
4158+
"PQ %" PRIu64 ", TxId %" PRIu64 ", ReadSetAcks.size %" PRISZT ", PredicatesReceived.size %" PRISZT,
4159+
TabletID(), tx.TxId,
41684160
tx.ReadSetAcks.size(), tx.PredicatesReceived.size());
41694161

41704162
SendEvReadSetToReceivers(ctx, tx);
@@ -4188,14 +4180,21 @@ void TPersQueue::CheckTxState(const TActorContext& ctx,
41884180
[[fallthrough]];
41894181

41904182
case NKikimrPQ::TTransaction::EXECUTING:
4191-
Y_ABORT_UNLESS(tx.PartitionRepliesCount <= tx.PartitionRepliesExpected);
4183+
Y_ABORT_UNLESS(tx.PartitionRepliesCount <= tx.PartitionRepliesExpected,
4184+
"PQ %" PRIu64 ", TxId %" PRIu64 ", PartitionRepliesCount %" PRISZT ", PartitionRepliesExpected %" PRISZT,
4185+
TabletID(), tx.TxId,
4186+
tx.PartitionRepliesCount, tx.PartitionRepliesExpected);
41924187

41934188
PQ_LOG_D("Received " << tx.PartitionRepliesCount <<
41944189
", Expected " << tx.PartitionRepliesExpected);
41954190

41964191
if (tx.PartitionRepliesCount == tx.PartitionRepliesExpected) {
4197-
Y_ABORT_UNLESS(!TxQueue.empty());
4198-
Y_ABORT_UNLESS(TxQueue.front().second == tx.TxId);
4192+
Y_ABORT_UNLESS(!TxQueue.empty(),
4193+
"PQ %" PRIu64 ", TxId %" PRIu64,
4194+
TabletID(), tx.TxId);
4195+
Y_ABORT_UNLESS(TxQueue.front().second == tx.TxId,
4196+
"PQ %" PRIu64 ", TxId %" PRIu64,
4197+
TabletID(), tx.TxId);
41994198

42004199
SendEvProposeTransactionResult(ctx, tx);
42014200

ydb/core/persqueue/pq_impl.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -504,7 +504,6 @@ class TPersQueue : public NKeyValue::TKeyValueFlat {
504504
const TWriteId& writeId) const;
505505
bool CheckTxWriteOperations(const NKikimrPQ::TDataTransaction& txBody) const;
506506

507-
void UpdateTopTxState(const TActorContext& ctx);
508507
void MoveTopTxToCalculating(TDistributedTransaction& tx, const TActorContext& ctx);
509508
};
510509

ydb/core/persqueue/ut/pqtablet_ut.cpp

Lines changed: 152 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -206,9 +206,9 @@ class TPQTabletFixture : public NUnitTest::TBaseFixture {
206206
void StartPQWriteTxsObserver();
207207
void WaitForPQWriteTxs();
208208

209-
template <class T> void WaitForEvent();
210-
void WaitForCalcPredicateResult();
211-
void WaitForProposePartitionConfigResult();
209+
template <class T> void WaitForEvent(size_t count);
210+
void WaitForCalcPredicateResult(size_t count = 1);
211+
void WaitForProposePartitionConfigResult(size_t count = 1);
212212

213213
void TestWaitingForTEvReadSet(size_t senders, size_t receivers);
214214

@@ -524,14 +524,16 @@ void TPQTabletFixture::WaitDropTabletReply(const TDropTabletReplyMatcher& matche
524524
}
525525

526526
template <class T>
527-
void TPQTabletFixture::WaitForEvent()
527+
void TPQTabletFixture::WaitForEvent(size_t count)
528528
{
529529
bool found = false;
530+
size_t received = 0;
530531

531532
TTestActorRuntimeBase::TEventObserver prev;
532-
auto observer = [&found, &prev](TAutoPtr<IEventHandle>& event) {
533+
auto observer = [&found, &prev, &received, count](TAutoPtr<IEventHandle>& event) {
533534
if (auto* msg = event->CastAsLocal<T>()) {
534-
found = true;
535+
++received;
536+
found = (received >= count);
535537
}
536538

537539
return prev ? prev(event) : TTestActorRuntimeBase::EEventAction::PROCESS;
@@ -549,14 +551,14 @@ void TPQTabletFixture::WaitForEvent()
549551
Ctx->Runtime->SetObserverFunc(prev);
550552
}
551553

552-
void TPQTabletFixture::WaitForCalcPredicateResult()
554+
void TPQTabletFixture::WaitForCalcPredicateResult(size_t count)
553555
{
554-
WaitForEvent<TEvPQ::TEvTxCalcPredicateResult>();
556+
WaitForEvent<TEvPQ::TEvTxCalcPredicateResult>(count);
555557
}
556558

557-
void TPQTabletFixture::WaitForProposePartitionConfigResult()
559+
void TPQTabletFixture::WaitForProposePartitionConfigResult(size_t count)
558560
{
559-
WaitForEvent<TEvPQ::TEvProposePartitionConfigResult>();
561+
WaitForEvent<TEvPQ::TEvProposePartitionConfigResult>(count);
560562
}
561563

562564
std::unique_ptr<TEvPersQueue::TEvRequest> TPQTabletFixture::MakeGetOwnershipRequest(const TGetOwnershipRequestParams& params,
@@ -1370,7 +1372,7 @@ Y_UNIT_TEST_F(Read_TEvTxCommit_After_Restart, TPQTabletFixture)
13701372

13711373
WaitForCalcPredicateResult();
13721374

1373-
// the transaction is now in the WAIT_RS state on disk and in memory
1375+
// the transaction is now in the WAIT_RS state in memory and PLANNED state in disk
13741376

13751377
PQTabletRestart(*Ctx);
13761378

@@ -1416,7 +1418,145 @@ Y_UNIT_TEST_F(Config_TEvTxCommit_After_Restart, TPQTabletFixture)
14161418

14171419
WaitForProposePartitionConfigResult();
14181420

1419-
// the transaction is now in the WAIT_RS state on disk and in memory
1421+
// the transaction is now in the WAIT_RS state in memory and PLANNED state in disk
1422+
1423+
PQTabletRestart(*Ctx);
1424+
1425+
tablet->SendReadSet(*Ctx->Runtime, {.Step=100, .TxId=txId, .Target=Ctx->TabletId, .Decision=NKikimrTx::TReadSetData::DECISION_COMMIT});
1426+
1427+
WaitProposeTransactionResponse({.TxId=txId,
1428+
.Status=NKikimrPQ::TEvProposeTransactionResult::COMPLETE});
1429+
1430+
tablet->SendReadSetAck(*Ctx->Runtime, {.Step=100, .TxId=txId, .Source=Ctx->TabletId});
1431+
WaitReadSetAck(*tablet, {.Step=100, .TxId=txId, .Source=mockTabletId, .Target=Ctx->TabletId, .Consumer=Ctx->TabletId});
1432+
}
1433+
1434+
Y_UNIT_TEST_F(One_Tablet_For_All_Partitions, TPQTabletFixture)
1435+
{
1436+
const ui64 txId = 67890;
1437+
1438+
PQTabletPrepare({.partitions=1}, {}, *Ctx);
1439+
1440+
auto tabletConfig = NHelpers::MakeConfig({.Version=2,
1441+
.Consumers={
1442+
{.Consumer="client-1", .Generation=0},
1443+
{.Consumer="client-3", .Generation=7}
1444+
},
1445+
.Partitions={
1446+
{.Id=0},
1447+
{.Id=1},
1448+
{.Id=2}
1449+
},
1450+
.AllPartitions={
1451+
{.Id=0, .TabletId=Ctx->TabletId, .Children={1, 2}, .Parents={}},
1452+
{.Id=1, .TabletId=Ctx->TabletId, .Children={}, .Parents={0}},
1453+
{.Id=2, .TabletId=Ctx->TabletId, .Children={}, .Parents={0}}
1454+
}});
1455+
1456+
SendProposeTransactionRequest({.TxId=txId,
1457+
.Configs=NHelpers::TConfigParams{
1458+
.Tablet=tabletConfig,
1459+
.Bootstrap=NHelpers::MakeBootstrapConfig(),
1460+
}});
1461+
WaitProposeTransactionResponse({.TxId=txId,
1462+
.Status=NKikimrPQ::TEvProposeTransactionResult::PREPARED});
1463+
1464+
SendPlanStep({.Step=100, .TxIds={txId}});
1465+
1466+
WaitForProposePartitionConfigResult(2);
1467+
1468+
// the transaction is now in the WAIT_RS state in memory and PLANNED state in disk
1469+
1470+
PQTabletRestart(*Ctx);
1471+
1472+
WaitProposeTransactionResponse({.TxId=txId,
1473+
.Status=NKikimrPQ::TEvProposeTransactionResult::COMPLETE});
1474+
}
1475+
1476+
Y_UNIT_TEST_F(One_New_Partition_In_Another_Tablet, TPQTabletFixture)
1477+
{
1478+
const ui64 txId = 67890;
1479+
const ui64 mockTabletId = 22222;
1480+
1481+
NHelpers::TPQTabletMock* tablet = CreatePQTabletMock(mockTabletId);
1482+
PQTabletPrepare({.partitions=1}, {}, *Ctx);
1483+
1484+
auto tabletConfig = NHelpers::MakeConfig({.Version=2,
1485+
.Consumers={
1486+
{.Consumer="client-1", .Generation=0},
1487+
{.Consumer="client-3", .Generation=7}
1488+
},
1489+
.Partitions={
1490+
{.Id=0},
1491+
{.Id=1},
1492+
},
1493+
.AllPartitions={
1494+
{.Id=0, .TabletId=Ctx->TabletId, .Children={1, 2}, .Parents={}},
1495+
{.Id=1, .TabletId=Ctx->TabletId, .Children={}, .Parents={0}},
1496+
{.Id=2, .TabletId=mockTabletId, .Children={}, .Parents={0}}
1497+
}});
1498+
1499+
SendProposeTransactionRequest({.TxId=txId,
1500+
.Configs=NHelpers::TConfigParams{
1501+
.Tablet=tabletConfig,
1502+
.Bootstrap=NHelpers::MakeBootstrapConfig(),
1503+
}});
1504+
WaitProposeTransactionResponse({.TxId=txId,
1505+
.Status=NKikimrPQ::TEvProposeTransactionResult::PREPARED});
1506+
1507+
SendPlanStep({.Step=100, .TxIds={txId}});
1508+
1509+
WaitForProposePartitionConfigResult(2);
1510+
1511+
// the transaction is now in the WAIT_RS state in memory and PLANNED state in disk
1512+
1513+
PQTabletRestart(*Ctx);
1514+
1515+
tablet->SendReadSet(*Ctx->Runtime, {.Step=100, .TxId=txId, .Target=Ctx->TabletId, .Decision=NKikimrTx::TReadSetData::DECISION_COMMIT});
1516+
1517+
WaitProposeTransactionResponse({.TxId=txId,
1518+
.Status=NKikimrPQ::TEvProposeTransactionResult::COMPLETE});
1519+
1520+
tablet->SendReadSetAck(*Ctx->Runtime, {.Step=100, .TxId=txId, .Source=Ctx->TabletId});
1521+
WaitReadSetAck(*tablet, {.Step=100, .TxId=txId, .Source=mockTabletId, .Target=Ctx->TabletId, .Consumer=Ctx->TabletId});
1522+
}
1523+
1524+
Y_UNIT_TEST_F(All_New_Partitions_In_Another_Tablet, TPQTabletFixture)
1525+
{
1526+
const ui64 txId = 67890;
1527+
const ui64 mockTabletId = 22222;
1528+
1529+
NHelpers::TPQTabletMock* tablet = CreatePQTabletMock(mockTabletId);
1530+
PQTabletPrepare({.partitions=1}, {}, *Ctx);
1531+
1532+
auto tabletConfig = NHelpers::MakeConfig({.Version=2,
1533+
.Consumers={
1534+
{.Consumer="client-1", .Generation=0},
1535+
{.Consumer="client-3", .Generation=7}
1536+
},
1537+
.Partitions={
1538+
{.Id=0},
1539+
{.Id=1},
1540+
},
1541+
.AllPartitions={
1542+
{.Id=0, .TabletId=Ctx->TabletId, .Children={}, .Parents={2}},
1543+
{.Id=1, .TabletId=Ctx->TabletId, .Children={}, .Parents={2}},
1544+
{.Id=2, .TabletId=mockTabletId, .Children={0, 1}, .Parents={}}
1545+
}});
1546+
1547+
SendProposeTransactionRequest({.TxId=txId,
1548+
.Configs=NHelpers::TConfigParams{
1549+
.Tablet=tabletConfig,
1550+
.Bootstrap=NHelpers::MakeBootstrapConfig(),
1551+
}});
1552+
WaitProposeTransactionResponse({.TxId=txId,
1553+
.Status=NKikimrPQ::TEvProposeTransactionResult::PREPARED});
1554+
1555+
SendPlanStep({.Step=100, .TxIds={txId}});
1556+
1557+
WaitForProposePartitionConfigResult(2);
1558+
1559+
// the transaction is now in the WAIT_RS state in memory and PLANNED state in disk
14201560

14211561
PQTabletRestart(*Ctx);
14221562

0 commit comments

Comments
 (0)