From 455c7331120007a0229740968a991671ef09f369 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Thu, 14 Nov 2024 22:08:31 +0000 Subject: [PATCH 01/29] work in progress --- .../kqp/executer_actor/kqp_data_executer.cpp | 1 + .../kqp/session_actor/kqp_query_state.cpp | 8 +- ydb/core/kqp/session_actor/kqp_query_state.h | 2 +- .../kqp/session_actor/kqp_session_actor.cpp | 1 - ydb/core/kqp/topics/kqp_topics.cpp | 74 +- ydb/core/kqp/topics/kqp_topics.h | 40 +- ydb/core/persqueue/events/internal.h | 9 +- ydb/core/persqueue/partition.cpp | 145 ++- ydb/core/persqueue/pq_impl.cpp | 13 +- ydb/core/persqueue/transaction.cpp | 2 +- .../ut/ut_with_sdk/autoscaling_ut.cpp | 1154 +++++++++-------- ydb/core/persqueue/ut/ut_with_sdk/ya.make | 4 +- ydb/core/protos/kqp.proto | 29 +- ydb/core/protos/pqconfig.proto | 7 +- ydb/services/lib/actors/type_definitions.h | 4 + .../actors/commit_offset_actor.cpp | 75 +- .../persqueue_v1/actors/commit_offset_actor.h | 137 ++ .../actors/read_init_auth_actor.cpp | 12 +- .../update_offsets_in_transaction_actor.cpp | 17 +- 19 files changed, 1041 insertions(+), 693 deletions(-) diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index 8937cd378bf7..3469111c67cc 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -2162,6 +2162,7 @@ class TKqpDataExecuter : public TKqpExecuterBase consumer; if (operations.HasConsumer()) { @@ -442,7 +442,6 @@ void TKqpQueryState::AddOffsetsToTransaction() { } TopicOperations = NTopic::TTopicOperations(); - for (auto& topic : operations.GetTopics()) { auto path = CanonizePath(NPersQueue::GetFullTopicPath(GetDatabase(), topic.path())); @@ -452,8 +451,7 @@ void TKqpQueryState::AddOffsetsToTransaction() { } else { for (auto& range : partition.partition_offsets()) { YQL_ENSURE(consumer.Defined()); - - TopicOperations.AddOperation(path, partition.partition_id(), *consumer, range); + TopicOperations.AddOperation(path, partition.partition_id(), *consumer, range, partition.force_commit(), partition.kill_read_session(), partition.only_check_commited_to_finish()); } } } @@ -474,7 +472,7 @@ std::unique_ptr TKqpQueryState::BuildSchemeC auto navigate = std::make_unique(); navigate->DatabaseName = CanonizePath(GetDatabase()); - const auto& operations = GetTopicOperations(); + const auto& operations = GetTopicOperationsFromRequest(); TMaybe consumer; if (operations.HasConsumer()) consumer = operations.GetConsumer(); diff --git a/ydb/core/kqp/session_actor/kqp_query_state.h b/ydb/core/kqp/session_actor/kqp_query_state.h index 57411dde5964..f8df052feda1 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.h +++ b/ydb/core/kqp/session_actor/kqp_query_state.h @@ -332,7 +332,7 @@ class TKqpQueryState : public TNonCopyable { return RequestEv->GetQuery(); } - const ::NKikimrKqp::TTopicOperationsRequest& GetTopicOperations() const { + const ::NKikimrKqp::TTopicOperationsRequest& GetTopicOperationsFromRequest() const { return RequestEv->GetTopicOperations(); } diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 137c75b4c084..26a2e05b5c39 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -1358,7 +1358,6 @@ class TKqpSessionActor : public TActorBootstrapped { } } } - request.TopicOperations = std::move(txCtx.TopicOperations); } else if (QueryState->ShouldAcquireLocks(tx) && (!txCtx.HasOlapTable || Settings.TableService.GetEnableOlapSink())) { request.AcquireLocksTxId = txCtx.Locks.GetLockTxId(); diff --git a/ydb/core/kqp/topics/kqp_topics.cpp b/ydb/core/kqp/topics/kqp_topics.cpp index fcd14d18934a..58882e717fb3 100644 --- a/ydb/core/kqp/topics/kqp_topics.cpp +++ b/ydb/core/kqp/topics/kqp_topics.cpp @@ -1,6 +1,7 @@ #include "kqp_topics.h" #include +#include #include #include @@ -29,18 +30,37 @@ bool TConsumerOperations::IsValid() const return Offsets_.GetNumIntervals() == 1; } -std::pair TConsumerOperations::GetRange() const +std::pair TConsumerOperations::GetOffsetsCommitRange() const { Y_ABORT_UNLESS(IsValid()); return {Offsets_.Min(), Offsets_.Max()}; } -void TConsumerOperations::AddOperation(const TString& consumer, const Ydb::Topic::OffsetsRange& range) +bool TConsumerOperations::GetForceCommit() const +{ + return ForceCommit_; +} + +bool TConsumerOperations::GetKillReadSession() const +{ + return KillReadSession_; +} + +bool TConsumerOperations::GetOnlyCheckCommitedToFinish() const +{ + return OnlyCheckCommitedToFinish_; +} + +void TConsumerOperations::AddOperation(const TString& consumer, + const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, + bool forceCommit, + bool killReadSession, + bool onlyCheckCommitedToFinish) { Y_ABORT_UNLESS(Consumer_.Empty() || Consumer_ == consumer); - AddOperationImpl(consumer, range.start(), range.end()); + AddOperationImpl(consumer, range.start(), range.end(), forceCommit, killReadSession, onlyCheckCommitedToFinish); } void TConsumerOperations::Merge(const TConsumerOperations& rhs) @@ -49,12 +69,16 @@ void TConsumerOperations::Merge(const TConsumerOperations& rhs) Y_ABORT_UNLESS(Consumer_.Empty() || Consumer_ == rhs.Consumer_); for (auto& range : rhs.Offsets_) { - AddOperationImpl(*rhs.Consumer_, range.first, range.second); + AddOperationImpl(*rhs.Consumer_, range.first, range.second, rhs.GetForceCommit(), rhs.GetKillReadSession(), rhs.GetOnlyCheckCommitedToFinish()); } } void TConsumerOperations::AddOperationImpl(const TString& consumer, - ui64 begin, ui64 end) + ui64 begin, + ui64 end, + bool forceCommit, + bool killReadSession, + bool onlyCheckCommitedToFinish) { if (Offsets_.Intersects(begin, end)) { ythrow TOffsetsRangeIntersectExpection() << "offset ranges intersect"; @@ -65,6 +89,10 @@ void TConsumerOperations::AddOperationImpl(const TString& consumer, } Offsets_.InsertInterval(begin, end); + + ForceCommit_ = forceCommit; + KillReadSession_ = killReadSession; + OnlyCheckCommitedToFinish_ = onlyCheckCommitedToFinish; } // @@ -76,9 +104,13 @@ bool TTopicPartitionOperations::IsValid() const [](auto& x) { return x.second.IsValid(); }); } -void TTopicPartitionOperations::AddOperation(const TString& topic, ui32 partition, +void TTopicPartitionOperations::AddOperation(const TString& topic, + ui32 partition, const TString& consumer, - const Ydb::Topic::OffsetsRange& range) + const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, + bool forceCommit, + bool killReadSession, + bool onlyCheckCommitedToFinish) { Y_ABORT_UNLESS(Topic_.Empty() || Topic_ == topic); Y_ABORT_UNLESS(Partition_.Empty() || Partition_ == partition); @@ -88,7 +120,7 @@ void TTopicPartitionOperations::AddOperation(const TString& topic, ui32 partitio Partition_ = partition; } - Operations_[consumer].AddOperation(consumer, range); + Operations_[consumer].AddOperation(consumer, range, forceCommit, killReadSession, onlyCheckCommitedToFinish); } void TTopicPartitionOperations::AddOperation(const TString& topic, ui32 partition, @@ -117,11 +149,14 @@ void TTopicPartitionOperations::BuildTopicTxs(TTopicOperationTransactions& txs) for (auto& [consumer, operations] : Operations_) { NKikimrPQ::TPartitionOperation* o = t.tx.MutableOperations()->Add(); o->SetPartitionId(*Partition_); - auto [begin, end] = operations.GetRange(); - o->SetBegin(begin); - o->SetEnd(end); + auto [begin, end] = operations.GetOffsetsCommitRange(); + o->SetCommitOffsetsBegin(begin); + o->SetCommitOffsetsEnd(end); o->SetConsumer(consumer); o->SetPath(*Topic_); + o->SetKillReadSession(operations.GetKillReadSession()); + o->SetForceCommit(operations.GetForceCommit()); + o->SetOnlyCheckCommitedToFinish(operations.GetOnlyCheckCommitedToFinish()); } if (HasWriteOperations_) { @@ -256,14 +291,23 @@ bool TTopicOperations::TabletHasReadOperations(ui64 tabletId) const return false; } -void TTopicOperations::AddOperation(const TString& topic, ui32 partition, +void TTopicOperations::AddOperation(const TString& topic, + ui32 partition, const TString& consumer, - const Ydb::Topic::OffsetsRange& range) + const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, + bool forceCommit, + bool killReadSession, + bool onlyCheckCommitedToFinish + ) { TTopicPartition key{topic, partition}; - Operations_[key].AddOperation(topic, partition, + Operations_[key].AddOperation(topic, + partition, consumer, - range); + range, + forceCommit, + killReadSession, + onlyCheckCommitedToFinish); HasReadOperations_ = true; } diff --git a/ydb/core/kqp/topics/kqp_topics.h b/ydb/core/kqp/topics/kqp_topics.h index b04e7a186f5e..d17282d149ed 100644 --- a/ydb/core/kqp/topics/kqp_topics.h +++ b/ydb/core/kqp/topics/kqp_topics.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -26,20 +27,36 @@ class TConsumerOperations { public: bool IsValid() const; - std::pair GetRange() const; + std::pair GetOffsetsCommitRange() const; - ui64 GetBegin() const; - ui64 GetEnd() const; + ui64 GetOffsetCommitBegin() const; + ui64 GetOffsetCommitEnd() const; + + bool GetForceCommit() const; + bool GetKillReadSession() const; + bool GetOnlyCheckCommitedToFinish() const; + + void AddOperation(const TString& consumer, + const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, + bool forceCommit = false, + bool killReadSession = false, + bool onlyCheckCommitedToFinish = false); - void AddOperation(const TString& consumer, const Ydb::Topic::OffsetsRange& range); void Merge(const TConsumerOperations& rhs); private: void AddOperationImpl(const TString& consumer, - ui64 begin, ui64 end); + ui64 begin, + ui64 end, + bool forceCommit = false, + bool killReadSession = false, + bool onlyCheckCommitedToFinish = false); TMaybe Consumer_; TDisjointIntervalTree Offsets_; + bool ForceCommit_ = false; + bool KillReadSession_ = false; + bool OnlyCheckCommitedToFinish_ = false; }; struct TTopicOperationTransaction { @@ -53,9 +70,13 @@ class TTopicPartitionOperations { public: bool IsValid() const; - void AddOperation(const TString& topic, ui32 partition, + void AddOperation(const TString& topic, + ui32 partition, const TString& consumer, - const Ydb::Topic::OffsetsRange& range); + const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, + bool forceCommit = false, + bool killReadSession = false, + bool onlyCheckCommitedToFinish = false); void AddOperation(const TString& topic, ui32 partition, TMaybe supportivePartition); @@ -108,7 +129,10 @@ class TTopicOperations { void AddOperation(const TString& topic, ui32 partition, const TString& consumer, - const Ydb::Topic::OffsetsRange& range); + const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, + bool forceCommit, + bool killReadSession, + bool onlyCheckCommitedToFinish); void AddOperation(const TString& topic, ui32 partition, TMaybe supportivePartition); diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index 176b12f48e66..574484e8886d 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -811,11 +811,14 @@ struct TEvPQ { { } - void AddOperation(TString consumer, ui64 begin, ui64 end) { + void AddOperation(TString consumer, ui64 begin, ui64 end, bool forceCommit = false, bool killReadSession = false, bool onlyCheckCommitedToFinish = false) { NKikimrPQ::TPartitionOperation operation; - operation.SetBegin(begin); - operation.SetEnd(end); + operation.SetCommitOffsetsBegin(begin); + operation.SetCommitOffsetsEnd(end); operation.SetConsumer(std::move(consumer)); + operation.SetForceCommit(forceCommit); + operation.SetKillReadSession(killReadSession); + operation.SetOnlyCheckCommitedToFinish(onlyCheckCommitedToFinish); Operations.push_back(std::move(operation)); } diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 9a37c987e1ce..9ef81c908bae 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -1970,23 +1970,23 @@ void TPartition::ContinueProcessTxsAndUserActs(const TActorContext&) msg->WaitPreviousWriteSpan.End(); } switch (std::visit(visitor, front.Event)) { - case EProcessResult::Continue: - MoveUserActOrTxToCommitState(); - FirstEvent = false; - break; - case EProcessResult::ContinueDrop: - UserActionAndTransactionEvents.pop_front(); - break; - case EProcessResult::Break: - MoveUserActOrTxToCommitState(); - BatchingState = ETxBatchingState::Finishing; - FirstEvent = false; - break; - case EProcessResult::Blocked: - BatchingState = ETxBatchingState::Executing; - return; - case EProcessResult::NotReady: - return; + case EProcessResult::Continue: + MoveUserActOrTxToCommitState(); + FirstEvent = false; + break; + case EProcessResult::ContinueDrop: + UserActionAndTransactionEvents.pop_front(); + break; + case EProcessResult::Break: + MoveUserActOrTxToCommitState(); + BatchingState = ETxBatchingState::Finishing; + FirstEvent = false; + break; + case EProcessResult::Blocked: + BatchingState = ETxBatchingState::Executing; + return; + case EProcessResult::NotReady: + return; } CurrentBatchSize += 1; } @@ -2329,42 +2329,48 @@ TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPr break; } - bool isAffectedConsumer = AffectedUsers.contains(consumer); + bool isAffectedConsumer = AffectedUsers.contains(consumer); // savnik check TUserInfoBase& userInfo = GetOrCreatePendingUser(consumer); - if (operation.GetBegin() > operation.GetEnd()) { - PQ_LOG_D("Partition " << Partition << - " Consumer '" << consumer << "'" << - " Bad request (invalid range) " << - " Begin " << operation.GetBegin() << - " End " << operation.GetEnd()); - ok = false; - } else if (userInfo.Offset != (i64)operation.GetBegin()) { - PQ_LOG_D("Partition " << Partition << - " Consumer '" << consumer << "'" << - " Bad request (gap) " << - " Offset " << userInfo.Offset << - " Begin " << operation.GetBegin()); - ok = false; - } else if (operation.GetEnd() > EndOffset) { - PQ_LOG_D("Partition " << Partition << - " Consumer '" << consumer << "'" << - " Bad request (behind the last offset) " << - " EndOffset " << EndOffset << - " End " << operation.GetEnd()); - ok = false; - } + if (operation.HasOnlyCheckCommitedToFinish() && operation.GetOnlyCheckCommitedToFinish()) { + if (static_cast(userInfo.Offset) != EndOffset) { // savnik что если откатят коммит пока выполняется транзакция + ok = false; + } + } else { + if (!operation.GetForceCommit() && operation.GetCommitOffsetsBegin() > operation.GetCommitOffsetsEnd()) { // savnik check default + PQ_LOG_D("Partition " << Partition << + " Consumer '" << consumer << "'" << + " Bad request (invalid range) " << + " Begin " << operation.GetCommitOffsetsBegin() << + " End " << operation.GetCommitOffsetsEnd()); + ok = false; + } else if (!operation.GetForceCommit() && userInfo.Offset != (i64)operation.GetCommitOffsetsBegin()) { + PQ_LOG_D("Partition " << Partition << + " Consumer '" << consumer << "'" << + " Bad request (gap) " << + " Offset " << userInfo.Offset << + " Begin " << operation.GetCommitOffsetsBegin()); + ok = false; + } else if (!operation.GetForceCommit() && operation.GetCommitOffsetsEnd() > EndOffset) { + PQ_LOG_D("Partition " << Partition << + " Consumer '" << consumer << "'" << + " Bad request (behind the last offset) " << + " EndOffset " << EndOffset << + " End " << operation.GetCommitOffsetsEnd()); + ok = false; + } - if (!ok) { - if (!isAffectedConsumer) { - AffectedUsers.erase(consumer); + if (!ok) { + if (!isAffectedConsumer) { + AffectedUsers.erase(consumer); + } + break; } - break; + consumers.insert(consumer); } - consumers.insert(consumer); } if (ok) { - TxAffectedConsumers.insert(consumers.begin(), consumers.end()); + TxAffectedConsumers.insert(consumers.begin(), consumers.end()); // savnik check } predicate = ok; return EProcessResult::Continue; @@ -2515,10 +2521,27 @@ void TPartition::CommitTransaction(TSimpleSharedPtr& t) for (auto& operation : t->Tx->Operations) { TUserInfoBase& userInfo = GetOrCreatePendingUser(operation.GetConsumer()); - Y_ABORT_UNLESS(userInfo.Offset == (i64)operation.GetBegin()); + if (!operation.HasForceCommit() || !operation.GetForceCommit()) { // savnik check default + Y_ABORT_UNLESS(userInfo.Offset == (i64)operation.GetCommitOffsetsBegin()); + } + + if (operation.GetCommitOffsetsEnd() < StartOffset) { + userInfo.Offset = StartOffset; + } else if (operation.GetCommitOffsetsEnd() > EndOffset) { + userInfo.Offset = EndOffset; + } else { + userInfo.Offset = operation.GetCommitOffsetsEnd(); + } - userInfo.Offset = operation.GetEnd(); + if (operation.GetKillReadSession()) { // savnik check default + userInfo.Session = ""; + userInfo.PartitionSessionId = 0; + userInfo.Generation = 0; + userInfo.Step = 0; // savnik check need drop this + userInfo.PipeClient = {}; + } } + CommitWriteOperations(*t); ChangePlanStepAndTxId(t->Tx->Step, t->Tx->TxId); ScheduleReplyCommitDone(t->Tx->Step, t->Tx->TxId); @@ -2784,12 +2807,12 @@ TPartition::EProcessResult TPartition::PreProcessImmediateTx(const NKikimrPQ::TE Y_ABORT_UNLESS(tx.HasData()); THashSet consumers; for (auto& operation : tx.GetData().GetOperations()) { - if (!operation.HasBegin() || !operation.HasEnd() || !operation.HasConsumer()) { + if (!operation.HasCommitOffsetsBegin() || !operation.HasCommitOffsetsEnd() || !operation.HasConsumer()) { continue; //Write operation - handled separately via WriteInfo } - Y_ABORT_UNLESS(operation.GetBegin() <= (ui64)Max(), "Unexpected begin offset: %" PRIu64, operation.GetBegin()); - Y_ABORT_UNLESS(operation.GetEnd() <= (ui64)Max(), "Unexpected end offset: %" PRIu64, operation.GetEnd()); + Y_ABORT_UNLESS(operation.GetCommitOffsetsBegin() <= (ui64)Max(), "Unexpected begin offset: %" PRIu64, operation.GetCommitOffsetsBegin()); + Y_ABORT_UNLESS(operation.GetCommitOffsetsEnd() <= (ui64)Max(), "Unexpected end offset: %" PRIu64, operation.GetCommitOffsetsEnd()); const TString& user = operation.GetConsumer(); if (TxAffectedConsumers.contains(user)) { @@ -2802,7 +2825,7 @@ TPartition::EProcessResult TPartition::PreProcessImmediateTx(const NKikimrPQ::TE "the consumer has been deleted"); return EProcessResult::ContinueDrop; } - if (operation.GetBegin() > operation.GetEnd()) { + if (operation.GetCommitOffsetsBegin() > operation.GetCommitOffsetsEnd()) { ScheduleReplyPropose(tx, NKikimrPQ::TEvProposeTransactionResult::BAD_REQUEST, NKikimrPQ::TError::BAD_REQUEST, @@ -2833,12 +2856,12 @@ void TPartition::ExecImmediateTx(TTransaction& t) return; } for (const auto& operation : record.GetData().GetOperations()) { - if (!operation.HasBegin() || !operation.HasEnd() || !operation.HasConsumer()) { + if (!operation.HasCommitOffsetsBegin() || !operation.HasCommitOffsetsEnd() || !operation.HasConsumer()) { continue; //Write operation - handled separately via WriteInfo } - Y_ABORT_UNLESS(operation.GetBegin() <= (ui64)Max(), "Unexpected begin offset: %" PRIu64, operation.GetBegin()); - Y_ABORT_UNLESS(operation.GetEnd() <= (ui64)Max(), "Unexpected end offset: %" PRIu64, operation.GetEnd()); + Y_ABORT_UNLESS(operation.GetCommitOffsetsBegin() <= (ui64)Max(), "Unexpected begin offset: %" PRIu64, operation.GetCommitOffsetsBegin()); + Y_ABORT_UNLESS(operation.GetCommitOffsetsEnd() <= (ui64)Max(), "Unexpected end offset: %" PRIu64, operation.GetCommitOffsetsEnd()); const TString& user = operation.GetConsumer(); if (!PendingUsersInfo.contains(user) && AffectedUsers.contains(user)) { @@ -2850,7 +2873,7 @@ void TPartition::ExecImmediateTx(TTransaction& t) } TUserInfoBase& userInfo = GetOrCreatePendingUser(user); - if (operation.GetBegin() > operation.GetEnd()) { + if (operation.GetCommitOffsetsBegin() > operation.GetCommitOffsetsEnd()) { ScheduleReplyPropose(record, NKikimrPQ::TEvProposeTransactionResult::BAD_REQUEST, NKikimrPQ::TError::BAD_REQUEST, @@ -2858,7 +2881,7 @@ void TPartition::ExecImmediateTx(TTransaction& t) return; } - if (userInfo.Offset != (i64)operation.GetBegin()) { + if (userInfo.Offset != (i64)operation.GetCommitOffsetsBegin()) { ScheduleReplyPropose(record, NKikimrPQ::TEvProposeTransactionResult::ABORTED, NKikimrPQ::TError::BAD_REQUEST, @@ -2866,14 +2889,14 @@ void TPartition::ExecImmediateTx(TTransaction& t) return; } - if (operation.GetEnd() > EndOffset) { + if (operation.GetCommitOffsetsEnd() > EndOffset) { ScheduleReplyPropose(record, NKikimrPQ::TEvProposeTransactionResult::BAD_REQUEST, NKikimrPQ::TError::BAD_REQUEST, "incorrect offset range (commit to the future)"); return; } - userInfo.Offset = operation.GetEnd(); + userInfo.Offset = operation.GetCommitOffsetsEnd(); } CommitWriteOperations(t); @@ -3107,7 +3130,7 @@ void TPartition::EmulatePostProcessUserAct(const TEvPQ::TEvSetClientInfo& act, bool setSession = act.Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION; bool dropSession = act.Type == TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION; - bool strictCommitOffset = (act.Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET && act.SessionId.empty()); + bool commitNotFromReadSession = (act.Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET && act.SessionId.empty()); if (act.Type == TEvPQ::TEvSetClientInfo::ESCI_DROP_READ_RULE) { userInfo.ReadRuleGeneration = 0; @@ -3152,7 +3175,7 @@ void TPartition::EmulatePostProcessUserAct(const TEvPQ::TEvSetClientInfo& act, userInfo.Step = step; userInfo.PipeClient = act.PipeClient; userInfo.PartitionSessionId = act.PartitionSessionId; - } else if ((dropSession && act.PipeClient == userInfo.PipeClient) || strictCommitOffset) { + } else if ((dropSession && act.PipeClient == userInfo.PipeClient) || commitNotFromReadSession) { userInfo.Session = ""; userInfo.PartitionSessionId = 0; userInfo.Generation = 0; diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 2918c297ba7c..55ec6f4dbc5f 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -3198,7 +3198,7 @@ bool TPersQueue::CheckTxWriteOperations(const NKikimrPQ::TDataTransaction& txBod for (auto& operation : txBody.GetOperations()) { auto isWrite = [](const NKikimrPQ::TPartitionOperation& o) { - return !o.HasBegin(); + return !o.HasCommitOffsetsBegin(); }; if (isWrite(operation)) { @@ -3921,7 +3921,7 @@ TMaybe TPersQueue::FindPartitionId(const NKikimrPQ::TDataTransacti { auto hasWriteOperation = [](const auto& txBody) { for (const auto& o : txBody.GetOperations()) { - if (!o.HasBegin()) { + if (!o.HasCommitOffsetsBegin()) { return true; } } @@ -3973,10 +3973,13 @@ void TPersQueue::SendEvTxCalcPredicateToPartitions(const TActorContext& ctx, event = std::make_unique(tx.Step, tx.TxId); } - if (operation.HasBegin()) { + if (operation.HasCommitOffsetsBegin()) { event->AddOperation(operation.GetConsumer(), - operation.GetBegin(), - operation.GetEnd()); + operation.GetCommitOffsetsBegin(), + operation.GetCommitOffsetsEnd(), + operation.HasForceCommit() ? operation.GetForceCommit() : false, + operation.HasKillReadSession() ? operation.GetKillReadSession() : false, + operation.HasOnlyCheckCommitedToFinish() ? operation.GetOnlyCheckCommitedToFinish() : false); } } diff --git a/ydb/core/persqueue/transaction.cpp b/ydb/core/persqueue/transaction.cpp index 63fe2a184b20..9042afed69bd 100644 --- a/ydb/core/persqueue/transaction.cpp +++ b/ydb/core/persqueue/transaction.cpp @@ -77,7 +77,7 @@ void TDistributedTransaction::InitPartitions(const google::protobuf::RepeatedPtr Partitions.clear(); for (auto& o : operations) { - if (!o.HasBegin()) { + if (!o.HasCommitOffsetsBegin()) { HasWriteOperations = true; } diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index b39ad0c0c7fb..c2f77843cd53 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -19,212 +19,212 @@ using namespace NKikimr::NPQ::NTest; Y_UNIT_TEST_SUITE(TopicAutoscaling) { - void SimpleTest(SdkVersion sdk, bool autoscaleAwareSDK) { - TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(); + // void SimpleTest(SdkVersion sdk, bool autoscaleAwareSDK) { + // TTopicSdkTestSetup setup = CreateSetup(); + // setup.CreateTopicWithAutoscale(); - TTopicClient client = setup.MakeClient(); + // TTopicClient client = setup.MakeClient(); - auto writeSession1 = CreateWriteSession(client, "producer-1"); - auto writeSession2 = CreateWriteSession(client, "producer-2"); + // auto writeSession1 = CreateWriteSession(client, "producer-1"); + // auto writeSession2 = CreateWriteSession(client, "producer-2"); - auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 2, .AutoCommit = !autoscaleAwareSDK, .AutoPartitioningSupport = autoscaleAwareSDK }); - readSession->Run(); + // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 2, .AutoCommit = !autoscaleAwareSDK, .AutoPartitioningSupport = autoscaleAwareSDK }); + // readSession->Run(); - UNIT_ASSERT(writeSession1->Write(Msg("message_1.1", 2))); - UNIT_ASSERT(writeSession2->Write(Msg("message_2.1", 3))); + // UNIT_ASSERT(writeSession1->Write(Msg("message_1.1", 2))); + // UNIT_ASSERT(writeSession2->Write(Msg("message_2.1", 3))); - readSession->WaitAllMessages(); + // readSession->WaitAllMessages(); - for(const auto& info : readSession->GetReceivedMessages()) { - if (info.Data == "message_1.1") { - UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); - UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); - } else if (info.Data == "message_2.1") { - UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); - UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); - } else { - UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - } - } + // for(const auto& info : readSession->GetReceivedMessages()) { + // if (info.Data == "message_1.1") { + // UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); + // UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); + // } else if (info.Data == "message_2.1") { + // UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); + // UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); + // } else { + // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + // } + // } - writeSession1->Close(TDuration::Seconds(1)); - writeSession2->Close(TDuration::Seconds(1)); - readSession->Close(); - } + // writeSession1->Close(TDuration::Seconds(1)); + // writeSession2->Close(TDuration::Seconds(1)); + // readSession->Close(); + // } - Y_UNIT_TEST(Simple_BeforeAutoscaleAwareSDK) { - SimpleTest(SdkVersion::Topic, false); - } + // Y_UNIT_TEST(Simple_BeforeAutoscaleAwareSDK) { + // SimpleTest(SdkVersion::Topic, false); + // } - Y_UNIT_TEST(Simple_AutoscaleAwareSDK) { - SimpleTest(SdkVersion::Topic, true); - } + // Y_UNIT_TEST(Simple_AutoscaleAwareSDK) { + // SimpleTest(SdkVersion::Topic, true); + // } - Y_UNIT_TEST(Simple_PQv1) { - SimpleTest(SdkVersion::PQv1, false); - } + // Y_UNIT_TEST(Simple_PQv1) { + // SimpleTest(SdkVersion::PQv1, false); + // } - void ReadingAfterSplitTest(SdkVersion sdk, bool autoscaleAwareSDK, bool autoCommit) { - TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(); + // void ReadingAfterSplitTest(SdkVersion sdk, bool autoscaleAwareSDK, bool autoCommit) { + // TTopicSdkTestSetup setup = CreateSetup(); + // setup.CreateTopicWithAutoscale(); - TTopicClient client = setup.MakeClient(); + // TTopicClient client = setup.MakeClient(); - auto writeSession = CreateWriteSession(client, "producer-1"); + // auto writeSession = CreateWriteSession(client, "producer-1"); - UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); + // UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); - ui64 txId = 1006; - SplitPartition(setup, ++txId, 0, "a"); + // ui64 txId = 1006; + // SplitPartition(setup, ++txId, 0, "a"); - UNIT_ASSERT(writeSession->Write(Msg("message_2.1", 3))); + // UNIT_ASSERT(writeSession->Write(Msg("message_2.1", 3))); - SplitPartition(setup, ++txId, 2, "d"); + // SplitPartition(setup, ++txId, 2, "d"); - UNIT_ASSERT(writeSession->Write(Msg("message_3.1", 5))); + // UNIT_ASSERT(writeSession->Write(Msg("message_3.1", 5))); - auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 3, .AutoCommit = autoCommit, .AutoPartitioningSupport = autoscaleAwareSDK }); - readSession->Run(); - readSession->WaitAllMessages(); + // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 3, .AutoCommit = autoCommit, .AutoPartitioningSupport = autoscaleAwareSDK }); + // readSession->Run(); + // readSession->WaitAllMessages(); - for(const auto& info : readSession->GetReceivedMessages()) { - if (info.Data == "message_1.1") { - UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); - UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); - } else if (info.Data == "message_2.1") { - UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); - UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); - } else if (info.Data == "message_3.1") { - UNIT_ASSERT_VALUES_EQUAL(4, info.PartitionId); - UNIT_ASSERT_VALUES_EQUAL(5, info.SeqNo); - } else { - UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - } - } + // for(const auto& info : readSession->GetReceivedMessages()) { + // if (info.Data == "message_1.1") { + // UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); + // UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); + // } else if (info.Data == "message_2.1") { + // UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); + // UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); + // } else if (info.Data == "message_3.1") { + // UNIT_ASSERT_VALUES_EQUAL(4, info.PartitionId); + // UNIT_ASSERT_VALUES_EQUAL(5, info.SeqNo); + // } else { + // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + // } + // } - writeSession->Close(TDuration::Seconds(1)); - readSession->Close(); - } + // writeSession->Close(TDuration::Seconds(1)); + // readSession->Close(); + // } - Y_UNIT_TEST(ReadingAfterSplitTest_BeforeAutoscaleAwareSDK) { - ReadingAfterSplitTest(SdkVersion::Topic, false, true); - } + // Y_UNIT_TEST(ReadingAfterSplitTest_BeforeAutoscaleAwareSDK) { + // ReadingAfterSplitTest(SdkVersion::Topic, false, true); + // } - Y_UNIT_TEST(ReadingAfterSplitTest_AutoscaleAwareSDK) { - ReadingAfterSplitTest(SdkVersion::Topic, true, false); - } + // Y_UNIT_TEST(ReadingAfterSplitTest_AutoscaleAwareSDK) { + // ReadingAfterSplitTest(SdkVersion::Topic, true, false); + // } - Y_UNIT_TEST(ReadingAfterSplitTest_AutoscaleAwareSDK_AutoCommit) { - ReadingAfterSplitTest(SdkVersion::Topic, true, false); - } + // Y_UNIT_TEST(ReadingAfterSplitTest_AutoscaleAwareSDK_AutoCommit) { + // ReadingAfterSplitTest(SdkVersion::Topic, true, false); + // } - Y_UNIT_TEST(ReadingAfterSplitTest_PQv1) { - ReadingAfterSplitTest(SdkVersion::PQv1, false, true); - } + // Y_UNIT_TEST(ReadingAfterSplitTest_PQv1) { + // ReadingAfterSplitTest(SdkVersion::PQv1, false, true); + // } - void ReadingAfterSplitTest_PreferedPartition(SdkVersion sdk, bool autoscaleAwareSDK) { - TTopicSdkTestSetup setup = CreateSetup(); - setup.CreateTopicWithAutoscale(); + // void ReadingAfterSplitTest_PreferedPartition(SdkVersion sdk, bool autoscaleAwareSDK) { + // TTopicSdkTestSetup setup = CreateSetup(); + // setup.CreateTopicWithAutoscale(); - TTopicClient client = setup.MakeClient(); + // TTopicClient client = setup.MakeClient(); - auto writeSession = CreateWriteSession(client, "producer-1"); + // auto writeSession = CreateWriteSession(client, "producer-1"); - UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); + // UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); - ui64 txId = 1006; - SplitPartition(setup, ++txId, 0, "a"); + // ui64 txId = 1006; + // SplitPartition(setup, ++txId, 0, "a"); - UNIT_ASSERT(writeSession->Write(Msg("message_2.1", 3))); + // UNIT_ASSERT(writeSession->Write(Msg("message_2.1", 3))); - SplitPartition(setup, ++txId, 2, "d"); + // SplitPartition(setup, ++txId, 2, "d"); - UNIT_ASSERT(writeSession->Write(Msg("message_3.1", 5))); + // UNIT_ASSERT(writeSession->Write(Msg("message_3.1", 5))); - auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 1, .AutoCommit = !autoscaleAwareSDK, .Partitions= {2}, .AutoPartitioningSupport = autoscaleAwareSDK }); - readSession->Run(); - readSession->WaitAllMessages(); + // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 1, .AutoCommit = !autoscaleAwareSDK, .Partitions= {2}, .AutoPartitioningSupport = autoscaleAwareSDK }); + // readSession->Run(); + // readSession->WaitAllMessages(); - Sleep(TDuration::Seconds(1)); + // Sleep(TDuration::Seconds(1)); - for(const auto& info : readSession->GetReceivedMessages()) { - if (info.Data == "message_2.1") { - UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); - UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); - } else { - UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - } - } + // for(const auto& info : readSession->GetReceivedMessages()) { + // if (info.Data == "message_2.1") { + // UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); + // UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); + // } else { + // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + // } + // } - writeSession->Close(TDuration::Seconds(1)); - readSession->Close(); - } + // writeSession->Close(TDuration::Seconds(1)); + // readSession->Close(); + // } - Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_BeforeAutoscaleAwareSDK) { - ReadingAfterSplitTest_PreferedPartition(SdkVersion::Topic, false); - } + // Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_BeforeAutoscaleAwareSDK) { + // ReadingAfterSplitTest_PreferedPartition(SdkVersion::Topic, false); + // } - Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_AutoscaleAwareSDK) { - ReadingAfterSplitTest_PreferedPartition(SdkVersion::Topic, true); - } + // Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_AutoscaleAwareSDK) { + // ReadingAfterSplitTest_PreferedPartition(SdkVersion::Topic, true); + // } - Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_PQv1) { - ReadingAfterSplitTest_PreferedPartition(SdkVersion::PQv1, false); - } + // Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_PQv1) { + // ReadingAfterSplitTest_PreferedPartition(SdkVersion::PQv1, false); + // } void PartitionSplit_oldSDK(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - TTopicClient client = setup.MakeClient(); + // TTopicClient client = setup.MakeClient(); - auto writeSession = CreateWriteSession(client, "producer-1"); + // auto writeSession = CreateWriteSession(client, "producer-1"); - auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 2, .AutoCommit = false, .AutoPartitioningSupport = false }); - readSession->Run(); + // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 2, .AutoCommit = false, .AutoPartitioningSupport = false }); + // readSession->Run(); - UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); + // UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); - ui64 txId = 1006; - SplitPartition(setup, ++txId, 0, "a"); + // ui64 txId = 1006; + // SplitPartition(setup, ++txId, 0, "a"); - UNIT_ASSERT(writeSession->Write(Msg("message_1.2", 3))); + // UNIT_ASSERT(writeSession->Write(Msg("message_1.2", 3))); - Sleep(TDuration::Seconds(1)); // Wait read session events + // Sleep(TDuration::Seconds(1)); // Wait read session events - readSession->WaitAndAssertPartitions({0}, "We are reading only one partition because offset is not commited"); - readSession->Run(); - readSession->SetAutoCommit(true); - readSession->Commit(); - readSession->WaitAndAssertPartitions({0, 1, 2}, "We are reading all partitions because offset is commited"); - readSession->Run(); + // readSession->WaitAndAssertPartitions({0}, "We are reading only one partition because offset is not commited"); + // readSession->Run(); + // readSession->SetAutoCommit(true); + // readSession->Commit(); + // readSession->WaitAndAssertPartitions({0, 1, 2}, "We are reading all partitions because offset is commited"); + // readSession->Run(); - readSession->WaitAllMessages(); + // readSession->WaitAllMessages(); - for(const auto& info : readSession->GetReceivedMessages()) { - if (info.Data == "message_1.1") { - UNIT_ASSERT_EQUAL(0, info.PartitionId); - UNIT_ASSERT_EQUAL(2, info.SeqNo); - } else if (info.Data == "message_1.2") { - UNIT_ASSERT(1 == info.PartitionId || 2 == info.PartitionId); - UNIT_ASSERT_EQUAL(3, info.SeqNo); - } else { - UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - } - } + // for(const auto& info : readSession->GetReceivedMessages()) { + // if (info.Data == "message_1.1") { + // UNIT_ASSERT_EQUAL(0, info.PartitionId); + // UNIT_ASSERT_EQUAL(2, info.SeqNo); + // } else if (info.Data == "message_1.2") { + // UNIT_ASSERT(1 == info.PartitionId || 2 == info.PartitionId); + // UNIT_ASSERT_EQUAL(3, info.SeqNo); + // } else { + // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + // } + // } - writeSession->Close(TDuration::Seconds(1)); - readSession->Close(); - } + // writeSession->Close(TDuration::Seconds(1)); + // readSession->Close(); + // } - Y_UNIT_TEST(PartitionSplit_BeforeAutoscaleAwareSDK) { - PartitionSplit_oldSDK(SdkVersion::Topic); - } + // Y_UNIT_TEST(PartitionSplit_BeforeAutoscaleAwareSDK) { + // PartitionSplit_oldSDK(SdkVersion::Topic); + // } - Y_UNIT_TEST(PartitionSplit_PQv1) { - PartitionSplit_oldSDK(SdkVersion::PQv1); - } + // Y_UNIT_TEST(PartitionSplit_PQv1) { + // PartitionSplit_oldSDK(SdkVersion::PQv1); + // } Y_UNIT_TEST(PartitionSplit_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); @@ -261,28 +261,28 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { } } - UNIT_ASSERT_VALUES_EQUAL_C(1, readSession->GetEndedPartitionEvents().size(), "Only one partition was ended"); - auto ev = readSession->GetEndedPartitionEvents().front(); - UNIT_ASSERT_VALUES_EQUAL_C(std::vector{}, ev.AdjacentPartitionIds, "There isn`t adjacent partitions after split"); - std::vector children = {1, 2}; - UNIT_ASSERT_VALUES_EQUAL_C(children, ev.ChildPartitionIds, ""); + // UNIT_ASSERT_VALUES_EQUAL_C(1, readSession->GetEndedPartitionEvents().size(), "Only one partition was ended"); + // auto ev = readSession->GetEndedPartitionEvents().front(); + // UNIT_ASSERT_VALUES_EQUAL_C(std::vector{}, ev.AdjacentPartitionIds, "There isn`t adjacent partitions after split"); + // std::vector children = {1, 2}; + // UNIT_ASSERT_VALUES_EQUAL_C(children, ev.ChildPartitionIds, ""); - writeSession->Close(TDuration::Seconds(1)); - readSession->Close(); - } + // writeSession->Close(TDuration::Seconds(1)); + // readSession->Close(); + // } void PartitionSplit_PreferedPartition(SdkVersion sdk, bool autoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - TTopicClient client = setup.MakeClient(); + // TTopicClient client = setup.MakeClient(); - auto writeSession1 = CreateWriteSession(client, "producer-1"); - auto writeSession2 = CreateWriteSession(client, "producer-2"); - auto writeSession3 = CreateWriteSession(client, "producer-3", 0); + // auto writeSession1 = CreateWriteSession(client, "producer-1"); + // auto writeSession2 = CreateWriteSession(client, "producer-2"); + // auto writeSession3 = CreateWriteSession(client, "producer-3", 0); - auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 6, .AutoCommit = !autoscaleAwareSDK, .AutoPartitioningSupport = autoscaleAwareSDK }); - readSession->Run(); + // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 6, .AutoCommit = !autoscaleAwareSDK, .AutoPartitioningSupport = autoscaleAwareSDK }); + // readSession->Run(); UNIT_ASSERT(writeSession1->Write(Msg("message_1.1", 2))); UNIT_ASSERT(writeSession2->Write(Msg("message_2.1", 3))); @@ -367,507 +367,507 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(writeSession1->Write(Msg("message_1.1", 2))); UNIT_ASSERT(writeSession2->Write(Msg("message_2.1", 3))); - ui64 txId = 1012; - MergePartition(setup, ++txId, 0, 1); - - UNIT_ASSERT(writeSession1->Write(Msg("message_1.2", 5))); // Will be fail because partition is not writable after merge - UNIT_ASSERT(writeSession2->Write(Msg("message_2.2", 7))); // Will be fail because partition is not writable after merge - - auto writeSession3 = CreateWriteSession(client, "producer-2", 2); - - UNIT_ASSERT(writeSession3->Write(Msg("message_3.1", 2))); // Will be ignored because duplicated SeqNo - UNIT_ASSERT(writeSession3->Write(Msg("message_3.2", 11))); - - readSession->WaitAllMessages(); - - for(const auto& info : readSession->GetReceivedMessages()) { - if (info.Data == TString("message_1.1")) { - UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); - UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); - } else if (info.Data == TString("message_2.1")) { - UNIT_ASSERT_VALUES_EQUAL(1, info.PartitionId); - UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); - } else if (info.Data == TString("message_3.2")) { - UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); - UNIT_ASSERT_VALUES_EQUAL(11, info.SeqNo); - } else { - UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - } - } - - if (autoscaleAwareSDK) { - UNIT_ASSERT_VALUES_EQUAL_C(2, readSession->GetEndedPartitionEvents().size(), "Two partition was ended which was merged"); - for (auto ev : readSession->GetEndedPartitionEvents()) { - UNIT_ASSERT(ev.AdjacentPartitionIds == std::vector{0} || ev.AdjacentPartitionIds == std::vector{1}); - UNIT_ASSERT_VALUES_EQUAL_C(std::vector{2}, ev.ChildPartitionIds, ""); - } - } else { - UNIT_ASSERT_VALUES_EQUAL_C(0, readSession->GetEndedPartitionEvents().size(), "OLD SDK"); - } - - - writeSession1->Close(TDuration::Seconds(2)); - writeSession2->Close(TDuration::Seconds(2)); - writeSession3->Close(TDuration::Seconds(2)); - readSession->Close(); - } - - Y_UNIT_TEST(PartitionMerge_PreferedPartition_BeforeAutoscaleAwareSDK) { - PartitionMerge_PreferedPartition(SdkVersion::Topic, false); - } - - Y_UNIT_TEST(PartitionMerge_PreferedPartition_AutoscaleAwareSDK) { - PartitionMerge_PreferedPartition(SdkVersion::Topic, true); - } - - Y_UNIT_TEST(PartitionMerge_PreferedPartition_PQv1) { - PartitionMerge_PreferedPartition(SdkVersion::Topic, false); - } + // ui64 txId = 1012; + // MergePartition(setup, ++txId, 0, 1); + + // UNIT_ASSERT(writeSession1->Write(Msg("message_1.2", 5))); // Will be fail because partition is not writable after merge + // UNIT_ASSERT(writeSession2->Write(Msg("message_2.2", 7))); // Will be fail because partition is not writable after merge + + // auto writeSession3 = CreateWriteSession(client, "producer-2", 2); + + // UNIT_ASSERT(writeSession3->Write(Msg("message_3.1", 2))); // Will be ignored because duplicated SeqNo + // UNIT_ASSERT(writeSession3->Write(Msg("message_3.2", 11))); + + // readSession->WaitAllMessages(); + + // for(const auto& info : readSession->GetReceivedMessages()) { + // if (info.Data == TString("message_1.1")) { + // UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); + // UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); + // } else if (info.Data == TString("message_2.1")) { + // UNIT_ASSERT_VALUES_EQUAL(1, info.PartitionId); + // UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); + // } else if (info.Data == TString("message_3.2")) { + // UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); + // UNIT_ASSERT_VALUES_EQUAL(11, info.SeqNo); + // } else { + // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + // } + // } + + // if (autoscaleAwareSDK) { + // UNIT_ASSERT_VALUES_EQUAL_C(2, readSession->GetEndedPartitionEvents().size(), "Two partition was ended which was merged"); + // for (auto ev : readSession->GetEndedPartitionEvents()) { + // UNIT_ASSERT(ev.AdjacentPartitionIds == std::vector{0} || ev.AdjacentPartitionIds == std::vector{1}); + // UNIT_ASSERT_VALUES_EQUAL_C(std::vector{2}, ev.ChildPartitionIds, ""); + // } + // } else { + // UNIT_ASSERT_VALUES_EQUAL_C(0, readSession->GetEndedPartitionEvents().size(), "OLD SDK"); + // } + + + // writeSession1->Close(TDuration::Seconds(2)); + // writeSession2->Close(TDuration::Seconds(2)); + // writeSession3->Close(TDuration::Seconds(2)); + // readSession->Close(); + // } + + // Y_UNIT_TEST(PartitionMerge_PreferedPartition_BeforeAutoscaleAwareSDK) { + // PartitionMerge_PreferedPartition(SdkVersion::Topic, false); + // } + + // Y_UNIT_TEST(PartitionMerge_PreferedPartition_AutoscaleAwareSDK) { + // PartitionMerge_PreferedPartition(SdkVersion::Topic, true); + // } + + // Y_UNIT_TEST(PartitionMerge_PreferedPartition_PQv1) { + // PartitionMerge_PreferedPartition(SdkVersion::Topic, false); + // } void PartitionSplit_ReadEmptyPartitions(SdkVersion sdk, bool autoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - TTopicClient client = setup.MakeClient(); - auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoPartitioningSupport = autoscaleAwareSDK }); + // TTopicClient client = setup.MakeClient(); + // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoPartitioningSupport = autoscaleAwareSDK }); - readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); + // readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); - ui64 txId = 1023; - SplitPartition(setup, ++txId, 0, "a"); + // ui64 txId = 1023; + // SplitPartition(setup, ++txId, 0, "a"); - readSession->WaitAndAssertPartitions({0, 1, 2}, "After split must read all partitions because parent partition is empty"); + // readSession->WaitAndAssertPartitions({0, 1, 2}, "After split must read all partitions because parent partition is empty"); - readSession->Close(); - } + // readSession->Close(); + // } - Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_BeforeAutoscaleAwareSDK) { - PartitionSplit_ReadEmptyPartitions(SdkVersion::Topic, false); - } + // Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_BeforeAutoscaleAwareSDK) { + // PartitionSplit_ReadEmptyPartitions(SdkVersion::Topic, false); + // } - Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_AutoscaleAwareSDK) { - PartitionSplit_ReadEmptyPartitions(SdkVersion::Topic, true); - } + // Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_AutoscaleAwareSDK) { + // PartitionSplit_ReadEmptyPartitions(SdkVersion::Topic, true); + // } - Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_PQv1) { - PartitionSplit_ReadEmptyPartitions(SdkVersion::PQv1, false); - } + // Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_PQv1) { + // PartitionSplit_ReadEmptyPartitions(SdkVersion::PQv1, false); + // } void PartitionSplit_ReadNotEmptyPartitions(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - TTopicClient client = setup.MakeClient(); - auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .AutoPartitioningSupport = false }); + // TTopicClient client = setup.MakeClient(); + // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .AutoPartitioningSupport = false }); - auto writeSession = CreateWriteSession(client, "producer-1", 0); + // auto writeSession = CreateWriteSession(client, "producer-1", 0); - readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); + // readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); - UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - ui64 txId = 1023; - SplitPartition(setup, ++txId, 0, "a"); + // ui64 txId = 1023; + // SplitPartition(setup, ++txId, 0, "a"); - readSession->WaitAndAssertPartitions({0}, "After split must read only 0 partition because had been read not from the end of partition"); - readSession->WaitAndAssertPartitions({}, "Partition must be released for secondary read after 1 second"); - readSession->WaitAndAssertPartitions({0}, "Must secondary read for check read from end"); - readSession->WaitAndAssertPartitions({}, "Partition must be released for secondary read because start not from the end of partition after 2 seconds"); + // readSession->WaitAndAssertPartitions({0}, "After split must read only 0 partition because had been read not from the end of partition"); + // readSession->WaitAndAssertPartitions({}, "Partition must be released for secondary read after 1 second"); + // readSession->WaitAndAssertPartitions({0}, "Must secondary read for check read from end"); + // readSession->WaitAndAssertPartitions({}, "Partition must be released for secondary read because start not from the end of partition after 2 seconds"); - readSession->SetOffset(0, 1); - readSession->WaitAndAssertPartitions({0, 1, 2}, "Must read from all partitions because had been read from the end of partition"); + // readSession->SetOffset(0, 1); + // readSession->WaitAndAssertPartitions({0, 1, 2}, "Must read from all partitions because had been read from the end of partition"); - readSession->Close(); - } + // readSession->Close(); + // } - Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_BeforeAutoscaleAwareSDK) { - PartitionSplit_ReadNotEmptyPartitions(SdkVersion::Topic); - } + // Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_BeforeAutoscaleAwareSDK) { + // PartitionSplit_ReadNotEmptyPartitions(SdkVersion::Topic); + // } - Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_PQv1) { - PartitionSplit_ReadNotEmptyPartitions(SdkVersion::PQv1); - } + // Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_PQv1) { + // PartitionSplit_ReadNotEmptyPartitions(SdkVersion::PQv1); + // } Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - TTopicClient client = setup.MakeClient(); - auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); + // TTopicClient client = setup.MakeClient(); + // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); - auto writeSession = CreateWriteSession(client, "producer-1", 0); + // auto writeSession = CreateWriteSession(client, "producer-1", 0); - readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); + // readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); - UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - ui64 txId = 1023; - SplitPartition(setup, ++txId, 0, "a"); + // ui64 txId = 1023; + // SplitPartition(setup, ++txId, 0, "a"); - readSession->WaitAndAssertPartitions({0, 1, 2}, "Must read from all partitions because used new SDK"); + // readSession->WaitAndAssertPartitions({0, 1, 2}, "Must read from all partitions because used new SDK"); - readSession->Close(); - } + // readSession->Close(); + // } void PartitionSplit_ManySession(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - TTopicClient client = setup.MakeClient(); + // TTopicClient client = setup.MakeClient(); - auto writeSession = CreateWriteSession(client, "producer-1", 0); - UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + // auto writeSession = CreateWriteSession(client, "producer-1", 0); + // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - ui64 txId = 1023; - SplitPartition(setup, ++txId, 0, "a"); + // ui64 txId = 1023; + // SplitPartition(setup, ++txId, 0, "a"); - auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .Partitions = {0, 1, 2}, .AutoPartitioningSupport = false }); - readSession1->SetOffset(0, 1); - readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because read the partition 0 from offset 1"); - readSession1->SetOffset(0, 0); + // auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .Partitions = {0, 1, 2}, .AutoPartitioningSupport = false }); + // readSession1->SetOffset(0, 1); + // readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because read the partition 0 from offset 1"); + // readSession1->SetOffset(0, 0); - auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .Partitions = {0}, .AutoPartitioningSupport = false }); - readSession2->SetOffset(0, 0); + // auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .Partitions = {0}, .AutoPartitioningSupport = false }); + // readSession2->SetOffset(0, 0); - readSession2->WaitAndAssertPartitions({0}, "Must read partition 0 because it defined in the readSession"); - readSession2->Run(); + // readSession2->WaitAndAssertPartitions({0}, "Must read partition 0 because it defined in the readSession"); + // readSession2->Run(); - readSession1->WaitAndAssertPartitions({}, "Must release all partitions becase readSession2 read not from EndOffset"); - readSession1->Run(); + // readSession1->WaitAndAssertPartitions({}, "Must release all partitions becase readSession2 read not from EndOffset"); + // readSession1->Run(); - readSession1->WaitAndAssertPartitions({0}, "Partition 0 must rebalance to other sessions (Session-0)"); + // readSession1->WaitAndAssertPartitions({0}, "Partition 0 must rebalance to other sessions (Session-0)"); - readSession1->Close(); - readSession2->Close(); - } + // readSession1->Close(); + // readSession2->Close(); + // } - Y_UNIT_TEST(PartitionSplit_ManySession_BeforeAutoscaleAwareSDK) { - PartitionSplit_ManySession(SdkVersion::Topic); - } + // Y_UNIT_TEST(PartitionSplit_ManySession_BeforeAutoscaleAwareSDK) { + // PartitionSplit_ManySession(SdkVersion::Topic); + // } - Y_UNIT_TEST(PartitionSplit_ManySession_PQv1) { - PartitionSplit_ManySession(SdkVersion::PQv1); - } + // Y_UNIT_TEST(PartitionSplit_ManySession_PQv1) { + // PartitionSplit_ManySession(SdkVersion::PQv1); + // } Y_UNIT_TEST(PartitionSplit_ManySession_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - TTopicClient client = setup.MakeClient(); + // TTopicClient client = setup.MakeClient(); - auto writeSession = CreateWriteSession(client, "producer-1", 0); - UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + // auto writeSession = CreateWriteSession(client, "producer-1", 0); + // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - ui64 txId = 1023; - SplitPartition(setup, ++txId, 0, "a"); + // ui64 txId = 1023; + // SplitPartition(setup, ++txId, 0, "a"); - Sleep(TDuration::Seconds(1)); + // Sleep(TDuration::Seconds(1)); - auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); + // auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); - readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because used new SDK"); - readSession1->Commit(); - readSession1->Run(); + // readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because used new SDK"); + // readSession1->Commit(); + // readSession1->Run(); - auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); - readSession2->Run(); + // auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); + // readSession2->Run(); - Sleep(TDuration::Seconds(1)); + // Sleep(TDuration::Seconds(1)); - auto p1 = readSession1->GetPartitions(); - auto p2 = readSession2->GetPartitions(); + // auto p1 = readSession1->GetPartitions(); + // auto p2 = readSession2->GetPartitions(); - std::set partitions; - partitions.insert(p1.begin(), p1.end()); - partitions.insert(p2.begin(), p2.end()); + // std::set partitions; + // partitions.insert(p1.begin(), p1.end()); + // partitions.insert(p2.begin(), p2.end()); - std::set expected{0, 1, 2}; + // std::set expected{0, 1, 2}; - UNIT_ASSERT_VALUES_EQUAL(expected, partitions); + // UNIT_ASSERT_VALUES_EQUAL(expected, partitions); - readSession1->Close(); + // readSession1->Close(); - Sleep(TDuration::Seconds(1)); + // Sleep(TDuration::Seconds(1)); - UNIT_ASSERT_VALUES_EQUAL(expected, readSession2->GetPartitions()); + // UNIT_ASSERT_VALUES_EQUAL(expected, readSession2->GetPartitions()); - readSession2->Close(); - } + // readSession2->Close(); + // } Y_UNIT_TEST(PartitionSplit_ManySession_existed_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - TTopicClient client = setup.MakeClient(); + // TTopicClient client = setup.MakeClient(); - auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); - auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .Partitions = {0}, .AutoPartitioningSupport = true }); + // auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); + // auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .Partitions = {0}, .AutoPartitioningSupport = true }); - auto writeSession = CreateWriteSession(client, "producer-1", 0); - UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + // auto writeSession = CreateWriteSession(client, "producer-1", 0); + // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - ui64 txId = 1023; - SplitPartition(setup, ++txId, 0, "a"); + // ui64 txId = 1023; + // SplitPartition(setup, ++txId, 0, "a"); - readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because used new SDK"); - readSession1->Commit(); - readSession2->Run(); + // readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because used new SDK"); + // readSession1->Commit(); + // readSession2->Run(); - readSession2->WaitAndAssertPartitions({0}, "Must read partition 0 because it defined in the readSession"); - readSession2->Run(); + // readSession2->WaitAndAssertPartitions({0}, "Must read partition 0 because it defined in the readSession"); + // readSession2->Run(); - readSession1->WaitAndAssertPartitions({1, 2}, "Partition 0 must rebalance to other sessions (Session-0)"); + // readSession1->WaitAndAssertPartitions({1, 2}, "Partition 0 must rebalance to other sessions (Session-0)"); - readSession1->Close(); - readSession2->Close(); - } + // readSession1->Close(); + // readSession2->Close(); + // } Y_UNIT_TEST(CommitTopPast_BeforeAutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - TTopicClient client = setup.MakeClient(); - - auto writeSession = CreateWriteSession(client, "producer-1", 0); - UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - UNIT_ASSERT(writeSession->Write(Msg("message_2", 3))); - - ui64 txId = 1023; - SplitPartition(setup, ++txId, 0, "a"); - - auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer has just started reading the inactive partition and he can commit"); - - status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 1).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "A consumer who has not read to the end can commit messages forward."); - - status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "A consumer who has not read to the end can commit messages back."); - - status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 2).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer can commit at the end of the inactive partition."); - - status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::BAD_REQUEST, status.GetStatus(), "The consumer cannot commit an offset for inactive, read-to-the-end partitions."); - } - - Y_UNIT_TEST(ControlPlane_CreateAlterDescribe) { - auto autoscalingTestTopic = "autoscalit-topic"; - TTopicSdkTestSetup setup = CreateSetup(); - TTopicClient client = setup.MakeClient(); - - auto minParts = 5; - auto maxParts = 10; - auto scaleUpPercent = 80; - auto scaleDownPercent = 20; - auto threshold = 500; - auto strategy = EAutoPartitioningStrategy::ScaleUp; - - TCreateTopicSettings createSettings; - createSettings - .BeginConfigurePartitioningSettings() - .MinActivePartitions(minParts) - .MaxActivePartitions(maxParts) - .BeginConfigureAutoPartitioningSettings() - .UpUtilizationPercent(scaleUpPercent) - .DownUtilizationPercent(scaleDownPercent) - .StabilizationWindow(TDuration::Seconds(threshold)) - .Strategy(strategy) - .EndConfigureAutoPartitioningSettings() - .EndConfigurePartitioningSettings(); - client.CreateTopic(autoscalingTestTopic, createSettings).Wait(); - - TDescribeTopicSettings descSettings; - - auto describe = client.DescribeTopic(autoscalingTestTopic, descSettings).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(describe.GetStatus(), NYdb::EStatus::SUCCESS, describe.GetIssues().ToString()); - - - UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), minParts); - UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), maxParts); - UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), strategy); - UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), scaleDownPercent); - UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), scaleUpPercent); - UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), threshold); - - auto alterMinParts = 10; - auto alterMaxParts = 20; - auto alterScaleUpPercent = 90; - auto alterScaleDownPercent = 10; - auto alterThreshold = 700; - auto alterStrategy = EAutoPartitioningStrategy::ScaleUpAndDown; - - TAlterTopicSettings alterSettings; - alterSettings - .BeginAlterPartitioningSettings() - .MinActivePartitions(alterMinParts) - .MaxActivePartitions(alterMaxParts) - .BeginAlterAutoPartitioningSettings() - .DownUtilizationPercent(alterScaleDownPercent) - .UpUtilizationPercent(alterScaleUpPercent) - .StabilizationWindow(TDuration::Seconds(alterThreshold)) - .Strategy(alterStrategy) - .EndAlterAutoPartitioningSettings() - .EndAlterTopicPartitioningSettings(); - - client.AlterTopic(autoscalingTestTopic, alterSettings).Wait(); - - auto describeAfterAlter = client.DescribeTopic(autoscalingTestTopic).GetValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), alterMinParts); - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), alterMaxParts); - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), alterStrategy); - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), alterScaleUpPercent); - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), alterScaleDownPercent); - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), alterThreshold); - } - - Y_UNIT_TEST(ControlPlane_DisableAutoPartitioning) { - auto topicName = "autoscalit-topic"; - - TTopicSdkTestSetup setup = CreateSetup(); - TTopicClient client = setup.MakeClient(); - - { - TCreateTopicSettings createSettings; - createSettings - .BeginConfigurePartitioningSettings() - .MinActivePartitions(1) - .MaxActivePartitions(100) - .BeginConfigureAutoPartitioningSettings() - .Strategy(EAutoPartitioningStrategy::ScaleUp) - .EndConfigureAutoPartitioningSettings() - .EndConfigurePartitioningSettings(); - client.CreateTopic(topicName, createSettings).Wait(); - } - - { - TAlterTopicSettings alterSettings; - alterSettings - .BeginAlterPartitioningSettings() - .BeginAlterAutoPartitioningSettings() - .Strategy(EAutoPartitioningStrategy::Disabled) - .EndAlterAutoPartitioningSettings() - .EndAlterTopicPartitioningSettings(); - auto f = client.AlterTopic(topicName, alterSettings); - f.Wait(); - - auto v = f.GetValueSync(); - UNIT_ASSERT_C(!v.IsSuccess(), "Must receve error becuse disabling is not supported"); - } - } - - Y_UNIT_TEST(ControlPlane_BackCompatibility) { - auto topicName = "back-compatibility-test"; - - TTopicSdkTestSetup setup = CreateSetup(); - TTopicClient client = setup.MakeClient(); - - { - TCreateTopicSettings createSettings; - createSettings - .BeginConfigurePartitioningSettings() - .MinActivePartitions(3) - .EndConfigurePartitioningSettings(); - client.CreateTopic(topicName, createSettings).Wait(); - } - - { - auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 3); - } - - { - TAlterTopicSettings alterSettings; - alterSettings - .BeginAlterPartitioningSettings() - .MinActivePartitions(5) - .EndAlterTopicPartitioningSettings(); - client.AlterTopic(topicName, alterSettings).Wait(); - } - - { - auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 5); - } - } - - Y_UNIT_TEST(ControlPlane_PauseAutoPartitioning) { - auto topicName = "autoscalit-topic"; - - TTopicSdkTestSetup setup = CreateSetup(); - TTopicClient client = setup.MakeClient(); - - { - TCreateTopicSettings createSettings; - createSettings - .BeginConfigurePartitioningSettings() - .MinActivePartitions(1) - .MaxActivePartitions(100) - .BeginConfigureAutoPartitioningSettings() - .Strategy(EAutoPartitioningStrategy::ScaleUp) - .EndConfigureAutoPartitioningSettings() - .EndConfigurePartitioningSettings(); - client.CreateTopic(topicName, createSettings).Wait(); - } - - { - TAlterTopicSettings alterSettings; - alterSettings - .BeginAlterPartitioningSettings() - .MinActivePartitions(3) - .MaxActivePartitions(107) - .BeginAlterAutoPartitioningSettings() - .Strategy(EAutoPartitioningStrategy::Paused) - .EndAlterAutoPartitioningSettings() - .EndAlterTopicPartitioningSettings(); - auto f = client.AlterTopic(topicName, alterSettings); - f.Wait(); - - auto v = f.GetValueSync(); - UNIT_ASSERT_C(v.IsSuccess(), "Error: " << v); - } - - { - auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 3); - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), 107); - UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), EAutoPartitioningStrategy::Paused); - } - } - - Y_UNIT_TEST(ControlPlane_AutoscalingWithStorageSizeRetention) { - auto autoscalingTestTopic = "autoscalit-topic"; - TTopicSdkTestSetup setup = CreateSetup(); - TTopicClient client = setup.MakeClient(); - - TCreateTopicSettings createSettings; - createSettings - .RetentionStorageMb(1024) - .BeginConfigurePartitioningSettings() - .BeginConfigureAutoPartitioningSettings() - .Strategy(EAutoPartitioningStrategy::ScaleUp) - .EndConfigureAutoPartitioningSettings() - .EndConfigurePartitioningSettings(); - auto result = client.CreateTopic(autoscalingTestTopic, createSettings).GetValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::BAD_REQUEST); - - createSettings.RetentionStorageMb(0); - result = client.CreateTopic(autoscalingTestTopic, createSettings).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); - - TAlterTopicSettings alterSettings; - alterSettings - .SetRetentionStorageMb(1024); - - result = client.AlterTopic(autoscalingTestTopic, alterSettings).GetValueSync(); - - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::BAD_REQUEST); - } + // TTopicClient client = setup.MakeClient(); + + // auto writeSession = CreateWriteSession(client, "producer-1", 0); + // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + // UNIT_ASSERT(writeSession->Write(Msg("message_2", 3))); + + // ui64 txId = 1023; + // SplitPartition(setup, ++txId, 0, "a"); + + // auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); + // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer has just started reading the inactive partition and he can commit"); + + // status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 1).GetValueSync(); + // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "A consumer who has not read to the end can commit messages forward."); + + // status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); + // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "A consumer who has not read to the end can commit messages back."); + + // status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 2).GetValueSync(); + // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer can commit at the end of the inactive partition."); + + // status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); + // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::BAD_REQUEST, status.GetStatus(), "The consumer cannot commit an offset for inactive, read-to-the-end partitions."); + // } + + // Y_UNIT_TEST(ControlPlane_CreateAlterDescribe) { + // auto autoscalingTestTopic = "autoscalit-topic"; + // TTopicSdkTestSetup setup = CreateSetup(); + // TTopicClient client = setup.MakeClient(); + + // auto minParts = 5; + // auto maxParts = 10; + // auto scaleUpPercent = 80; + // auto scaleDownPercent = 20; + // auto threshold = 500; + // auto strategy = EAutoPartitioningStrategy::ScaleUp; + + // TCreateTopicSettings createSettings; + // createSettings + // .BeginConfigurePartitioningSettings() + // .MinActivePartitions(minParts) + // .MaxActivePartitions(maxParts) + // .BeginConfigureAutoPartitioningSettings() + // .UpUtilizationPercent(scaleUpPercent) + // .DownUtilizationPercent(scaleDownPercent) + // .StabilizationWindow(TDuration::Seconds(threshold)) + // .Strategy(strategy) + // .EndConfigureAutoPartitioningSettings() + // .EndConfigurePartitioningSettings(); + // client.CreateTopic(autoscalingTestTopic, createSettings).Wait(); + + // TDescribeTopicSettings descSettings; + + // auto describe = client.DescribeTopic(autoscalingTestTopic, descSettings).GetValueSync(); + // UNIT_ASSERT_VALUES_EQUAL_C(describe.GetStatus(), NYdb::EStatus::SUCCESS, describe.GetIssues().ToString()); + + + // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), minParts); + // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), maxParts); + // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), strategy); + // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), scaleDownPercent); + // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), scaleUpPercent); + // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), threshold); + + // auto alterMinParts = 10; + // auto alterMaxParts = 20; + // auto alterScaleUpPercent = 90; + // auto alterScaleDownPercent = 10; + // auto alterThreshold = 700; + // auto alterStrategy = EAutoPartitioningStrategy::ScaleUpAndDown; + + // TAlterTopicSettings alterSettings; + // alterSettings + // .BeginAlterPartitioningSettings() + // .MinActivePartitions(alterMinParts) + // .MaxActivePartitions(alterMaxParts) + // .BeginAlterAutoPartitioningSettings() + // .DownUtilizationPercent(alterScaleDownPercent) + // .UpUtilizationPercent(alterScaleUpPercent) + // .StabilizationWindow(TDuration::Seconds(alterThreshold)) + // .Strategy(alterStrategy) + // .EndAlterAutoPartitioningSettings() + // .EndAlterTopicPartitioningSettings(); + + // client.AlterTopic(autoscalingTestTopic, alterSettings).Wait(); + + // auto describeAfterAlter = client.DescribeTopic(autoscalingTestTopic).GetValueSync(); + + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), alterMinParts); + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), alterMaxParts); + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), alterStrategy); + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), alterScaleUpPercent); + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), alterScaleDownPercent); + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), alterThreshold); + // } + + // Y_UNIT_TEST(ControlPlane_DisableAutoPartitioning) { + // auto topicName = "autoscalit-topic"; + + // TTopicSdkTestSetup setup = CreateSetup(); + // TTopicClient client = setup.MakeClient(); + + // { + // TCreateTopicSettings createSettings; + // createSettings + // .BeginConfigurePartitioningSettings() + // .MinActivePartitions(1) + // .MaxActivePartitions(100) + // .BeginConfigureAutoPartitioningSettings() + // .Strategy(EAutoPartitioningStrategy::ScaleUp) + // .EndConfigureAutoPartitioningSettings() + // .EndConfigurePartitioningSettings(); + // client.CreateTopic(topicName, createSettings).Wait(); + // } + + // { + // TAlterTopicSettings alterSettings; + // alterSettings + // .BeginAlterPartitioningSettings() + // .BeginAlterAutoPartitioningSettings() + // .Strategy(EAutoPartitioningStrategy::Disabled) + // .EndAlterAutoPartitioningSettings() + // .EndAlterTopicPartitioningSettings(); + // auto f = client.AlterTopic(topicName, alterSettings); + // f.Wait(); + + // auto v = f.GetValueSync(); + // UNIT_ASSERT_C(!v.IsSuccess(), "Must receve error becuse disabling is not supported"); + // } + // } + + // Y_UNIT_TEST(ControlPlane_BackCompatibility) { + // auto topicName = "back-compatibility-test"; + + // TTopicSdkTestSetup setup = CreateSetup(); + // TTopicClient client = setup.MakeClient(); + + // { + // TCreateTopicSettings createSettings; + // createSettings + // .BeginConfigurePartitioningSettings() + // .MinActivePartitions(3) + // .EndConfigurePartitioningSettings(); + // client.CreateTopic(topicName, createSettings).Wait(); + // } + + // { + // auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); + + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 3); + // } + + // { + // TAlterTopicSettings alterSettings; + // alterSettings + // .BeginAlterPartitioningSettings() + // .MinActivePartitions(5) + // .EndAlterTopicPartitioningSettings(); + // client.AlterTopic(topicName, alterSettings).Wait(); + // } + + // { + // auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); + + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 5); + // } + // } + + // Y_UNIT_TEST(ControlPlane_PauseAutoPartitioning) { + // auto topicName = "autoscalit-topic"; + + // TTopicSdkTestSetup setup = CreateSetup(); + // TTopicClient client = setup.MakeClient(); + + // { + // TCreateTopicSettings createSettings; + // createSettings + // .BeginConfigurePartitioningSettings() + // .MinActivePartitions(1) + // .MaxActivePartitions(100) + // .BeginConfigureAutoPartitioningSettings() + // .Strategy(EAutoPartitioningStrategy::ScaleUp) + // .EndConfigureAutoPartitioningSettings() + // .EndConfigurePartitioningSettings(); + // client.CreateTopic(topicName, createSettings).Wait(); + // } + + // { + // TAlterTopicSettings alterSettings; + // alterSettings + // .BeginAlterPartitioningSettings() + // .MinActivePartitions(3) + // .MaxActivePartitions(107) + // .BeginAlterAutoPartitioningSettings() + // .Strategy(EAutoPartitioningStrategy::Paused) + // .EndAlterAutoPartitioningSettings() + // .EndAlterTopicPartitioningSettings(); + // auto f = client.AlterTopic(topicName, alterSettings); + // f.Wait(); + + // auto v = f.GetValueSync(); + // UNIT_ASSERT_C(v.IsSuccess(), "Error: " << v); + // } + + // { + // auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); + + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 3); + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), 107); + // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), EAutoPartitioningStrategy::Paused); + // } + // } + + // Y_UNIT_TEST(ControlPlane_AutoscalingWithStorageSizeRetention) { + // auto autoscalingTestTopic = "autoscalit-topic"; + // TTopicSdkTestSetup setup = CreateSetup(); + // TTopicClient client = setup.MakeClient(); + + // TCreateTopicSettings createSettings; + // createSettings + // .RetentionStorageMb(1024) + // .BeginConfigurePartitioningSettings() + // .BeginConfigureAutoPartitioningSettings() + // .Strategy(EAutoPartitioningStrategy::ScaleUp) + // .EndConfigureAutoPartitioningSettings() + // .EndConfigurePartitioningSettings(); + // auto result = client.CreateTopic(autoscalingTestTopic, createSettings).GetValueSync(); + + // UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::BAD_REQUEST); + + // createSettings.RetentionStorageMb(0); + // result = client.CreateTopic(autoscalingTestTopic, createSettings).GetValueSync(); + // UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); + + // TAlterTopicSettings alterSettings; + // alterSettings + // .SetRetentionStorageMb(1024); + + // result = client.AlterTopic(autoscalingTestTopic, alterSettings).GetValueSync(); + + // UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::BAD_REQUEST); + // } Y_UNIT_TEST(PartitionSplit_AutosplitByLoad) { TTopicSdkTestSetup setup = CreateSetup(); @@ -885,6 +885,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { .Strategy(EAutoPartitioningStrategy::ScaleUp) .EndConfigureAutoPartitioningSettings() .EndConfigurePartitioningSettings(); + + TConsumerSettings consumers(createSettings, TEST_CONSUMER); + createSettings.AppendConsumers(consumers); + client.CreateTopic(TEST_TOPIC, createSettings).Wait(); auto msg = TString(1_MB, 'a'); diff --git a/ydb/core/persqueue/ut/ut_with_sdk/ya.make b/ydb/core/persqueue/ut/ut_with_sdk/ya.make index fdb6c2ff4054..617ecf452d07 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/ya.make +++ b/ydb/core/persqueue/ut/ut_with_sdk/ya.make @@ -30,8 +30,8 @@ YQL_LAST_ABI_VERSION() SRCS( autoscaling_ut.cpp - balancing_ut.cpp - mirrorer_ut.cpp + # balancing_ut.cpp + # mirrorer_ut.cpp ) END() diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto index 46dee8ed2fd4..397b10e10085 100644 --- a/ydb/core/protos/kqp.proto +++ b/ydb/core/protos/kqp.proto @@ -69,10 +69,37 @@ enum EIsolationLevel { message TTopicOperationsRequest { optional string Consumer = 1; - repeated Ydb.Topic.UpdateOffsetsInTransactionRequest.TopicOffsets Topics = 2; + repeated TopicOffsets Topics = 2; optional uint32 SupportivePartition = 3; + + message TopicOffsets { + // Topic path. + optional string path = 1; + + // Ranges of offsets by partitions. + repeated PartitionOffsets partitions = 2; + + message PartitionOffsets { + // Partition identifier. + optional int64 partition_id = 1; + + // List of offset ranges. + repeated OffsetsRange partition_offsets = 2; + + optional bool force_commit = 3; + optional bool kill_read_session = 4; + optional bool only_check_commited_to_finish = 5; + + message OffsetsRange { + optional int64 start = 1; + optional int64 end = 2; + } + } + } } + + message TTopicOperationsResponse { message TWriteId { optional uint64 NodeId = 1; diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index b2f9d7097285..a03ff41ba084 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -952,11 +952,14 @@ message TYdsShardIterator { message TPartitionOperation { optional uint32 PartitionId = 1; - optional uint64 Begin = 2; - optional uint64 End = 3; + optional uint64 CommitOffsetsBegin = 2; + optional uint64 CommitOffsetsEnd = 3; optional string Consumer = 4; optional string Path = 5; // topic path optional uint32 SupportivePartition = 6; + optional bool ForceCommit = 7; + optional bool KillReadSession = 8; + optional bool OnlyCheckCommitedToFinish = 9; }; message TWriteId { diff --git a/ydb/services/lib/actors/type_definitions.h b/ydb/services/lib/actors/type_definitions.h index 6446d6965ab1..3b11dd91b1a2 100644 --- a/ydb/services/lib/actors/type_definitions.h +++ b/ydb/services/lib/actors/type_definitions.h @@ -1,5 +1,6 @@ #pragma once +#include "ydb/core/persqueue/utils.h" #include #include @@ -25,6 +26,7 @@ struct TTopicInitInfo { TString FolderId; NKikimrPQ::TPQTabletConfig::EMeteringMode MeteringMode; THashMap Partitions; + std::shared_ptr PartitionGraph; }; using TTopicInitInfoMap = THashMap; @@ -45,6 +47,7 @@ struct TTopicHolder { TVector Groups; THashMap Partitions; + std::shared_ptr PartitionGraph; inline static TTopicHolder FromTopicInfo(const TTopicInitInfo& info) { @@ -59,6 +62,7 @@ struct TTopicHolder { .MeteringMode = info.MeteringMode, .FullConverter = info.TopicNameConverter, .Partitions = info.Partitions, + .PartitionGraph = info.PartitionGraph // savnik тут будет пусто, т.к. в TTopicInitInfo это поле не сетится. Найти где оно инициализируется и исправить }; } }; diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index 9e51df7359ec..6ba1c332a53d 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -87,7 +87,6 @@ void TCommitOffsetActor::Die(const TActorContext& ctx) { } void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "CommitOffset auth ok, got " << ev->Get()->TopicAndTablets.size() << " topics"); TopicAndTablets = std::move(ev->Get()->TopicAndTablets); if (TopicAndTablets.empty()) { @@ -102,7 +101,47 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc return; } - ui64 tabletId = topicInitInfo.Partitions.at(PartitionId).TabletId; + auto commitRequest = dynamic_cast(GetProtoRequest()); + + auto* partitionNode = topicInitInfo.PartitionGraph->GetPartition(commitRequest->partition_id()); + + if (partitionNode->HierarhicalParents.size() == 0 && partitionNode->Children.size() == 0) { + SendCommit(topicInitInfo, commitRequest, ctx); + } else { + Kqp.Consumer = ClientId; + Kqp.DataBase = Request().GetDatabaseName().GetOrElse(TString()); // savnik if empty? + Kqp.Path = topic; + + for (auto& parent: partitionNode->HierarhicalParents) { + TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max()}; + Commits.push_back(commit); + } + + for (auto& child: partitionNode->Children) { + TKqpHelper::TCommitInfo commit {.PartitionId = child->Id, .Offset = 0}; + Commits.push_back(commit); + } + + TKqpHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->Getoffset()}; + Commits.push_back(commit); + + SendDistributedTxOffsets(ctx); + } +} + +void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const NActors::TActorContext& ctx) { + if (!Kqp.Handle(ev, ctx)) { + AnswerError("empty list of topics", PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx); // savnik + } + Kqp.BeginTransaction(ctx); +} + +void TCommitOffsetActor::SendDistributedTxOffsets(const TActorContext& ctx) { + Kqp.SendCreateSessionRequest(ctx); +} + +void TCommitOffsetActor::SendCommit(const TTopicInitInfo& topic, const Ydb::Topic::CommitOffsetRequest* commitRequest, const TActorContext& ctx) { + ui64 tabletId = topic.Partitions.at(PartitionId).TabletId; NTabletPipe::TClientConfig clientConfig; clientConfig.RetryPolicy = { @@ -113,23 +152,21 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc .DoFirstRetryInstantly = true }; - PipeClient = ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); - - auto client_req = dynamic_cast(GetProtoRequest()); + PipeClient = ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); // savnik close only in that case NKikimrClient::TPersQueueRequest request; - request.MutablePartitionRequest()->SetTopic(topicInitInfo.TopicNameConverter->GetPrimaryPath()); - request.MutablePartitionRequest()->SetPartition(client_req->partition_id()); + request.MutablePartitionRequest()->SetTopic(topic.TopicNameConverter->GetPrimaryPath()); + request.MutablePartitionRequest()->SetPartition(commitRequest->partition_id()); Y_ABORT_UNLESS(PipeClient); auto commit = request.MutablePartitionRequest()->MutableCmdSetClientOffset(); commit->SetClientId(ClientId); - commit->SetOffset(client_req->offset()); + commit->SetOffset(commitRequest->offset()); commit->SetStrict(true); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "strict CommitOffset, partition " << client_req->partition_id() - << " committing to position " << client_req->offset() /*<< " prev " << CommittedOffset + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "strict CommitOffset, partition " << commitRequest->partition_id() + << " committing to position " << commitRequest->offset() /*<< " prev " << CommittedOffset << " end " << EndOffset << " by cookie " << readId*/); TAutoPtr req(new TEvPersQueue::TEvRequest); @@ -138,6 +175,23 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc NTabletPipe::SendData(ctx, PipeClient, req.Release()); } +void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record; + + // savnik set error record.GetYdbStatus() + if (Kqp.Step == 0) { + Kqp.Step++; + Kqp.SendCommits(ev,Commits, ctx); + } else if (Kqp.Step == 1) { + Kqp.Step++; + Kqp.CommitTx(ctx); + } else { + Kqp.CloseKqpSession(ctx); + Ydb::Topic::CommitOffsetResult result; + Request().SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); + } +} void TCommitOffsetActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { if (ev->Get()->Record.GetStatus() != NMsgBusProxy::MSTATUS_OK) { @@ -186,5 +240,4 @@ void TCommitOffsetActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, con AnswerError(TStringBuilder() <<"pipe to tablet destroyed" << ev->Get()->TabletId, PersQueue::ErrorCode::TABLET_PIPE_DISCONNECTED, ctx); } - } diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.h b/ydb/services/persqueue_v1/actors/commit_offset_actor.h index 6eefc867ebb7..f0a945aa3bb3 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.h +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.h @@ -1,7 +1,11 @@ #pragma once #include "events.h" +#include "ydb/core/kqp/common/simple/services.h" +#include "ydb/services/metadata/service.h" + +#include #include #include @@ -12,6 +16,126 @@ namespace NKikimr::NGRpcProxy::V1 { using namespace NKikimr::NGRpcService; +class TKqpHelper { + + public: // savnik + + TString DataBase; + TString KqpSessionId; + TString Consumer; + TString Path; + TString TxId; + + int Step = 0; + + struct TCommitInfo { + ui64 PartitionId; + i64 Offset; + }; + + void SendCreateSessionRequest(const TActorContext& ctx) { + auto ev = MakeCreateSessionRequest(); + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); + } + + THolder MakeCreateSessionRequest() { + auto ev = MakeHolder(); + ev->Record.MutableRequest()->SetDatabase(DataBase); + return ev; + } + + bool Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& /*ctx*/) { + const auto& record = ev->Get()->Record; + + if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { + return false; + } + + KqpSessionId = record.GetResponse().GetSessionId(); + Y_ABORT_UNLESS(!KqpSessionId.empty()); + + return true; + } + + void CloseKqpSession(const TActorContext& ctx) { + if (KqpSessionId) { + auto ev = MakeCloseSessionRequest(); + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); + + KqpSessionId = ""; + } + } + + THolder MakeCloseSessionRequest() { + auto ev = MakeHolder(); + ev->Record.MutableRequest()->SetSessionId(KqpSessionId); + return ev; + } + + void SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, std::vector commits, const NActors::TActorContext& ctx) + { + // if (!AppData(ctx)->FeatureFlags.GetEnableTopicServiceTx()) { // savnik need this check? + // return Reply(Ydb::StatusIds::UNSUPPORTED, + // "Disabled transaction support for TopicService.", + // NKikimrIssues::TIssuesIds::DEFAULT_ERROR, + // ctx); + // } + + auto& record = ev->Get()->Record; + TxId = record.GetResponse().GetTxMeta().id(); + Y_ABORT_UNLESS(!TxId.empty()); + + + auto offsets = MakeHolder(); + offsets->Record.MutableRequest()->SetDatabase(DataBase); + offsets->Record.MutableRequest()->SetSessionId(KqpSessionId); + offsets->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_UNDEFINED); + offsets->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_TOPIC); + offsets->Record.MutableRequest()->MutableTxControl()->set_tx_id(TxId); + offsets->Record.MutableRequest()->MutableTopicOperations()->SetConsumer(Consumer); + + // savnik need set something else? + + auto* topic = offsets->Record.MutableRequest()->MutableTopicOperations()->AddTopics(); + + topic->set_path(Path); + + for(auto &c: commits) { + auto* partition = topic->add_partitions(); + partition->set_partition_id(c.PartitionId); + partition->set_force_commit(true); + partition->set_kill_read_session(true); + auto* offset = partition->add_partition_offsets(); + offset->set_end(c.Offset); + } + + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), offsets.Release()); + } + + void BeginTransaction(const NActors::TActorContext& ctx) { + auto begin = MakeHolder(); + + begin->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_BEGIN_TX); + begin->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write(); + begin->Record.MutableRequest()->SetSessionId(KqpSessionId); + begin->Record.MutableRequest()->SetDatabase(DataBase); + + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), begin.Release()); + } + + void CommitTx(const NActors::TActorContext& ctx) { + auto commit = MakeHolder(); + + commit->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_COMMIT_TX); + commit->Record.MutableRequest()->MutableTxControl()->set_tx_id(TxId); + commit->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true); + commit->Record.MutableRequest()->SetSessionId(KqpSessionId); + commit->Record.MutableRequest()->SetDatabase(DataBase); + + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), commit.Release()); + } +}; + class TCommitOffsetActor : public TRpcOperationRequestActor { using TBase = TRpcOperationRequestActor; @@ -52,6 +176,9 @@ class TCommitOffsetActor : public TRpcOperationRequestActor Commits; }; } diff --git a/ydb/services/persqueue_v1/actors/read_init_auth_actor.cpp b/ydb/services/persqueue_v1/actors/read_init_auth_actor.cpp index 320d8284db67..27592a7862d9 100644 --- a/ydb/services/persqueue_v1/actors/read_init_auth_actor.cpp +++ b/ydb/services/persqueue_v1/actors/read_init_auth_actor.cpp @@ -104,6 +104,7 @@ bool TReadInitAndAuthActor::ProcessTopicSchemeCacheResponse( topicsIter->second.MeteringMode = pqDescr.GetPQTabletConfig().GetMeteringMode(); topicsIter->second.DbPath = pqDescr.GetPQTabletConfig().GetYdbDatabasePath(); topicsIter->second.IsServerless = entry.DomainInfo->IsServerless(); + topicsIter->second.PartitionGraph = entry.PQGroupInfo->PartitionGraph; for (const auto& partitionDescription : pqDescr.GetPartitions()) { topicsIter->second.Partitions[partitionDescription.GetPartitionId()] = @@ -272,7 +273,16 @@ void TReadInitAndAuthActor::FinishInitialization(const TActorContext& ctx) { TTopicInitInfoMap res; for (auto& [name, holder] : Topics) { res.insert(std::make_pair(name, TTopicInitInfo{ - holder.FullConverter, holder.TabletID, holder.CloudId, holder.DbId, holder.DbPath, holder.IsServerless, holder.FolderId, holder.MeteringMode, holder.Partitions + holder.FullConverter, + holder.TabletID, + holder.CloudId, + holder.DbId, + holder.DbPath, + holder.IsServerless, + holder.FolderId, + holder.MeteringMode, + holder.Partitions, + holder.PartitionGraph })); } ctx.Send(ParentId, new TEvPQProxy::TEvAuthResultOk(std::move(res))); diff --git a/ydb/services/persqueue_v1/actors/update_offsets_in_transaction_actor.cpp b/ydb/services/persqueue_v1/actors/update_offsets_in_transaction_actor.cpp index 1c05509307b9..60f6e39edd40 100644 --- a/ydb/services/persqueue_v1/actors/update_offsets_in_transaction_actor.cpp +++ b/ydb/services/persqueue_v1/actors/update_offsets_in_transaction_actor.cpp @@ -61,7 +61,22 @@ void TUpdateOffsetsInTransactionActor::Proceed(const NActors::TActorContext& ctx ev->Record.MutableRequest()->MutableTxControl()->set_tx_id(req->tx().id()); ev->Record.MutableRequest()->MutableTopicOperations()->SetConsumer(req->consumer()); - *ev->Record.MutableRequest()->MutableTopicOperations()->MutableTopics() = req->topics(); + + for (const auto& topic : req->topics()) { + auto* newTopic = ev->Record.MutableRequest()->MutableTopicOperations()->AddTopics(); + newTopic->set_path(topic.path()); + + for (const auto& partition : topic.partitions()) { + auto* newPartition = newTopic->add_partitions(); + newPartition->set_partition_id(partition.partition_id()); + + for (const auto& offsetsRange : partition.partition_offsets()) { + auto* newOffsetsRange = newPartition->add_partition_offsets(); + newOffsetsRange->set_start(offsetsRange.start()); + newOffsetsRange->set_end(offsetsRange.end()); + } + } + } ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); } From 2b4b32ce39af29d0a3f7a2d5cef3fa5b84993749 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Thu, 14 Nov 2024 22:10:55 +0000 Subject: [PATCH 02/29] return uts --- .../ut/ut_with_sdk/autoscaling_ut.cpp | 1154 ++++++++--------- ydb/core/persqueue/ut/ut_with_sdk/ya.make | 4 +- 2 files changed, 577 insertions(+), 581 deletions(-) diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index c2f77843cd53..b39ad0c0c7fb 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -19,212 +19,212 @@ using namespace NKikimr::NPQ::NTest; Y_UNIT_TEST_SUITE(TopicAutoscaling) { - // void SimpleTest(SdkVersion sdk, bool autoscaleAwareSDK) { - // TTopicSdkTestSetup setup = CreateSetup(); - // setup.CreateTopicWithAutoscale(); + void SimpleTest(SdkVersion sdk, bool autoscaleAwareSDK) { + TTopicSdkTestSetup setup = CreateSetup(); + setup.CreateTopicWithAutoscale(); - // TTopicClient client = setup.MakeClient(); + TTopicClient client = setup.MakeClient(); - // auto writeSession1 = CreateWriteSession(client, "producer-1"); - // auto writeSession2 = CreateWriteSession(client, "producer-2"); + auto writeSession1 = CreateWriteSession(client, "producer-1"); + auto writeSession2 = CreateWriteSession(client, "producer-2"); - // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 2, .AutoCommit = !autoscaleAwareSDK, .AutoPartitioningSupport = autoscaleAwareSDK }); - // readSession->Run(); + auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 2, .AutoCommit = !autoscaleAwareSDK, .AutoPartitioningSupport = autoscaleAwareSDK }); + readSession->Run(); - // UNIT_ASSERT(writeSession1->Write(Msg("message_1.1", 2))); - // UNIT_ASSERT(writeSession2->Write(Msg("message_2.1", 3))); + UNIT_ASSERT(writeSession1->Write(Msg("message_1.1", 2))); + UNIT_ASSERT(writeSession2->Write(Msg("message_2.1", 3))); - // readSession->WaitAllMessages(); + readSession->WaitAllMessages(); - // for(const auto& info : readSession->GetReceivedMessages()) { - // if (info.Data == "message_1.1") { - // UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); - // UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); - // } else if (info.Data == "message_2.1") { - // UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); - // UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); - // } else { - // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - // } - // } + for(const auto& info : readSession->GetReceivedMessages()) { + if (info.Data == "message_1.1") { + UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); + UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); + } else if (info.Data == "message_2.1") { + UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); + UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); + } else { + UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + } + } - // writeSession1->Close(TDuration::Seconds(1)); - // writeSession2->Close(TDuration::Seconds(1)); - // readSession->Close(); - // } + writeSession1->Close(TDuration::Seconds(1)); + writeSession2->Close(TDuration::Seconds(1)); + readSession->Close(); + } - // Y_UNIT_TEST(Simple_BeforeAutoscaleAwareSDK) { - // SimpleTest(SdkVersion::Topic, false); - // } + Y_UNIT_TEST(Simple_BeforeAutoscaleAwareSDK) { + SimpleTest(SdkVersion::Topic, false); + } - // Y_UNIT_TEST(Simple_AutoscaleAwareSDK) { - // SimpleTest(SdkVersion::Topic, true); - // } + Y_UNIT_TEST(Simple_AutoscaleAwareSDK) { + SimpleTest(SdkVersion::Topic, true); + } - // Y_UNIT_TEST(Simple_PQv1) { - // SimpleTest(SdkVersion::PQv1, false); - // } + Y_UNIT_TEST(Simple_PQv1) { + SimpleTest(SdkVersion::PQv1, false); + } - // void ReadingAfterSplitTest(SdkVersion sdk, bool autoscaleAwareSDK, bool autoCommit) { - // TTopicSdkTestSetup setup = CreateSetup(); - // setup.CreateTopicWithAutoscale(); + void ReadingAfterSplitTest(SdkVersion sdk, bool autoscaleAwareSDK, bool autoCommit) { + TTopicSdkTestSetup setup = CreateSetup(); + setup.CreateTopicWithAutoscale(); - // TTopicClient client = setup.MakeClient(); + TTopicClient client = setup.MakeClient(); - // auto writeSession = CreateWriteSession(client, "producer-1"); + auto writeSession = CreateWriteSession(client, "producer-1"); - // UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); + UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); - // ui64 txId = 1006; - // SplitPartition(setup, ++txId, 0, "a"); + ui64 txId = 1006; + SplitPartition(setup, ++txId, 0, "a"); - // UNIT_ASSERT(writeSession->Write(Msg("message_2.1", 3))); + UNIT_ASSERT(writeSession->Write(Msg("message_2.1", 3))); - // SplitPartition(setup, ++txId, 2, "d"); + SplitPartition(setup, ++txId, 2, "d"); - // UNIT_ASSERT(writeSession->Write(Msg("message_3.1", 5))); + UNIT_ASSERT(writeSession->Write(Msg("message_3.1", 5))); - // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 3, .AutoCommit = autoCommit, .AutoPartitioningSupport = autoscaleAwareSDK }); - // readSession->Run(); - // readSession->WaitAllMessages(); + auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 3, .AutoCommit = autoCommit, .AutoPartitioningSupport = autoscaleAwareSDK }); + readSession->Run(); + readSession->WaitAllMessages(); - // for(const auto& info : readSession->GetReceivedMessages()) { - // if (info.Data == "message_1.1") { - // UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); - // UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); - // } else if (info.Data == "message_2.1") { - // UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); - // UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); - // } else if (info.Data == "message_3.1") { - // UNIT_ASSERT_VALUES_EQUAL(4, info.PartitionId); - // UNIT_ASSERT_VALUES_EQUAL(5, info.SeqNo); - // } else { - // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - // } - // } + for(const auto& info : readSession->GetReceivedMessages()) { + if (info.Data == "message_1.1") { + UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); + UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); + } else if (info.Data == "message_2.1") { + UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); + UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); + } else if (info.Data == "message_3.1") { + UNIT_ASSERT_VALUES_EQUAL(4, info.PartitionId); + UNIT_ASSERT_VALUES_EQUAL(5, info.SeqNo); + } else { + UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + } + } - // writeSession->Close(TDuration::Seconds(1)); - // readSession->Close(); - // } + writeSession->Close(TDuration::Seconds(1)); + readSession->Close(); + } - // Y_UNIT_TEST(ReadingAfterSplitTest_BeforeAutoscaleAwareSDK) { - // ReadingAfterSplitTest(SdkVersion::Topic, false, true); - // } + Y_UNIT_TEST(ReadingAfterSplitTest_BeforeAutoscaleAwareSDK) { + ReadingAfterSplitTest(SdkVersion::Topic, false, true); + } - // Y_UNIT_TEST(ReadingAfterSplitTest_AutoscaleAwareSDK) { - // ReadingAfterSplitTest(SdkVersion::Topic, true, false); - // } + Y_UNIT_TEST(ReadingAfterSplitTest_AutoscaleAwareSDK) { + ReadingAfterSplitTest(SdkVersion::Topic, true, false); + } - // Y_UNIT_TEST(ReadingAfterSplitTest_AutoscaleAwareSDK_AutoCommit) { - // ReadingAfterSplitTest(SdkVersion::Topic, true, false); - // } + Y_UNIT_TEST(ReadingAfterSplitTest_AutoscaleAwareSDK_AutoCommit) { + ReadingAfterSplitTest(SdkVersion::Topic, true, false); + } - // Y_UNIT_TEST(ReadingAfterSplitTest_PQv1) { - // ReadingAfterSplitTest(SdkVersion::PQv1, false, true); - // } + Y_UNIT_TEST(ReadingAfterSplitTest_PQv1) { + ReadingAfterSplitTest(SdkVersion::PQv1, false, true); + } - // void ReadingAfterSplitTest_PreferedPartition(SdkVersion sdk, bool autoscaleAwareSDK) { - // TTopicSdkTestSetup setup = CreateSetup(); - // setup.CreateTopicWithAutoscale(); + void ReadingAfterSplitTest_PreferedPartition(SdkVersion sdk, bool autoscaleAwareSDK) { + TTopicSdkTestSetup setup = CreateSetup(); + setup.CreateTopicWithAutoscale(); - // TTopicClient client = setup.MakeClient(); + TTopicClient client = setup.MakeClient(); - // auto writeSession = CreateWriteSession(client, "producer-1"); + auto writeSession = CreateWriteSession(client, "producer-1"); - // UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); + UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); - // ui64 txId = 1006; - // SplitPartition(setup, ++txId, 0, "a"); + ui64 txId = 1006; + SplitPartition(setup, ++txId, 0, "a"); - // UNIT_ASSERT(writeSession->Write(Msg("message_2.1", 3))); + UNIT_ASSERT(writeSession->Write(Msg("message_2.1", 3))); - // SplitPartition(setup, ++txId, 2, "d"); + SplitPartition(setup, ++txId, 2, "d"); - // UNIT_ASSERT(writeSession->Write(Msg("message_3.1", 5))); + UNIT_ASSERT(writeSession->Write(Msg("message_3.1", 5))); - // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 1, .AutoCommit = !autoscaleAwareSDK, .Partitions= {2}, .AutoPartitioningSupport = autoscaleAwareSDK }); - // readSession->Run(); - // readSession->WaitAllMessages(); + auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 1, .AutoCommit = !autoscaleAwareSDK, .Partitions= {2}, .AutoPartitioningSupport = autoscaleAwareSDK }); + readSession->Run(); + readSession->WaitAllMessages(); - // Sleep(TDuration::Seconds(1)); + Sleep(TDuration::Seconds(1)); - // for(const auto& info : readSession->GetReceivedMessages()) { - // if (info.Data == "message_2.1") { - // UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); - // UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); - // } else { - // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - // } - // } + for(const auto& info : readSession->GetReceivedMessages()) { + if (info.Data == "message_2.1") { + UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); + UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); + } else { + UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + } + } - // writeSession->Close(TDuration::Seconds(1)); - // readSession->Close(); - // } + writeSession->Close(TDuration::Seconds(1)); + readSession->Close(); + } - // Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_BeforeAutoscaleAwareSDK) { - // ReadingAfterSplitTest_PreferedPartition(SdkVersion::Topic, false); - // } + Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_BeforeAutoscaleAwareSDK) { + ReadingAfterSplitTest_PreferedPartition(SdkVersion::Topic, false); + } - // Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_AutoscaleAwareSDK) { - // ReadingAfterSplitTest_PreferedPartition(SdkVersion::Topic, true); - // } + Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_AutoscaleAwareSDK) { + ReadingAfterSplitTest_PreferedPartition(SdkVersion::Topic, true); + } - // Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_PQv1) { - // ReadingAfterSplitTest_PreferedPartition(SdkVersion::PQv1, false); - // } + Y_UNIT_TEST(ReadingAfterSplitTest_PreferedPartition_PQv1) { + ReadingAfterSplitTest_PreferedPartition(SdkVersion::PQv1, false); + } void PartitionSplit_oldSDK(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - // TTopicClient client = setup.MakeClient(); + TTopicClient client = setup.MakeClient(); - // auto writeSession = CreateWriteSession(client, "producer-1"); + auto writeSession = CreateWriteSession(client, "producer-1"); - // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 2, .AutoCommit = false, .AutoPartitioningSupport = false }); - // readSession->Run(); + auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 2, .AutoCommit = false, .AutoPartitioningSupport = false }); + readSession->Run(); - // UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); + UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); - // ui64 txId = 1006; - // SplitPartition(setup, ++txId, 0, "a"); + ui64 txId = 1006; + SplitPartition(setup, ++txId, 0, "a"); - // UNIT_ASSERT(writeSession->Write(Msg("message_1.2", 3))); + UNIT_ASSERT(writeSession->Write(Msg("message_1.2", 3))); - // Sleep(TDuration::Seconds(1)); // Wait read session events + Sleep(TDuration::Seconds(1)); // Wait read session events - // readSession->WaitAndAssertPartitions({0}, "We are reading only one partition because offset is not commited"); - // readSession->Run(); - // readSession->SetAutoCommit(true); - // readSession->Commit(); - // readSession->WaitAndAssertPartitions({0, 1, 2}, "We are reading all partitions because offset is commited"); - // readSession->Run(); + readSession->WaitAndAssertPartitions({0}, "We are reading only one partition because offset is not commited"); + readSession->Run(); + readSession->SetAutoCommit(true); + readSession->Commit(); + readSession->WaitAndAssertPartitions({0, 1, 2}, "We are reading all partitions because offset is commited"); + readSession->Run(); - // readSession->WaitAllMessages(); + readSession->WaitAllMessages(); - // for(const auto& info : readSession->GetReceivedMessages()) { - // if (info.Data == "message_1.1") { - // UNIT_ASSERT_EQUAL(0, info.PartitionId); - // UNIT_ASSERT_EQUAL(2, info.SeqNo); - // } else if (info.Data == "message_1.2") { - // UNIT_ASSERT(1 == info.PartitionId || 2 == info.PartitionId); - // UNIT_ASSERT_EQUAL(3, info.SeqNo); - // } else { - // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - // } - // } + for(const auto& info : readSession->GetReceivedMessages()) { + if (info.Data == "message_1.1") { + UNIT_ASSERT_EQUAL(0, info.PartitionId); + UNIT_ASSERT_EQUAL(2, info.SeqNo); + } else if (info.Data == "message_1.2") { + UNIT_ASSERT(1 == info.PartitionId || 2 == info.PartitionId); + UNIT_ASSERT_EQUAL(3, info.SeqNo); + } else { + UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + } + } - // writeSession->Close(TDuration::Seconds(1)); - // readSession->Close(); - // } + writeSession->Close(TDuration::Seconds(1)); + readSession->Close(); + } - // Y_UNIT_TEST(PartitionSplit_BeforeAutoscaleAwareSDK) { - // PartitionSplit_oldSDK(SdkVersion::Topic); - // } + Y_UNIT_TEST(PartitionSplit_BeforeAutoscaleAwareSDK) { + PartitionSplit_oldSDK(SdkVersion::Topic); + } - // Y_UNIT_TEST(PartitionSplit_PQv1) { - // PartitionSplit_oldSDK(SdkVersion::PQv1); - // } + Y_UNIT_TEST(PartitionSplit_PQv1) { + PartitionSplit_oldSDK(SdkVersion::PQv1); + } Y_UNIT_TEST(PartitionSplit_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); @@ -261,28 +261,28 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { } } - // UNIT_ASSERT_VALUES_EQUAL_C(1, readSession->GetEndedPartitionEvents().size(), "Only one partition was ended"); - // auto ev = readSession->GetEndedPartitionEvents().front(); - // UNIT_ASSERT_VALUES_EQUAL_C(std::vector{}, ev.AdjacentPartitionIds, "There isn`t adjacent partitions after split"); - // std::vector children = {1, 2}; - // UNIT_ASSERT_VALUES_EQUAL_C(children, ev.ChildPartitionIds, ""); + UNIT_ASSERT_VALUES_EQUAL_C(1, readSession->GetEndedPartitionEvents().size(), "Only one partition was ended"); + auto ev = readSession->GetEndedPartitionEvents().front(); + UNIT_ASSERT_VALUES_EQUAL_C(std::vector{}, ev.AdjacentPartitionIds, "There isn`t adjacent partitions after split"); + std::vector children = {1, 2}; + UNIT_ASSERT_VALUES_EQUAL_C(children, ev.ChildPartitionIds, ""); - // writeSession->Close(TDuration::Seconds(1)); - // readSession->Close(); - // } + writeSession->Close(TDuration::Seconds(1)); + readSession->Close(); + } void PartitionSplit_PreferedPartition(SdkVersion sdk, bool autoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - // TTopicClient client = setup.MakeClient(); + TTopicClient client = setup.MakeClient(); - // auto writeSession1 = CreateWriteSession(client, "producer-1"); - // auto writeSession2 = CreateWriteSession(client, "producer-2"); - // auto writeSession3 = CreateWriteSession(client, "producer-3", 0); + auto writeSession1 = CreateWriteSession(client, "producer-1"); + auto writeSession2 = CreateWriteSession(client, "producer-2"); + auto writeSession3 = CreateWriteSession(client, "producer-3", 0); - // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 6, .AutoCommit = !autoscaleAwareSDK, .AutoPartitioningSupport = autoscaleAwareSDK }); - // readSession->Run(); + auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .ExpectedMessagesCount = 6, .AutoCommit = !autoscaleAwareSDK, .AutoPartitioningSupport = autoscaleAwareSDK }); + readSession->Run(); UNIT_ASSERT(writeSession1->Write(Msg("message_1.1", 2))); UNIT_ASSERT(writeSession2->Write(Msg("message_2.1", 3))); @@ -367,507 +367,507 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(writeSession1->Write(Msg("message_1.1", 2))); UNIT_ASSERT(writeSession2->Write(Msg("message_2.1", 3))); - // ui64 txId = 1012; - // MergePartition(setup, ++txId, 0, 1); - - // UNIT_ASSERT(writeSession1->Write(Msg("message_1.2", 5))); // Will be fail because partition is not writable after merge - // UNIT_ASSERT(writeSession2->Write(Msg("message_2.2", 7))); // Will be fail because partition is not writable after merge - - // auto writeSession3 = CreateWriteSession(client, "producer-2", 2); - - // UNIT_ASSERT(writeSession3->Write(Msg("message_3.1", 2))); // Will be ignored because duplicated SeqNo - // UNIT_ASSERT(writeSession3->Write(Msg("message_3.2", 11))); - - // readSession->WaitAllMessages(); - - // for(const auto& info : readSession->GetReceivedMessages()) { - // if (info.Data == TString("message_1.1")) { - // UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); - // UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); - // } else if (info.Data == TString("message_2.1")) { - // UNIT_ASSERT_VALUES_EQUAL(1, info.PartitionId); - // UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); - // } else if (info.Data == TString("message_3.2")) { - // UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); - // UNIT_ASSERT_VALUES_EQUAL(11, info.SeqNo); - // } else { - // UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); - // } - // } - - // if (autoscaleAwareSDK) { - // UNIT_ASSERT_VALUES_EQUAL_C(2, readSession->GetEndedPartitionEvents().size(), "Two partition was ended which was merged"); - // for (auto ev : readSession->GetEndedPartitionEvents()) { - // UNIT_ASSERT(ev.AdjacentPartitionIds == std::vector{0} || ev.AdjacentPartitionIds == std::vector{1}); - // UNIT_ASSERT_VALUES_EQUAL_C(std::vector{2}, ev.ChildPartitionIds, ""); - // } - // } else { - // UNIT_ASSERT_VALUES_EQUAL_C(0, readSession->GetEndedPartitionEvents().size(), "OLD SDK"); - // } - - - // writeSession1->Close(TDuration::Seconds(2)); - // writeSession2->Close(TDuration::Seconds(2)); - // writeSession3->Close(TDuration::Seconds(2)); - // readSession->Close(); - // } - - // Y_UNIT_TEST(PartitionMerge_PreferedPartition_BeforeAutoscaleAwareSDK) { - // PartitionMerge_PreferedPartition(SdkVersion::Topic, false); - // } - - // Y_UNIT_TEST(PartitionMerge_PreferedPartition_AutoscaleAwareSDK) { - // PartitionMerge_PreferedPartition(SdkVersion::Topic, true); - // } - - // Y_UNIT_TEST(PartitionMerge_PreferedPartition_PQv1) { - // PartitionMerge_PreferedPartition(SdkVersion::Topic, false); - // } + ui64 txId = 1012; + MergePartition(setup, ++txId, 0, 1); + + UNIT_ASSERT(writeSession1->Write(Msg("message_1.2", 5))); // Will be fail because partition is not writable after merge + UNIT_ASSERT(writeSession2->Write(Msg("message_2.2", 7))); // Will be fail because partition is not writable after merge + + auto writeSession3 = CreateWriteSession(client, "producer-2", 2); + + UNIT_ASSERT(writeSession3->Write(Msg("message_3.1", 2))); // Will be ignored because duplicated SeqNo + UNIT_ASSERT(writeSession3->Write(Msg("message_3.2", 11))); + + readSession->WaitAllMessages(); + + for(const auto& info : readSession->GetReceivedMessages()) { + if (info.Data == TString("message_1.1")) { + UNIT_ASSERT_VALUES_EQUAL(0, info.PartitionId); + UNIT_ASSERT_VALUES_EQUAL(2, info.SeqNo); + } else if (info.Data == TString("message_2.1")) { + UNIT_ASSERT_VALUES_EQUAL(1, info.PartitionId); + UNIT_ASSERT_VALUES_EQUAL(3, info.SeqNo); + } else if (info.Data == TString("message_3.2")) { + UNIT_ASSERT_VALUES_EQUAL(2, info.PartitionId); + UNIT_ASSERT_VALUES_EQUAL(11, info.SeqNo); + } else { + UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + } + } + + if (autoscaleAwareSDK) { + UNIT_ASSERT_VALUES_EQUAL_C(2, readSession->GetEndedPartitionEvents().size(), "Two partition was ended which was merged"); + for (auto ev : readSession->GetEndedPartitionEvents()) { + UNIT_ASSERT(ev.AdjacentPartitionIds == std::vector{0} || ev.AdjacentPartitionIds == std::vector{1}); + UNIT_ASSERT_VALUES_EQUAL_C(std::vector{2}, ev.ChildPartitionIds, ""); + } + } else { + UNIT_ASSERT_VALUES_EQUAL_C(0, readSession->GetEndedPartitionEvents().size(), "OLD SDK"); + } + + + writeSession1->Close(TDuration::Seconds(2)); + writeSession2->Close(TDuration::Seconds(2)); + writeSession3->Close(TDuration::Seconds(2)); + readSession->Close(); + } + + Y_UNIT_TEST(PartitionMerge_PreferedPartition_BeforeAutoscaleAwareSDK) { + PartitionMerge_PreferedPartition(SdkVersion::Topic, false); + } + + Y_UNIT_TEST(PartitionMerge_PreferedPartition_AutoscaleAwareSDK) { + PartitionMerge_PreferedPartition(SdkVersion::Topic, true); + } + + Y_UNIT_TEST(PartitionMerge_PreferedPartition_PQv1) { + PartitionMerge_PreferedPartition(SdkVersion::Topic, false); + } void PartitionSplit_ReadEmptyPartitions(SdkVersion sdk, bool autoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - // TTopicClient client = setup.MakeClient(); - // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoPartitioningSupport = autoscaleAwareSDK }); + TTopicClient client = setup.MakeClient(); + auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoPartitioningSupport = autoscaleAwareSDK }); - // readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); + readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); - // ui64 txId = 1023; - // SplitPartition(setup, ++txId, 0, "a"); + ui64 txId = 1023; + SplitPartition(setup, ++txId, 0, "a"); - // readSession->WaitAndAssertPartitions({0, 1, 2}, "After split must read all partitions because parent partition is empty"); + readSession->WaitAndAssertPartitions({0, 1, 2}, "After split must read all partitions because parent partition is empty"); - // readSession->Close(); - // } + readSession->Close(); + } - // Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_BeforeAutoscaleAwareSDK) { - // PartitionSplit_ReadEmptyPartitions(SdkVersion::Topic, false); - // } + Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_BeforeAutoscaleAwareSDK) { + PartitionSplit_ReadEmptyPartitions(SdkVersion::Topic, false); + } - // Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_AutoscaleAwareSDK) { - // PartitionSplit_ReadEmptyPartitions(SdkVersion::Topic, true); - // } + Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_AutoscaleAwareSDK) { + PartitionSplit_ReadEmptyPartitions(SdkVersion::Topic, true); + } - // Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_PQv1) { - // PartitionSplit_ReadEmptyPartitions(SdkVersion::PQv1, false); - // } + Y_UNIT_TEST(PartitionSplit_ReadEmptyPartitions_PQv1) { + PartitionSplit_ReadEmptyPartitions(SdkVersion::PQv1, false); + } void PartitionSplit_ReadNotEmptyPartitions(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - // TTopicClient client = setup.MakeClient(); - // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .AutoPartitioningSupport = false }); + TTopicClient client = setup.MakeClient(); + auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .AutoPartitioningSupport = false }); - // auto writeSession = CreateWriteSession(client, "producer-1", 0); + auto writeSession = CreateWriteSession(client, "producer-1", 0); - // readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); + readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); - // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - // ui64 txId = 1023; - // SplitPartition(setup, ++txId, 0, "a"); + ui64 txId = 1023; + SplitPartition(setup, ++txId, 0, "a"); - // readSession->WaitAndAssertPartitions({0}, "After split must read only 0 partition because had been read not from the end of partition"); - // readSession->WaitAndAssertPartitions({}, "Partition must be released for secondary read after 1 second"); - // readSession->WaitAndAssertPartitions({0}, "Must secondary read for check read from end"); - // readSession->WaitAndAssertPartitions({}, "Partition must be released for secondary read because start not from the end of partition after 2 seconds"); + readSession->WaitAndAssertPartitions({0}, "After split must read only 0 partition because had been read not from the end of partition"); + readSession->WaitAndAssertPartitions({}, "Partition must be released for secondary read after 1 second"); + readSession->WaitAndAssertPartitions({0}, "Must secondary read for check read from end"); + readSession->WaitAndAssertPartitions({}, "Partition must be released for secondary read because start not from the end of partition after 2 seconds"); - // readSession->SetOffset(0, 1); - // readSession->WaitAndAssertPartitions({0, 1, 2}, "Must read from all partitions because had been read from the end of partition"); + readSession->SetOffset(0, 1); + readSession->WaitAndAssertPartitions({0, 1, 2}, "Must read from all partitions because had been read from the end of partition"); - // readSession->Close(); - // } + readSession->Close(); + } - // Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_BeforeAutoscaleAwareSDK) { - // PartitionSplit_ReadNotEmptyPartitions(SdkVersion::Topic); - // } + Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_BeforeAutoscaleAwareSDK) { + PartitionSplit_ReadNotEmptyPartitions(SdkVersion::Topic); + } - // Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_PQv1) { - // PartitionSplit_ReadNotEmptyPartitions(SdkVersion::PQv1); - // } + Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_PQv1) { + PartitionSplit_ReadNotEmptyPartitions(SdkVersion::PQv1); + } Y_UNIT_TEST(PartitionSplit_ReadNotEmptyPartitions_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - // TTopicClient client = setup.MakeClient(); - // auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); + TTopicClient client = setup.MakeClient(); + auto readSession = CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); - // auto writeSession = CreateWriteSession(client, "producer-1", 0); + auto writeSession = CreateWriteSession(client, "producer-1", 0); - // readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); + readSession->WaitAndAssertPartitions({0}, "Must read all exists partitions"); - // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - // ui64 txId = 1023; - // SplitPartition(setup, ++txId, 0, "a"); + ui64 txId = 1023; + SplitPartition(setup, ++txId, 0, "a"); - // readSession->WaitAndAssertPartitions({0, 1, 2}, "Must read from all partitions because used new SDK"); + readSession->WaitAndAssertPartitions({0, 1, 2}, "Must read from all partitions because used new SDK"); - // readSession->Close(); - // } + readSession->Close(); + } void PartitionSplit_ManySession(SdkVersion sdk) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - // TTopicClient client = setup.MakeClient(); + TTopicClient client = setup.MakeClient(); - // auto writeSession = CreateWriteSession(client, "producer-1", 0); - // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + auto writeSession = CreateWriteSession(client, "producer-1", 0); + UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - // ui64 txId = 1023; - // SplitPartition(setup, ++txId, 0, "a"); + ui64 txId = 1023; + SplitPartition(setup, ++txId, 0, "a"); - // auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .Partitions = {0, 1, 2}, .AutoPartitioningSupport = false }); - // readSession1->SetOffset(0, 1); - // readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because read the partition 0 from offset 1"); - // readSession1->SetOffset(0, 0); + auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .Partitions = {0, 1, 2}, .AutoPartitioningSupport = false }); + readSession1->SetOffset(0, 1); + readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because read the partition 0 from offset 1"); + readSession1->SetOffset(0, 0); - // auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .Partitions = {0}, .AutoPartitioningSupport = false }); - // readSession2->SetOffset(0, 0); + auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = sdk, .AutoCommit = false, .Partitions = {0}, .AutoPartitioningSupport = false }); + readSession2->SetOffset(0, 0); - // readSession2->WaitAndAssertPartitions({0}, "Must read partition 0 because it defined in the readSession"); - // readSession2->Run(); + readSession2->WaitAndAssertPartitions({0}, "Must read partition 0 because it defined in the readSession"); + readSession2->Run(); - // readSession1->WaitAndAssertPartitions({}, "Must release all partitions becase readSession2 read not from EndOffset"); - // readSession1->Run(); + readSession1->WaitAndAssertPartitions({}, "Must release all partitions becase readSession2 read not from EndOffset"); + readSession1->Run(); - // readSession1->WaitAndAssertPartitions({0}, "Partition 0 must rebalance to other sessions (Session-0)"); + readSession1->WaitAndAssertPartitions({0}, "Partition 0 must rebalance to other sessions (Session-0)"); - // readSession1->Close(); - // readSession2->Close(); - // } + readSession1->Close(); + readSession2->Close(); + } - // Y_UNIT_TEST(PartitionSplit_ManySession_BeforeAutoscaleAwareSDK) { - // PartitionSplit_ManySession(SdkVersion::Topic); - // } + Y_UNIT_TEST(PartitionSplit_ManySession_BeforeAutoscaleAwareSDK) { + PartitionSplit_ManySession(SdkVersion::Topic); + } - // Y_UNIT_TEST(PartitionSplit_ManySession_PQv1) { - // PartitionSplit_ManySession(SdkVersion::PQv1); - // } + Y_UNIT_TEST(PartitionSplit_ManySession_PQv1) { + PartitionSplit_ManySession(SdkVersion::PQv1); + } Y_UNIT_TEST(PartitionSplit_ManySession_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - // TTopicClient client = setup.MakeClient(); + TTopicClient client = setup.MakeClient(); - // auto writeSession = CreateWriteSession(client, "producer-1", 0); - // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + auto writeSession = CreateWriteSession(client, "producer-1", 0); + UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - // ui64 txId = 1023; - // SplitPartition(setup, ++txId, 0, "a"); + ui64 txId = 1023; + SplitPartition(setup, ++txId, 0, "a"); - // Sleep(TDuration::Seconds(1)); + Sleep(TDuration::Seconds(1)); - // auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); + auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); - // readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because used new SDK"); - // readSession1->Commit(); - // readSession1->Run(); + readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because used new SDK"); + readSession1->Commit(); + readSession1->Run(); - // auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); - // readSession2->Run(); + auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); + readSession2->Run(); - // Sleep(TDuration::Seconds(1)); + Sleep(TDuration::Seconds(1)); - // auto p1 = readSession1->GetPartitions(); - // auto p2 = readSession2->GetPartitions(); + auto p1 = readSession1->GetPartitions(); + auto p2 = readSession2->GetPartitions(); - // std::set partitions; - // partitions.insert(p1.begin(), p1.end()); - // partitions.insert(p2.begin(), p2.end()); + std::set partitions; + partitions.insert(p1.begin(), p1.end()); + partitions.insert(p2.begin(), p2.end()); - // std::set expected{0, 1, 2}; + std::set expected{0, 1, 2}; - // UNIT_ASSERT_VALUES_EQUAL(expected, partitions); + UNIT_ASSERT_VALUES_EQUAL(expected, partitions); - // readSession1->Close(); + readSession1->Close(); - // Sleep(TDuration::Seconds(1)); + Sleep(TDuration::Seconds(1)); - // UNIT_ASSERT_VALUES_EQUAL(expected, readSession2->GetPartitions()); + UNIT_ASSERT_VALUES_EQUAL(expected, readSession2->GetPartitions()); - // readSession2->Close(); - // } + readSession2->Close(); + } Y_UNIT_TEST(PartitionSplit_ManySession_existed_AutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - // TTopicClient client = setup.MakeClient(); + TTopicClient client = setup.MakeClient(); - // auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); - // auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .Partitions = {0}, .AutoPartitioningSupport = true }); + auto readSession1 = CreateTestReadSession({ .Name="Session-1", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .AutoPartitioningSupport = true }); + auto readSession2 = CreateTestReadSession({ .Name="Session-2", .Setup=setup, .Sdk = SdkVersion::Topic, .AutoCommit = false, .Partitions = {0}, .AutoPartitioningSupport = true }); - // auto writeSession = CreateWriteSession(client, "producer-1", 0); - // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + auto writeSession = CreateWriteSession(client, "producer-1", 0); + UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - // ui64 txId = 1023; - // SplitPartition(setup, ++txId, 0, "a"); + ui64 txId = 1023; + SplitPartition(setup, ++txId, 0, "a"); - // readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because used new SDK"); - // readSession1->Commit(); - // readSession2->Run(); + readSession1->WaitAndAssertPartitions({0, 1, 2}, "Must read all exists partitions because used new SDK"); + readSession1->Commit(); + readSession2->Run(); - // readSession2->WaitAndAssertPartitions({0}, "Must read partition 0 because it defined in the readSession"); - // readSession2->Run(); + readSession2->WaitAndAssertPartitions({0}, "Must read partition 0 because it defined in the readSession"); + readSession2->Run(); - // readSession1->WaitAndAssertPartitions({1, 2}, "Partition 0 must rebalance to other sessions (Session-0)"); + readSession1->WaitAndAssertPartitions({1, 2}, "Partition 0 must rebalance to other sessions (Session-0)"); - // readSession1->Close(); - // readSession2->Close(); - // } + readSession1->Close(); + readSession2->Close(); + } Y_UNIT_TEST(CommitTopPast_BeforeAutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); - // TTopicClient client = setup.MakeClient(); - - // auto writeSession = CreateWriteSession(client, "producer-1", 0); - // UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); - // UNIT_ASSERT(writeSession->Write(Msg("message_2", 3))); - - // ui64 txId = 1023; - // SplitPartition(setup, ++txId, 0, "a"); - - // auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); - // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer has just started reading the inactive partition and he can commit"); - - // status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 1).GetValueSync(); - // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "A consumer who has not read to the end can commit messages forward."); - - // status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); - // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "A consumer who has not read to the end can commit messages back."); - - // status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 2).GetValueSync(); - // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer can commit at the end of the inactive partition."); - - // status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); - // UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::BAD_REQUEST, status.GetStatus(), "The consumer cannot commit an offset for inactive, read-to-the-end partitions."); - // } - - // Y_UNIT_TEST(ControlPlane_CreateAlterDescribe) { - // auto autoscalingTestTopic = "autoscalit-topic"; - // TTopicSdkTestSetup setup = CreateSetup(); - // TTopicClient client = setup.MakeClient(); - - // auto minParts = 5; - // auto maxParts = 10; - // auto scaleUpPercent = 80; - // auto scaleDownPercent = 20; - // auto threshold = 500; - // auto strategy = EAutoPartitioningStrategy::ScaleUp; - - // TCreateTopicSettings createSettings; - // createSettings - // .BeginConfigurePartitioningSettings() - // .MinActivePartitions(minParts) - // .MaxActivePartitions(maxParts) - // .BeginConfigureAutoPartitioningSettings() - // .UpUtilizationPercent(scaleUpPercent) - // .DownUtilizationPercent(scaleDownPercent) - // .StabilizationWindow(TDuration::Seconds(threshold)) - // .Strategy(strategy) - // .EndConfigureAutoPartitioningSettings() - // .EndConfigurePartitioningSettings(); - // client.CreateTopic(autoscalingTestTopic, createSettings).Wait(); - - // TDescribeTopicSettings descSettings; - - // auto describe = client.DescribeTopic(autoscalingTestTopic, descSettings).GetValueSync(); - // UNIT_ASSERT_VALUES_EQUAL_C(describe.GetStatus(), NYdb::EStatus::SUCCESS, describe.GetIssues().ToString()); - - - // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), minParts); - // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), maxParts); - // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), strategy); - // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), scaleDownPercent); - // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), scaleUpPercent); - // UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), threshold); - - // auto alterMinParts = 10; - // auto alterMaxParts = 20; - // auto alterScaleUpPercent = 90; - // auto alterScaleDownPercent = 10; - // auto alterThreshold = 700; - // auto alterStrategy = EAutoPartitioningStrategy::ScaleUpAndDown; - - // TAlterTopicSettings alterSettings; - // alterSettings - // .BeginAlterPartitioningSettings() - // .MinActivePartitions(alterMinParts) - // .MaxActivePartitions(alterMaxParts) - // .BeginAlterAutoPartitioningSettings() - // .DownUtilizationPercent(alterScaleDownPercent) - // .UpUtilizationPercent(alterScaleUpPercent) - // .StabilizationWindow(TDuration::Seconds(alterThreshold)) - // .Strategy(alterStrategy) - // .EndAlterAutoPartitioningSettings() - // .EndAlterTopicPartitioningSettings(); - - // client.AlterTopic(autoscalingTestTopic, alterSettings).Wait(); - - // auto describeAfterAlter = client.DescribeTopic(autoscalingTestTopic).GetValueSync(); - - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), alterMinParts); - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), alterMaxParts); - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), alterStrategy); - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), alterScaleUpPercent); - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), alterScaleDownPercent); - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), alterThreshold); - // } - - // Y_UNIT_TEST(ControlPlane_DisableAutoPartitioning) { - // auto topicName = "autoscalit-topic"; - - // TTopicSdkTestSetup setup = CreateSetup(); - // TTopicClient client = setup.MakeClient(); - - // { - // TCreateTopicSettings createSettings; - // createSettings - // .BeginConfigurePartitioningSettings() - // .MinActivePartitions(1) - // .MaxActivePartitions(100) - // .BeginConfigureAutoPartitioningSettings() - // .Strategy(EAutoPartitioningStrategy::ScaleUp) - // .EndConfigureAutoPartitioningSettings() - // .EndConfigurePartitioningSettings(); - // client.CreateTopic(topicName, createSettings).Wait(); - // } - - // { - // TAlterTopicSettings alterSettings; - // alterSettings - // .BeginAlterPartitioningSettings() - // .BeginAlterAutoPartitioningSettings() - // .Strategy(EAutoPartitioningStrategy::Disabled) - // .EndAlterAutoPartitioningSettings() - // .EndAlterTopicPartitioningSettings(); - // auto f = client.AlterTopic(topicName, alterSettings); - // f.Wait(); - - // auto v = f.GetValueSync(); - // UNIT_ASSERT_C(!v.IsSuccess(), "Must receve error becuse disabling is not supported"); - // } - // } - - // Y_UNIT_TEST(ControlPlane_BackCompatibility) { - // auto topicName = "back-compatibility-test"; - - // TTopicSdkTestSetup setup = CreateSetup(); - // TTopicClient client = setup.MakeClient(); - - // { - // TCreateTopicSettings createSettings; - // createSettings - // .BeginConfigurePartitioningSettings() - // .MinActivePartitions(3) - // .EndConfigurePartitioningSettings(); - // client.CreateTopic(topicName, createSettings).Wait(); - // } - - // { - // auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); - - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 3); - // } - - // { - // TAlterTopicSettings alterSettings; - // alterSettings - // .BeginAlterPartitioningSettings() - // .MinActivePartitions(5) - // .EndAlterTopicPartitioningSettings(); - // client.AlterTopic(topicName, alterSettings).Wait(); - // } - - // { - // auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); - - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 5); - // } - // } - - // Y_UNIT_TEST(ControlPlane_PauseAutoPartitioning) { - // auto topicName = "autoscalit-topic"; - - // TTopicSdkTestSetup setup = CreateSetup(); - // TTopicClient client = setup.MakeClient(); - - // { - // TCreateTopicSettings createSettings; - // createSettings - // .BeginConfigurePartitioningSettings() - // .MinActivePartitions(1) - // .MaxActivePartitions(100) - // .BeginConfigureAutoPartitioningSettings() - // .Strategy(EAutoPartitioningStrategy::ScaleUp) - // .EndConfigureAutoPartitioningSettings() - // .EndConfigurePartitioningSettings(); - // client.CreateTopic(topicName, createSettings).Wait(); - // } - - // { - // TAlterTopicSettings alterSettings; - // alterSettings - // .BeginAlterPartitioningSettings() - // .MinActivePartitions(3) - // .MaxActivePartitions(107) - // .BeginAlterAutoPartitioningSettings() - // .Strategy(EAutoPartitioningStrategy::Paused) - // .EndAlterAutoPartitioningSettings() - // .EndAlterTopicPartitioningSettings(); - // auto f = client.AlterTopic(topicName, alterSettings); - // f.Wait(); - - // auto v = f.GetValueSync(); - // UNIT_ASSERT_C(v.IsSuccess(), "Error: " << v); - // } - - // { - // auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); - - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 3); - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), 107); - // UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), EAutoPartitioningStrategy::Paused); - // } - // } - - // Y_UNIT_TEST(ControlPlane_AutoscalingWithStorageSizeRetention) { - // auto autoscalingTestTopic = "autoscalit-topic"; - // TTopicSdkTestSetup setup = CreateSetup(); - // TTopicClient client = setup.MakeClient(); - - // TCreateTopicSettings createSettings; - // createSettings - // .RetentionStorageMb(1024) - // .BeginConfigurePartitioningSettings() - // .BeginConfigureAutoPartitioningSettings() - // .Strategy(EAutoPartitioningStrategy::ScaleUp) - // .EndConfigureAutoPartitioningSettings() - // .EndConfigurePartitioningSettings(); - // auto result = client.CreateTopic(autoscalingTestTopic, createSettings).GetValueSync(); - - // UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::BAD_REQUEST); - - // createSettings.RetentionStorageMb(0); - // result = client.CreateTopic(autoscalingTestTopic, createSettings).GetValueSync(); - // UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); - - // TAlterTopicSettings alterSettings; - // alterSettings - // .SetRetentionStorageMb(1024); - - // result = client.AlterTopic(autoscalingTestTopic, alterSettings).GetValueSync(); - - // UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::BAD_REQUEST); - // } + TTopicClient client = setup.MakeClient(); + + auto writeSession = CreateWriteSession(client, "producer-1", 0); + UNIT_ASSERT(writeSession->Write(Msg("message_1", 2))); + UNIT_ASSERT(writeSession->Write(Msg("message_2", 3))); + + ui64 txId = 1023; + SplitPartition(setup, ++txId, 0, "a"); + + auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer has just started reading the inactive partition and he can commit"); + + status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 1).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "A consumer who has not read to the end can commit messages forward."); + + status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "A consumer who has not read to the end can commit messages back."); + + status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 2).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer can commit at the end of the inactive partition."); + + status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::BAD_REQUEST, status.GetStatus(), "The consumer cannot commit an offset for inactive, read-to-the-end partitions."); + } + + Y_UNIT_TEST(ControlPlane_CreateAlterDescribe) { + auto autoscalingTestTopic = "autoscalit-topic"; + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + auto minParts = 5; + auto maxParts = 10; + auto scaleUpPercent = 80; + auto scaleDownPercent = 20; + auto threshold = 500; + auto strategy = EAutoPartitioningStrategy::ScaleUp; + + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(minParts) + .MaxActivePartitions(maxParts) + .BeginConfigureAutoPartitioningSettings() + .UpUtilizationPercent(scaleUpPercent) + .DownUtilizationPercent(scaleDownPercent) + .StabilizationWindow(TDuration::Seconds(threshold)) + .Strategy(strategy) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings(); + client.CreateTopic(autoscalingTestTopic, createSettings).Wait(); + + TDescribeTopicSettings descSettings; + + auto describe = client.DescribeTopic(autoscalingTestTopic, descSettings).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(describe.GetStatus(), NYdb::EStatus::SUCCESS, describe.GetIssues().ToString()); + + + UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), minParts); + UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), maxParts); + UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), strategy); + UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), scaleDownPercent); + UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), scaleUpPercent); + UNIT_ASSERT_VALUES_EQUAL(describe.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), threshold); + + auto alterMinParts = 10; + auto alterMaxParts = 20; + auto alterScaleUpPercent = 90; + auto alterScaleDownPercent = 10; + auto alterThreshold = 700; + auto alterStrategy = EAutoPartitioningStrategy::ScaleUpAndDown; + + TAlterTopicSettings alterSettings; + alterSettings + .BeginAlterPartitioningSettings() + .MinActivePartitions(alterMinParts) + .MaxActivePartitions(alterMaxParts) + .BeginAlterAutoPartitioningSettings() + .DownUtilizationPercent(alterScaleDownPercent) + .UpUtilizationPercent(alterScaleUpPercent) + .StabilizationWindow(TDuration::Seconds(alterThreshold)) + .Strategy(alterStrategy) + .EndAlterAutoPartitioningSettings() + .EndAlterTopicPartitioningSettings(); + + client.AlterTopic(autoscalingTestTopic, alterSettings).Wait(); + + auto describeAfterAlter = client.DescribeTopic(autoscalingTestTopic).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), alterMinParts); + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), alterMaxParts); + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), alterStrategy); + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetUpUtilizationPercent(), alterScaleUpPercent); + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetDownUtilizationPercent(), alterScaleDownPercent); + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStabilizationWindow().Seconds(), alterThreshold); + } + + Y_UNIT_TEST(ControlPlane_DisableAutoPartitioning) { + auto topicName = "autoscalit-topic"; + + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + { + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(1) + .MaxActivePartitions(100) + .BeginConfigureAutoPartitioningSettings() + .Strategy(EAutoPartitioningStrategy::ScaleUp) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings(); + client.CreateTopic(topicName, createSettings).Wait(); + } + + { + TAlterTopicSettings alterSettings; + alterSettings + .BeginAlterPartitioningSettings() + .BeginAlterAutoPartitioningSettings() + .Strategy(EAutoPartitioningStrategy::Disabled) + .EndAlterAutoPartitioningSettings() + .EndAlterTopicPartitioningSettings(); + auto f = client.AlterTopic(topicName, alterSettings); + f.Wait(); + + auto v = f.GetValueSync(); + UNIT_ASSERT_C(!v.IsSuccess(), "Must receve error becuse disabling is not supported"); + } + } + + Y_UNIT_TEST(ControlPlane_BackCompatibility) { + auto topicName = "back-compatibility-test"; + + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + { + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(3) + .EndConfigurePartitioningSettings(); + client.CreateTopic(topicName, createSettings).Wait(); + } + + { + auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 3); + } + + { + TAlterTopicSettings alterSettings; + alterSettings + .BeginAlterPartitioningSettings() + .MinActivePartitions(5) + .EndAlterTopicPartitioningSettings(); + client.AlterTopic(topicName, alterSettings).Wait(); + } + + { + auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 5); + } + } + + Y_UNIT_TEST(ControlPlane_PauseAutoPartitioning) { + auto topicName = "autoscalit-topic"; + + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + { + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(1) + .MaxActivePartitions(100) + .BeginConfigureAutoPartitioningSettings() + .Strategy(EAutoPartitioningStrategy::ScaleUp) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings(); + client.CreateTopic(topicName, createSettings).Wait(); + } + + { + TAlterTopicSettings alterSettings; + alterSettings + .BeginAlterPartitioningSettings() + .MinActivePartitions(3) + .MaxActivePartitions(107) + .BeginAlterAutoPartitioningSettings() + .Strategy(EAutoPartitioningStrategy::Paused) + .EndAlterAutoPartitioningSettings() + .EndAlterTopicPartitioningSettings(); + auto f = client.AlterTopic(topicName, alterSettings); + f.Wait(); + + auto v = f.GetValueSync(); + UNIT_ASSERT_C(v.IsSuccess(), "Error: " << v); + } + + { + auto describeAfterAlter = client.DescribeTopic(topicName).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMinActivePartitions(), 3); + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetMaxActivePartitions(), 107); + UNIT_ASSERT_VALUES_EQUAL(describeAfterAlter.GetTopicDescription().GetPartitioningSettings().GetAutoPartitioningSettings().GetStrategy(), EAutoPartitioningStrategy::Paused); + } + } + + Y_UNIT_TEST(ControlPlane_AutoscalingWithStorageSizeRetention) { + auto autoscalingTestTopic = "autoscalit-topic"; + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + TCreateTopicSettings createSettings; + createSettings + .RetentionStorageMb(1024) + .BeginConfigurePartitioningSettings() + .BeginConfigureAutoPartitioningSettings() + .Strategy(EAutoPartitioningStrategy::ScaleUp) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings(); + auto result = client.CreateTopic(autoscalingTestTopic, createSettings).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::BAD_REQUEST); + + createSettings.RetentionStorageMb(0); + result = client.CreateTopic(autoscalingTestTopic, createSettings).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS); + + TAlterTopicSettings alterSettings; + alterSettings + .SetRetentionStorageMb(1024); + + result = client.AlterTopic(autoscalingTestTopic, alterSettings).GetValueSync(); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::BAD_REQUEST); + } Y_UNIT_TEST(PartitionSplit_AutosplitByLoad) { TTopicSdkTestSetup setup = CreateSetup(); @@ -885,10 +885,6 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { .Strategy(EAutoPartitioningStrategy::ScaleUp) .EndConfigureAutoPartitioningSettings() .EndConfigurePartitioningSettings(); - - TConsumerSettings consumers(createSettings, TEST_CONSUMER); - createSettings.AppendConsumers(consumers); - client.CreateTopic(TEST_TOPIC, createSettings).Wait(); auto msg = TString(1_MB, 'a'); diff --git a/ydb/core/persqueue/ut/ut_with_sdk/ya.make b/ydb/core/persqueue/ut/ut_with_sdk/ya.make index 617ecf452d07..fdb6c2ff4054 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/ya.make +++ b/ydb/core/persqueue/ut/ut_with_sdk/ya.make @@ -30,8 +30,8 @@ YQL_LAST_ABI_VERSION() SRCS( autoscaling_ut.cpp - # balancing_ut.cpp - # mirrorer_ut.cpp + balancing_ut.cpp + mirrorer_ut.cpp ) END() From 664bce9fa4414fe8da077c68fceeee3d154ff7d5 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Fri, 15 Nov 2024 00:41:18 +0000 Subject: [PATCH 03/29] refactoring and ut --- ydb/core/kqp/topics/kqp_topics.cpp | 10 +- .../ut/ut_with_sdk/autoscaling_ut.cpp | 81 +++++++++++ ydb/services/lib/actors/type_definitions.h | 2 +- .../actors/commit_offset_actor.cpp | 43 +++--- .../persqueue_v1/actors/commit_offset_actor.h | 126 +---------------- .../actors/kqp_commit_offset_helper.cpp | 132 ++++++++++++++++++ .../actors/kqp_commit_offset_helper.h | 51 +++++++ ydb/services/persqueue_v1/actors/ya.make | 2 + 8 files changed, 297 insertions(+), 150 deletions(-) create mode 100644 ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp create mode 100644 ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h diff --git a/ydb/core/kqp/topics/kqp_topics.cpp b/ydb/core/kqp/topics/kqp_topics.cpp index 58882e717fb3..c5d01c3b13ed 100644 --- a/ydb/core/kqp/topics/kqp_topics.cpp +++ b/ydb/core/kqp/topics/kqp_topics.cpp @@ -27,14 +27,16 @@ static void UpdateSupportivePartition(TMaybe& lhs, const TMaybe& rhs // bool TConsumerOperations::IsValid() const { - return Offsets_.GetNumIntervals() == 1; + return Offsets_.GetNumIntervals() <= 1; } std::pair TConsumerOperations::GetOffsetsCommitRange() const { Y_ABORT_UNLESS(IsValid()); - return {Offsets_.Min(), Offsets_.Max()}; + if (Offsets_.Empty()) { + return {0,0}; + } else return {Offsets_.Min(), Offsets_.Max()}; } bool TConsumerOperations::GetForceCommit() const @@ -88,7 +90,9 @@ void TConsumerOperations::AddOperationImpl(const TString& consumer, Consumer_ = consumer; } - Offsets_.InsertInterval(begin, end); + if (end != 0) { + Offsets_.InsertInterval(begin, end); + } ForceCommit_ = forceCommit; KillReadSession_ = killReadSession; diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index b39ad0c0c7fb..1fc252e06257 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -622,6 +622,87 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { readSession2->Close(); } + Y_UNIT_TEST(PartitionSplit_OffsetCommit) { + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(1) + .MaxActivePartitions(100) + .BeginConfigureAutoPartitioningSettings() + .UpUtilizationPercent(2) + .DownUtilizationPercent(1) + .StabilizationWindow(TDuration::Seconds(2)) + .Strategy(EAutoPartitioningStrategy::ScaleUp) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings(); + + TConsumerSettings consumers(createSettings, TEST_CONSUMER); + createSettings.AppendConsumers(consumers); + + client.CreateTopic(TEST_TOPIC, createSettings).Wait(); + + auto msg = TString(1_MB, 'a'); + + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, TEST_TOPIC, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, TEST_TOPIC, false); + + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 1))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 2))); + Sleep(TDuration::Seconds(5)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 1); + } + + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 3))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, 4))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 5))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, 6))); + Sleep(TDuration::Seconds(5)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); + } + + auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, TEST_TOPIC, false); + auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, TEST_TOPIC, false); + + { + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); + UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 8))); + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 9))); + UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 10))); + Sleep(TDuration::Seconds(5)); + auto describe2 = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe2.GetTopicDescription().GetPartitions().size(), 5); + } + + auto status = client.CommitOffset(TEST_TOPIC, 1, TEST_CONSUMER, 2).GetValueSync(); + UNIT_ASSERT(status.IsSuccess()); + + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + UNIT_ASSERT(description.GetPartitions().size() == 5); + + auto stats0 = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats0.Defined()); + UNIT_ASSERT(stats0->GetCommittedOffset() == 6); + + auto stats1 = description.GetPartitions().at(1).GetPartitionConsumerStats(); + UNIT_ASSERT(stats1.Defined()); + UNIT_ASSERT(stats1->GetCommittedOffset() == 2); + + auto stats3 = description.GetPartitions().at(3).GetPartitionConsumerStats(); + UNIT_ASSERT(stats3.Defined()); + UNIT_ASSERT(stats3->GetCommittedOffset() == 0); + } + Y_UNIT_TEST(CommitTopPast_BeforeAutoscaleAwareSDK) { TTopicSdkTestSetup setup = CreateSetup(); setup.CreateTopicWithAutoscale(std::string{TEST_TOPIC}, std::string{TEST_CONSUMER}, 1, 100); diff --git a/ydb/services/lib/actors/type_definitions.h b/ydb/services/lib/actors/type_definitions.h index 3b11dd91b1a2..5c169e0d8948 100644 --- a/ydb/services/lib/actors/type_definitions.h +++ b/ydb/services/lib/actors/type_definitions.h @@ -62,7 +62,7 @@ struct TTopicHolder { .MeteringMode = info.MeteringMode, .FullConverter = info.TopicNameConverter, .Partitions = info.Partitions, - .PartitionGraph = info.PartitionGraph // savnik тут будет пусто, т.к. в TTopicInitInfo это поле не сетится. Найти где оно инициализируется и исправить + .PartitionGraph = info.PartitionGraph }; } }; diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index 6ba1c332a53d..b1ce0702bdab 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -108,36 +108,37 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc if (partitionNode->HierarhicalParents.size() == 0 && partitionNode->Children.size() == 0) { SendCommit(topicInitInfo, commitRequest, ctx); } else { - Kqp.Consumer = ClientId; - Kqp.DataBase = Request().GetDatabaseName().GetOrElse(TString()); // savnik if empty? - Kqp.Path = topic; + std::vector commits; for (auto& parent: partitionNode->HierarhicalParents) { TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max()}; - Commits.push_back(commit); + commits.push_back(commit); } for (auto& child: partitionNode->Children) { TKqpHelper::TCommitInfo commit {.PartitionId = child->Id, .Offset = 0}; - Commits.push_back(commit); + commits.push_back(commit); } TKqpHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->Getoffset()}; - Commits.push_back(commit); + commits.push_back(commit); + + // savnik if empty database? + Kqp = std::make_unique(Request().GetDatabaseName().GetOrElse(TString()), ClientId, topic, commits); SendDistributedTxOffsets(ctx); } } void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const NActors::TActorContext& ctx) { - if (!Kqp.Handle(ev, ctx)) { + if (!Kqp->Handle(ev, ctx)) { AnswerError("empty list of topics", PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx); // savnik } - Kqp.BeginTransaction(ctx); + Kqp->BeginTransaction(ctx); } void TCommitOffsetActor::SendDistributedTxOffsets(const TActorContext& ctx) { - Kqp.SendCreateSessionRequest(ctx); + Kqp->SendCreateSessionRequest(ctx); } void TCommitOffsetActor::SendCommit(const TTopicInitInfo& topic, const Ydb::Topic::CommitOffsetRequest* commitRequest, const TActorContext& ctx) { @@ -177,16 +178,15 @@ void TCommitOffsetActor::SendCommit(const TTopicInitInfo& topic, const Ydb::Topi void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { auto& record = ev->Get()->Record; + if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { // savnik finish this part! + Ydb::Topic::CommitOffsetResult result; + Request().SendResult(result, record.GetYdbStatus()); + Die(ctx); + } - // savnik set error record.GetYdbStatus() - if (Kqp.Step == 0) { - Kqp.Step++; - Kqp.SendCommits(ev,Commits, ctx); - } else if (Kqp.Step == 1) { - Kqp.Step++; - Kqp.CommitTx(ctx); - } else { - Kqp.CloseKqpSession(ctx); + auto step = Kqp->Handle(ev, ctx); + + if (step == TKqpHelper::ECurrentStep::DONE) { Ydb::Topic::CommitOffsetResult result; Request().SendResult(result, Ydb::StatusIds::SUCCESS); Die(ctx); @@ -213,16 +213,15 @@ void TCommitOffsetActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActo void TCommitOffsetActor::AnswerError(const TString& errorReason, const PersQueue::ErrorCode::ErrorCode errorCode, const NActors::TActorContext& ctx) { - Ydb::Topic::CommitOffsetResponse response; response.mutable_operation()->set_ready(true); auto issue = response.mutable_operation()->add_issues(); FillIssue(issue, errorCode, errorReason); - response.mutable_operation()->set_status(ConvertPersQueueInternalCodeToStatus(errorCode)); - Reply(ConvertPersQueueInternalCodeToStatus(errorCode), response.operation().issues(), ctx); + auto status = ConvertPersQueueInternalCodeToStatus(errorCode); + response.mutable_operation()->set_status(status); + Reply(status, response.operation().issues(), ctx); } - void TCommitOffsetActor::Handle(TEvPQProxy::TEvCloseSession::TPtr& ev, const TActorContext& ctx) { AnswerError(ev->Get()->Reason, ev->Get()->ErrorCode, ctx); } diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.h b/ydb/services/persqueue_v1/actors/commit_offset_actor.h index f0a945aa3bb3..f36203e967aa 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.h +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.h @@ -1,8 +1,7 @@ #pragma once #include "events.h" -#include "ydb/core/kqp/common/simple/services.h" -#include "ydb/services/metadata/service.h" +#include "kqp_commit_offset_helper.h" #include @@ -16,126 +15,6 @@ namespace NKikimr::NGRpcProxy::V1 { using namespace NKikimr::NGRpcService; -class TKqpHelper { - - public: // savnik - - TString DataBase; - TString KqpSessionId; - TString Consumer; - TString Path; - TString TxId; - - int Step = 0; - - struct TCommitInfo { - ui64 PartitionId; - i64 Offset; - }; - - void SendCreateSessionRequest(const TActorContext& ctx) { - auto ev = MakeCreateSessionRequest(); - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); - } - - THolder MakeCreateSessionRequest() { - auto ev = MakeHolder(); - ev->Record.MutableRequest()->SetDatabase(DataBase); - return ev; - } - - bool Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& /*ctx*/) { - const auto& record = ev->Get()->Record; - - if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { - return false; - } - - KqpSessionId = record.GetResponse().GetSessionId(); - Y_ABORT_UNLESS(!KqpSessionId.empty()); - - return true; - } - - void CloseKqpSession(const TActorContext& ctx) { - if (KqpSessionId) { - auto ev = MakeCloseSessionRequest(); - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); - - KqpSessionId = ""; - } - } - - THolder MakeCloseSessionRequest() { - auto ev = MakeHolder(); - ev->Record.MutableRequest()->SetSessionId(KqpSessionId); - return ev; - } - - void SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, std::vector commits, const NActors::TActorContext& ctx) - { - // if (!AppData(ctx)->FeatureFlags.GetEnableTopicServiceTx()) { // savnik need this check? - // return Reply(Ydb::StatusIds::UNSUPPORTED, - // "Disabled transaction support for TopicService.", - // NKikimrIssues::TIssuesIds::DEFAULT_ERROR, - // ctx); - // } - - auto& record = ev->Get()->Record; - TxId = record.GetResponse().GetTxMeta().id(); - Y_ABORT_UNLESS(!TxId.empty()); - - - auto offsets = MakeHolder(); - offsets->Record.MutableRequest()->SetDatabase(DataBase); - offsets->Record.MutableRequest()->SetSessionId(KqpSessionId); - offsets->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_UNDEFINED); - offsets->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_TOPIC); - offsets->Record.MutableRequest()->MutableTxControl()->set_tx_id(TxId); - offsets->Record.MutableRequest()->MutableTopicOperations()->SetConsumer(Consumer); - - // savnik need set something else? - - auto* topic = offsets->Record.MutableRequest()->MutableTopicOperations()->AddTopics(); - - topic->set_path(Path); - - for(auto &c: commits) { - auto* partition = topic->add_partitions(); - partition->set_partition_id(c.PartitionId); - partition->set_force_commit(true); - partition->set_kill_read_session(true); - auto* offset = partition->add_partition_offsets(); - offset->set_end(c.Offset); - } - - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), offsets.Release()); - } - - void BeginTransaction(const NActors::TActorContext& ctx) { - auto begin = MakeHolder(); - - begin->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_BEGIN_TX); - begin->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write(); - begin->Record.MutableRequest()->SetSessionId(KqpSessionId); - begin->Record.MutableRequest()->SetDatabase(DataBase); - - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), begin.Release()); - } - - void CommitTx(const NActors::TActorContext& ctx) { - auto commit = MakeHolder(); - - commit->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_COMMIT_TX); - commit->Record.MutableRequest()->MutableTxControl()->set_tx_id(TxId); - commit->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true); - commit->Record.MutableRequest()->SetSessionId(KqpSessionId); - commit->Record.MutableRequest()->SetDatabase(DataBase); - - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), commit.Release()); - } -}; - class TCommitOffsetActor : public TRpcOperationRequestActor { using TBase = TRpcOperationRequestActor; @@ -219,8 +98,7 @@ class TCommitOffsetActor : public TRpcOperationRequestActor Commits; + std::unique_ptr Kqp; }; } diff --git a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp new file mode 100644 index 000000000000..3361f092bedf --- /dev/null +++ b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp @@ -0,0 +1,132 @@ +#include "kqp_commit_offset_helper.h" +#include "ydb/core/kqp/common/simple/services.h" + +namespace NKikimr::NGRpcProxy::V1 { + +TKqpHelper::TKqpHelper(TString database, TString consumer, TString path, std::vector commits) + : DataBase(database) + , Consumer(consumer) + , Path(path) + , Commits(std::move(commits)) // savnik move + , Step(BEGIN_TRANSACTION_SENDED) +{} + +TKqpHelper::ECurrentStep TKqpHelper::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { + switch (Step) { + case BEGIN_TRANSACTION_SENDED: + Step = OFFSETS_SENDED; + SendCommits(ev, ctx); + break; + case OFFSETS_SENDED: + Step = COMMIT_SENDED; + CommitTx(ctx); + break; + case COMMIT_SENDED: + Step = DONE; + CloseKqpSession(ctx); // savnik wait? + break; + case DONE: + break; + } + return Step; +} + +void TKqpHelper::SendCreateSessionRequest(const TActorContext& ctx) { + auto ev = MakeCreateSessionRequest(); + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); +} + +void TKqpHelper::BeginTransaction(const NActors::TActorContext& ctx) { + auto begin = MakeHolder(); + + begin->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_BEGIN_TX); + begin->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write(); + begin->Record.MutableRequest()->SetSessionId(KqpSessionId); + begin->Record.MutableRequest()->SetDatabase(DataBase); + + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), begin.Release()); +} + +bool TKqpHelper::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& /*ctx*/) { + const auto& record = ev->Get()->Record; + + if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { + return false; + } + + KqpSessionId = record.GetResponse().GetSessionId(); + Y_ABORT_UNLESS(!KqpSessionId.empty()); + + return true; +} + +void TKqpHelper::CloseKqpSession(const TActorContext& ctx) { + if (KqpSessionId) { + auto ev = MakeCloseSessionRequest(); + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); + KqpSessionId = ""; + } +} + +THolder TKqpHelper::MakeCreateSessionRequest() { + auto ev = MakeHolder(); + ev->Record.MutableRequest()->SetDatabase(DataBase); + return ev; +} + +THolder TKqpHelper::MakeCloseSessionRequest() { + auto ev = MakeHolder(); + ev->Record.MutableRequest()->SetSessionId(KqpSessionId); + return ev; +} + +void TKqpHelper::SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const NActors::TActorContext& ctx) { + // if (!AppData(ctx)->FeatureFlags.GetEnableTopicServiceTx()) { // savnik need this check? + // return Reply(Ydb::StatusIds::UNSUPPORTED, + // "Disabled transaction support for TopicService.", + // NKikimrIssues::TIssuesIds::DEFAULT_ERROR, + // ctx); + // } + + auto& record = ev->Get()->Record; + TxId = record.GetResponse().GetTxMeta().id(); + Y_ABORT_UNLESS(!TxId.empty()); + + auto offsets = MakeHolder(); + offsets->Record.MutableRequest()->SetDatabase(DataBase); + offsets->Record.MutableRequest()->SetSessionId(KqpSessionId); + offsets->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_UNDEFINED); + offsets->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_TOPIC); + offsets->Record.MutableRequest()->MutableTxControl()->set_tx_id(TxId); + offsets->Record.MutableRequest()->MutableTopicOperations()->SetConsumer(Consumer); + + // savnik need set something else? + + auto* topic = offsets->Record.MutableRequest()->MutableTopicOperations()->AddTopics(); + topic->set_path(Path); + + for(auto &c: Commits) { + auto* partition = topic->add_partitions(); + partition->set_partition_id(c.PartitionId); + partition->set_force_commit(true); + partition->set_kill_read_session(true); + auto* offset = partition->add_partition_offsets(); + offset->set_end(c.Offset); + } + + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), offsets.Release()); +} + +void TKqpHelper::CommitTx(const NActors::TActorContext& ctx) { + auto commit = MakeHolder(); + + commit->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_COMMIT_TX); + commit->Record.MutableRequest()->MutableTxControl()->set_tx_id(TxId); + commit->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true); + commit->Record.MutableRequest()->SetSessionId(KqpSessionId); + commit->Record.MutableRequest()->SetDatabase(DataBase); + + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), commit.Release()); +} + +} // namespace NKikimr::NGRpcProxy::V1 diff --git a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h new file mode 100644 index 000000000000..8e701f947bc9 --- /dev/null +++ b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include + +namespace NKikimr::NGRpcProxy::V1 { + +using namespace NKikimr::NGRpcService; + +class TKqpHelper { +public: + enum ECurrentStep { + BEGIN_TRANSACTION_SENDED, + OFFSETS_SENDED, + COMMIT_SENDED, + DONE + }; + + struct TCommitInfo { + ui64 PartitionId; + i64 Offset; + }; + + TKqpHelper(TString database, TString consumer, TString path, std::vector commits); + + ECurrentStep Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx); + void SendCreateSessionRequest(const TActorContext& ctx); + void BeginTransaction(const NActors::TActorContext& ctx); + bool Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx); + +private: + void CloseKqpSession(const TActorContext& ctx); + THolder MakeCreateSessionRequest(); + THolder MakeCloseSessionRequest(); + void SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const NActors::TActorContext& ctx); + void CommitTx(const NActors::TActorContext& ctx); + +private: + TString DataBase; + TString Consumer; + TString Path; + std::vector Commits; + ECurrentStep Step; + + TString TxId; + TString KqpSessionId; +}; + +} // namespace NKikimr::NGRpcProxy::V1 diff --git a/ydb/services/persqueue_v1/actors/ya.make b/ydb/services/persqueue_v1/actors/ya.make index 75aafd8cc5d6..6de67668912b 100644 --- a/ydb/services/persqueue_v1/actors/ya.make +++ b/ydb/services/persqueue_v1/actors/ya.make @@ -28,6 +28,8 @@ SRCS( codecs.cpp commit_offset_actor.h commit_offset_actor.cpp + kqp_commit_offset_helper.h + kqp_commit_offset_helper.cpp events.h persqueue_utils.h persqueue_utils.cpp From f09dc1b814227c63edd5c43ee297c848d42e0071 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Fri, 15 Nov 2024 00:49:54 +0000 Subject: [PATCH 04/29] fix --- ydb/core/kqp/topics/kqp_topics.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ydb/core/kqp/topics/kqp_topics.cpp b/ydb/core/kqp/topics/kqp_topics.cpp index c5d01c3b13ed..6bd734dcd07a 100644 --- a/ydb/core/kqp/topics/kqp_topics.cpp +++ b/ydb/core/kqp/topics/kqp_topics.cpp @@ -36,7 +36,9 @@ std::pair TConsumerOperations::GetOffsetsCommitRange() const if (Offsets_.Empty()) { return {0,0}; - } else return {Offsets_.Min(), Offsets_.Max()}; + } else { + return {Offsets_.Min(), Offsets_.Max()}; + } } bool TConsumerOperations::GetForceCommit() const From ffc75acfe65f6e522363ee20f9d488c2966c84b4 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Fri, 15 Nov 2024 07:04:57 +0000 Subject: [PATCH 05/29] fix --- ydb/core/persqueue/ut/partition_ut.cpp | 4 ++-- ydb/core/persqueue/ut/user_action_processor_ut.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 1f67f68deed2..c06681753ae3 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -957,8 +957,8 @@ void TPartitionFixture::SendProposeTransactionRequest(ui32 partition, auto* body = event->Record.MutableData(); auto* operation = body->MutableOperations()->Add(); operation->SetPartitionId(partition); - operation->SetBegin(begin); - operation->SetEnd(end); + operation->SetCommitOffsetsBegin(begin); + operation->SetCommitOffsetsEnd(end); operation->SetConsumer(client); operation->SetPath(topic); body->SetImmediate(immediate); diff --git a/ydb/core/persqueue/ut/user_action_processor_ut.cpp b/ydb/core/persqueue/ut/user_action_processor_ut.cpp index 0796c620f39b..a4fd26e8d51c 100644 --- a/ydb/core/persqueue/ut/user_action_processor_ut.cpp +++ b/ydb/core/persqueue/ut/user_action_processor_ut.cpp @@ -653,8 +653,8 @@ void TUserActionProcessorFixture::SendProposeTransactionRequest(ui32 partition, auto* body = event->Record.MutableTxBody(); auto* operation = body->MutableOperations()->Add(); operation->SetPartitionId(partition); - operation->SetBegin(begin); - operation->SetEnd(end); + operation->SetCommitOffsetsBegin(begin); + operation->SetCommitOffsetsEnd(end); operation->SetConsumer(client); operation->SetPath(topic); body->SetImmediate(immediate); From b625c97f3dc911b7676de10f510b3487d9b63f83 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Mon, 18 Nov 2024 11:35:44 +0000 Subject: [PATCH 06/29] few fixes --- ydb/core/persqueue/partition.cpp | 5 ++-- ydb/core/persqueue/ut/pqtablet_ut.cpp | 4 ++-- .../persqueue/ut/user_action_processor_ut.cpp | 4 ++-- ydb/core/persqueue/utils.cpp | 24 +++++++++++++++---- ydb/core/persqueue/utils.h | 4 +++- .../actors/commit_offset_actor.cpp | 2 +- 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 9ef81c908bae..779bffab3c21 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -175,7 +175,6 @@ NKikimrClient::TKeyValueRequest::EStorageChannel GetChannel(ui32 i) { return NKikimrClient::TKeyValueRequest::EStorageChannel(NKikimrClient::TKeyValueRequest::MAIN + i); } - void AddCheckDiskRequest(TEvKeyValue::TEvRequest *request, ui32 numChannels) { for (ui32 i = 0; i < numChannels; ++i) { request->Record.AddCmdGetStatus()->SetStorageChannel(GetChannel(i)); @@ -2332,8 +2331,8 @@ TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPr bool isAffectedConsumer = AffectedUsers.contains(consumer); // savnik check TUserInfoBase& userInfo = GetOrCreatePendingUser(consumer); - if (operation.HasOnlyCheckCommitedToFinish() && operation.GetOnlyCheckCommitedToFinish()) { - if (static_cast(userInfo.Offset) != EndOffset) { // savnik что если откатят коммит пока выполняется транзакция + if (operation.HasOnlyCheckCommitedToFinish() && operation.GetOnlyCheckCommitedToFinish() && !IsActive()) { + if (static_cast(userInfo.Offset) != EndOffset) { ok = false; } } else { diff --git a/ydb/core/persqueue/ut/pqtablet_ut.cpp b/ydb/core/persqueue/ut/pqtablet_ut.cpp index 86e5fe7f2df0..ad86fb76432d 100644 --- a/ydb/core/persqueue/ut/pqtablet_ut.cpp +++ b/ydb/core/persqueue/ut/pqtablet_ut.cpp @@ -331,8 +331,8 @@ void TPQTabletFixture::SendProposeTransactionRequest(const TProposeTransactionPa auto* operation = body->MutableOperations()->Add(); operation->SetPartitionId(txOp.Partition); if (txOp.Begin.Defined()) { - operation->SetBegin(*txOp.Begin); - operation->SetEnd(*txOp.End); + operation->SetCommitOffsetsBegin(*txOp.Begin); + operation->SetCommitOffsetsEnd(*txOp.End); operation->SetConsumer(*txOp.Consumer); } operation->SetPath(txOp.Path); diff --git a/ydb/core/persqueue/ut/user_action_processor_ut.cpp b/ydb/core/persqueue/ut/user_action_processor_ut.cpp index a4fd26e8d51c..348da76588cd 100644 --- a/ydb/core/persqueue/ut/user_action_processor_ut.cpp +++ b/ydb/core/persqueue/ut/user_action_processor_ut.cpp @@ -679,8 +679,8 @@ void TUserActionProcessorFixture::SendProposeTransactionRequest(const TProposeTr for (auto& txOp : params.TxOps) { auto* operation = body->MutableOperations()->Add(); operation->SetPartitionId(txOp.Partition); - operation->SetBegin(txOp.Begin); - operation->SetEnd(txOp.End); + operation->SetCommitOffsetsBegin(txOp.Begin); + operation->SetCommitOffsetsEnd(txOp.End); operation->SetConsumer(txOp.Consumer); operation->SetPath(txOp.Path); } diff --git a/ydb/core/persqueue/utils.cpp b/ydb/core/persqueue/utils.cpp index 7f7ca3d81e10..f2743180665c 100644 --- a/ydb/core/persqueue/utils.cpp +++ b/ydb/core/persqueue/utils.cpp @@ -266,7 +266,8 @@ std::unordered_map BuildGraph(const TCollection& pa } std::deque queue; - for(const auto& p : partitions) { + + for (const auto& p : partitions) { auto& node = result[GetPartitionId(p)]; node.Children.reserve(p.ChildPartitionIdsSize()); @@ -284,12 +285,12 @@ std::unordered_map BuildGraph(const TCollection& pa } } - while(!queue.empty()) { + while (!queue.empty()) { auto* n = queue.front(); queue.pop_front(); bool allCompleted = true; - for(auto* c : n->Parents) { + for (auto* c : n->Parents) { if (c->HierarhicalParents.empty() && !c->Parents.empty()) { allCompleted = false; break; @@ -297,7 +298,7 @@ std::unordered_map BuildGraph(const TCollection& pa } if (allCompleted) { - for(auto* c : n->Parents) { + for (auto* c : n->Parents) { n->HierarhicalParents.insert(c->HierarhicalParents.begin(), c->HierarhicalParents.end()); n->HierarhicalParents.insert(c); } @@ -305,6 +306,21 @@ std::unordered_map BuildGraph(const TCollection& pa } } + for (auto& [_, node] : result) { + queue.push_back(&node); + + while (!queue.empty()) { + auto* current = queue.front(); + queue.pop_front(); + + for (auto* child : current->Children) { + if (node.HierarhicalChildren.insert(child).second) { + queue.push_back(child); + } + } + } + } + return result; } diff --git a/ydb/core/persqueue/utils.h b/ydb/core/persqueue/utils.h index 4b4b296bdbb0..6b7b5157a8c7 100644 --- a/ydb/core/persqueue/utils.h +++ b/ydb/core/persqueue/utils.h @@ -48,7 +48,9 @@ class TPartitionGraph { // Direct children of this node std::vector Children; // All parents include parents of parents and so on - std::set HierarhicalParents; + std::set HierarhicalParents; // savnik rename to Ancestors? + // All children include children of children and so on + std::set HierarhicalChildren; bool IsRoot() const; bool IsParent(ui32 partitionId) const; diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index b1ce0702bdab..a3671dc30fa3 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -115,7 +115,7 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc commits.push_back(commit); } - for (auto& child: partitionNode->Children) { + for (auto& child: partitionNode->HierarhicalChildren) { TKqpHelper::TCommitInfo commit {.PartitionId = child->Id, .Offset = 0}; commits.push_back(commit); } From 53e51649a917afa6e895806c455b9d8c2d67c07e Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 27 Nov 2024 00:15:16 +0000 Subject: [PATCH 07/29] distributed tx commit on read session --- ydb/core/persqueue/partition.cpp | 96 ++++--- ydb/core/persqueue/partition.h | 13 +- ydb/core/persqueue/partition_read.cpp | 4 +- ydb/core/persqueue/user_info.cpp | 16 +- ydb/core/persqueue/user_info.h | 9 +- .../ut/common/autoscaling_ut_common.cpp | 3 +- .../ut/ut_with_sdk/autoscaling_ut.cpp | 268 ++++++++++++++++++ ydb/core/protos/msgbus_pq.proto | 2 +- ydb/core/protos/pqconfig.proto | 2 +- .../actors/commit_offset_actor.cpp | 80 +++--- .../persqueue_v1/actors/commit_offset_actor.h | 2 - ydb/services/persqueue_v1/actors/events.h | 4 +- .../actors/kqp_commit_offset_helper.cpp | 33 +-- .../actors/kqp_commit_offset_helper.h | 5 +- .../persqueue_v1/actors/partition_actor.cpp | 181 +++++++----- .../persqueue_v1/actors/partition_actor.h | 18 +- .../actors/read_session_actor.cpp | 41 ++- .../persqueue_v1/actors/read_session_actor.h | 1 + 18 files changed, 562 insertions(+), 216 deletions(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 779bffab3c21..4c6cbc3dad6f 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -167,8 +167,8 @@ void TPartition::ReplyOk(const TActorContext& ctx, const ui64 dst, NWilson::TSpa } void TPartition::ReplyGetClientOffsetOk(const TActorContext& ctx, const ui64 dst, const i64 offset, - const TInstant writeTimestamp, const TInstant createTimestamp) { - ctx.Send(Tablet, MakeReplyGetClientOffsetOk(dst, offset, writeTimestamp, createTimestamp).Release()); + const TInstant writeTimestamp, const TInstant createTimestamp, bool consumerHasAnyCommits) { + ctx.Send(Tablet, MakeReplyGetClientOffsetOk(dst, offset, writeTimestamp, createTimestamp, consumerHasAnyCommits).Release()); } NKikimrClient::TKeyValueRequest::EStorageChannel GetChannel(ui32 i) { @@ -2524,19 +2524,22 @@ void TPartition::CommitTransaction(TSimpleSharedPtr& t) Y_ABORT_UNLESS(userInfo.Offset == (i64)operation.GetCommitOffsetsBegin()); } - if (operation.GetCommitOffsetsEnd() < StartOffset) { + if (operation.GetCommitOffsetsEnd() <= StartOffset) { + userInfo.AnyCommits = false; userInfo.Offset = StartOffset; } else if (operation.GetCommitOffsetsEnd() > EndOffset) { + userInfo.AnyCommits = true; userInfo.Offset = EndOffset; } else { + userInfo.AnyCommits = true; userInfo.Offset = operation.GetCommitOffsetsEnd(); } - if (operation.GetKillReadSession()) { // savnik check default + if (operation.GetKillReadSession()) { userInfo.Session = ""; userInfo.PartitionSessionId = 0; userInfo.Generation = 0; - userInfo.Step = 0; // savnik check need drop this + userInfo.Step = 0; userInfo.PipeClient = {}; } } @@ -2662,6 +2665,9 @@ void TPartition::OnProcessTxsAndUserActsWriteComplete(const TActorContext& ctx) userInfo.Generation = actual->Generation; userInfo.Step = actual->Step; userInfo.Offset = actual->Offset; + if (userInfo.Offset <= (i64)StartOffset) { + userInfo.AnyCommits = false; + } userInfo.ReadRuleGeneration = actual->ReadRuleGeneration; userInfo.ReadFromTimestamp = actual->ReadFromTimestamp; userInfo.HasReadRule = true; @@ -2838,7 +2844,7 @@ TPartition::EProcessResult TPartition::PreProcessImmediateTx(const NKikimrPQ::TE return EProcessResult::Continue; } -void TPartition::ExecImmediateTx(TTransaction& t) +void TPartition::ExecImmediateTx(TTransaction& t) // savnik { --ImmediateTxCount; const auto& record = t.ProposeTransaction->GetRecord(); @@ -2870,7 +2876,7 @@ void TPartition::ExecImmediateTx(TTransaction& t) "the consumer has been deleted"); return; } - TUserInfoBase& userInfo = GetOrCreatePendingUser(user); + TUserInfoBase& pendingUserInfo = GetOrCreatePendingUser(user); if (operation.GetCommitOffsetsBegin() > operation.GetCommitOffsetsEnd()) { ScheduleReplyPropose(record, @@ -2880,7 +2886,7 @@ void TPartition::ExecImmediateTx(TTransaction& t) return; } - if (userInfo.Offset != (i64)operation.GetCommitOffsetsBegin()) { + if (pendingUserInfo.Offset != (i64)operation.GetCommitOffsetsBegin()) { ScheduleReplyPropose(record, NKikimrPQ::TEvProposeTransactionResult::ABORTED, NKikimrPQ::TError::BAD_REQUEST, @@ -2895,7 +2901,7 @@ void TPartition::ExecImmediateTx(TTransaction& t) "incorrect offset range (commit to the future)"); return; } - userInfo.Offset = operation.GetCommitOffsetsEnd(); + pendingUserInfo.Offset = operation.GetCommitOffsetsEnd(); } CommitWriteOperations(t); @@ -3031,7 +3037,7 @@ void TPartition::CommitUserAct(TEvPQ::TEvSetClientInfo& act) { userInfo.PipeClient = act.PipeClient; ScheduleReplyGetClientOffsetOk(act.Cookie, userInfo.Offset, - ts.first, ts.second); + ts.first, ts.second, ui->AnyCommits); return; } @@ -3127,7 +3133,7 @@ void TPartition::EmulatePostProcessUserAct(const TEvPQ::TEvSetClientInfo& act, ui32 step = act.Step; const ui64 readRuleGeneration = act.ReadRuleGeneration; - bool setSession = act.Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION; + bool createSession = act.Type == TEvPQ::TEvSetClientInfo::ESCI_CREATE_SESSION; bool dropSession = act.Type == TEvPQ::TEvSetClientInfo::ESCI_DROP_SESSION; bool commitNotFromReadSession = (act.Type == TEvPQ::TEvSetClientInfo::ESCI_OFFSET && act.SessionId.empty()); @@ -3136,6 +3142,7 @@ void TPartition::EmulatePostProcessUserAct(const TEvPQ::TEvSetClientInfo& act, userInfo.Session = ""; userInfo.Generation = userInfo.Step = 0; userInfo.Offset = 0; + userInfo.AnyCommits = false; PQ_LOG_D("Topic '" << TopicName() << "' partition " << Partition << " user " << user << " drop done" @@ -3151,24 +3158,25 @@ void TPartition::EmulatePostProcessUserAct(const TEvPQ::TEvSetClientInfo& act, userInfo.PartitionSessionId = 0; userInfo.Generation = userInfo.Step = 0; userInfo.Offset = 0; + userInfo.AnyCommits = false; if (userInfo.Important) { userInfo.Offset = StartOffset; } } else { - if (setSession || dropSession) { + if (createSession || dropSession) { offset = userInfo.Offset; auto *ui = UsersInfoStorage->GetIfExists(userInfo.User); auto ts = ui ? GetTime(*ui, userInfo.Offset) : std::make_pair(TInstant::Zero(), TInstant::Zero()); ScheduleReplyGetClientOffsetOk(act.Cookie, offset, - ts.first, ts.second); + ts.first, ts.second, ui->AnyCommits); } else { ScheduleReplyOk(act.Cookie); } - if (setSession) { + if (createSession) { userInfo.Session = session; userInfo.Generation = generation; userInfo.Step = step; @@ -3184,17 +3192,20 @@ void TPartition::EmulatePostProcessUserAct(const TEvPQ::TEvSetClientInfo& act, Y_ABORT_UNLESS(offset <= (ui64)Max(), "Unexpected Offset: %" PRIu64, offset); PQ_LOG_D("Topic '" << TopicName() << "' partition " << Partition << " user " << user - << (setSession || dropSession ? " session" : " offset") + << (createSession || dropSession ? " session" : " offset") << " is set to " << offset << " (startOffset " << StartOffset << ") session " << session ); userInfo.Offset = offset; + if (userInfo.Offset <= (i64)StartOffset) { + userInfo.AnyCommits = false; + } if (LastOffsetHasBeenCommited(userInfo)) { SendReadingFinished(user); } - auto counter = setSession ? COUNTER_PQ_CREATE_SESSION_OK : (dropSession ? COUNTER_PQ_DELETE_SESSION_OK : COUNTER_PQ_SET_CLIENT_OFFSET_OK); + auto counter = createSession ? COUNTER_PQ_CREATE_SESSION_OK : (dropSession ? COUNTER_PQ_DELETE_SESSION_OK : COUNTER_PQ_SET_CLIENT_OFFSET_OK); TabletCounters.Cumulative()[counter].Increment(1); } } @@ -3207,12 +3218,16 @@ void TPartition::ScheduleReplyOk(const ui64 dst) void TPartition::ScheduleReplyGetClientOffsetOk(const ui64 dst, const i64 offset, - const TInstant writeTimestamp, const TInstant createTimestamp) + const TInstant writeTimestamp, + const TInstant createTimestamp, + bool consumerHasAnyCommits) { Replies.emplace_back(Tablet, MakeReplyGetClientOffsetOk(dst, offset, - writeTimestamp, createTimestamp).Release()); + writeTimestamp, + createTimestamp, + consumerHasAnyCommits).Release()); } @@ -3287,7 +3302,8 @@ void TPartition::AddCmdWrite(NKikimrClient::TKeyValueRequest& request, const TKeyPrefix& ikey, const TKeyPrefix& ikeyDeprecated, ui64 offset, ui32 gen, ui32 step, const TString& session, ui64 readOffsetRewindSum, - ui64 readRuleGeneration) + ui64 readRuleGeneration, + bool anyCommits) { TBuffer idata; { @@ -3298,6 +3314,7 @@ void TPartition::AddCmdWrite(NKikimrClient::TKeyValueRequest& request, userData.SetSession(session); userData.SetOffsetRewindSum(readOffsetRewindSum); userData.SetReadRuleGeneration(readRuleGeneration); + userData.SetAnyCommits(anyCommits); TString out; Y_PROTOBUF_SUPPRESS_NODISCARD userData.SerializeToString(&out); @@ -3358,7 +3375,8 @@ void TPartition::AddCmdWriteUserInfos(NKikimrClient::TKeyValueRequest& request) userInfo->Offset, userInfo->Generation, userInfo->Step, userInfo->Session, ui ? ui->ReadOffsetRewindSum : 0, - userInfo->ReadRuleGeneration); + userInfo->ReadRuleGeneration, + userInfo->AnyCommits); } else { AddCmdDeleteRange(request, ikey, ikeyDeprecated); @@ -3387,27 +3405,27 @@ TUserInfoBase& TPartition::GetOrCreatePendingUser(const TString& user, TMaybe readRuleGeneration) { TUserInfoBase* userInfo = nullptr; - auto i = PendingUsersInfo.find(user); - if (i == PendingUsersInfo.end()) { - auto ui = UsersInfoStorage->GetIfExists(user); - auto [p, _] = PendingUsersInfo.emplace(user, UsersInfoStorage->CreateUserInfo(user, readRuleGeneration)); + auto pendingUserIt = PendingUsersInfo.find(user); + if (pendingUserIt == PendingUsersInfo.end()) { + auto userIt = UsersInfoStorage->GetIfExists(user); + auto [newPendingUserIt, _] = PendingUsersInfo.emplace(user, UsersInfoStorage->CreateUserInfo(user, readRuleGeneration)); - if (ui) { - p->second.Session = ui->Session; - p->second.PartitionSessionId = ui->PartitionSessionId; - p->second.PipeClient = ui->PipeClient; + if (userIt) { + newPendingUserIt->second.Session = userIt->Session; + newPendingUserIt->second.PartitionSessionId = userIt->PartitionSessionId; + newPendingUserIt->second.PipeClient = userIt->PipeClient; - p->second.Generation = ui->Generation; - p->second.Step = ui->Step; - p->second.Offset = ui->Offset; - p->second.ReadRuleGeneration = ui->ReadRuleGeneration; - p->second.Important = ui->Important; - p->second.ReadFromTimestamp = ui->ReadFromTimestamp; + newPendingUserIt->second.Generation = userIt->Generation; + newPendingUserIt->second.Step = userIt->Step; + newPendingUserIt->second.Offset = userIt->Offset; + newPendingUserIt->second.ReadRuleGeneration = userIt->ReadRuleGeneration; + newPendingUserIt->second.Important = userIt->Important; + newPendingUserIt->second.ReadFromTimestamp = userIt->ReadFromTimestamp; } - userInfo = &p->second; + userInfo = &newPendingUserIt->second; } else { - userInfo = &i->second; + userInfo = &pendingUserIt->second; } AffectedUsers.insert(user); @@ -3437,7 +3455,8 @@ THolder TPartition::MakeReplyOk(const ui64 dst) THolder TPartition::MakeReplyGetClientOffsetOk(const ui64 dst, const i64 offset, const TInstant writeTimestamp, - const TInstant createTimestamp) + const TInstant createTimestamp, + bool consumerHasAnyCommits) { auto response = MakeHolder(dst); NKikimrClient::TResponse& resp = *response->Response; @@ -3461,10 +3480,9 @@ THolder TPartition::MakeReplyGetClientOffsetOk(const ui } else { user->SetSizeLag(0); } - + user->SetClientHasAnyCommits(consumerHasAnyCommits); return response; } - THolder TPartition::MakeReplyError(const ui64 dst, NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error) diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h index 1cfa15852463..f780be84169c 100644 --- a/ydb/core/persqueue/partition.h +++ b/ydb/core/persqueue/partition.h @@ -158,7 +158,7 @@ class TPartition : public TActorBootstrapped { NKikimrPQ::TError::EKind kind, const TString& reason); void ReplyErrorForStoredWrites(const TActorContext& ctx); - void ReplyGetClientOffsetOk(const TActorContext& ctx, const ui64 dst, const i64 offset, const TInstant writeTimestamp, const TInstant createTimestamp); + void ReplyGetClientOffsetOk(const TActorContext& ctx, const ui64 dst, const i64 offset, const TInstant writeTimestamp, const TInstant createTimestamp, bool consumerHasAnyCommits); void ReplyOk(const TActorContext& ctx, const ui64 dst); void ReplyOk(const TActorContext& ctx, const ui64 dst, NWilson::TSpan& span); void ReplyOwnerOk(const TActorContext& ctx, const ui64 dst, const TString& ownerCookie, ui64 seqNo, NWilson::TSpan& span); @@ -345,7 +345,9 @@ class TPartition : public TActorBootstrapped { void ScheduleReplyOk(const ui64 dst); void ScheduleReplyGetClientOffsetOk(const ui64 dst, const i64 offset, - const TInstant writeTimestamp, const TInstant createTimestamp); + const TInstant writeTimestamp, + const TInstant createTimestamp, + bool consumerHasAnyCommits); void ScheduleReplyError(const ui64 dst, NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error); @@ -361,7 +363,8 @@ class TPartition : public TActorBootstrapped { const TKeyPrefix& ikey, const TKeyPrefix& ikeyDeprecated, ui64 offset, ui32 gen, ui32 step, const TString& session, ui64 readOffsetRewindSum, - ui64 readRuleGeneration); + ui64 readRuleGeneration, + bool anyCommits); void AddCmdWriteTxMeta(NKikimrClient::TKeyValueRequest& request); void AddCmdWriteUserInfos(NKikimrClient::TKeyValueRequest& request); void AddCmdWriteConfig(NKikimrClient::TKeyValueRequest& request); @@ -374,7 +377,9 @@ class TPartition : public TActorBootstrapped { THolder MakeReplyOk(const ui64 dst); THolder MakeReplyGetClientOffsetOk(const ui64 dst, const i64 offset, - const TInstant writeTimestamp, const TInstant createTimestamp); + const TInstant writeTimestamp, + const TInstant createTimestamp, + bool consumerHasAnyCommits); THolder MakeReplyError(const ui64 dst, NPersQueue::NErrorCode::EErrorCode errorCode, const TString& error); diff --git a/ydb/core/persqueue/partition_read.cpp b/ydb/core/persqueue/partition_read.cpp index 4161ea3fd99d..ab94848f6dbb 100644 --- a/ydb/core/persqueue/partition_read.cpp +++ b/ydb/core/persqueue/partition_read.cpp @@ -252,7 +252,7 @@ void TPartition::InitUserInfoForImportantClients(const TActorContext& ctx) { } if (!userInfo) { userInfo = &UsersInfoStorage->Create( - ctx, consumer.GetName(), 0, true, "", 0, 0, 0, 0, 0, TInstant::Zero(), {} + ctx, consumer.GetName(), 0, true, "", 0, 0, 0, 0, 0, TInstant::Zero(), {}, false ); } if (userInfo->Offset < (i64)StartOffset) @@ -314,7 +314,7 @@ void TPartition::Handle(TEvPQ::TEvGetClientOffset::TPtr& ev, const TActorContext ui64 offset = Max(userInfo.Offset, 0); auto ts = GetTime(userInfo, offset); TabletCounters.Cumulative()[COUNTER_PQ_GET_CLIENT_OFFSET_OK].Increment(1); - ReplyGetClientOffsetOk(ctx, ev->Get()->Cookie, userInfo.Offset, ts.first, ts.second); + ReplyGetClientOffsetOk(ctx, ev->Get()->Cookie, userInfo.Offset, ts.first, ts.second, userInfo.AnyCommits); } void TPartition::Handle(TEvPQ::TEvSetClientInfo::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/core/persqueue/user_info.cpp b/ydb/core/persqueue/user_info.cpp index 92bf080dbf5c..0d75bbabc876 100644 --- a/ydb/core/persqueue/user_info.cpp +++ b/ydb/core/persqueue/user_info.cpp @@ -95,7 +95,7 @@ void TUsersInfoStorage::ParseDeprecated(const TString& key, const TString& data, Y_ABORT_UNLESS(offset <= (ui64)Max(), "Offset is too big: %" PRIu64, offset); if (!userInfo) { - Create(ctx, user, 0, false, session, 0, gen, step, static_cast(offset), 0, TInstant::Zero(), {}); + Create(ctx, user, 0, false, session, 0, gen, step, static_cast(offset), 0, TInstant::Zero(), {}, false); //savnik check anyCommits } else { userInfo->Session = session; userInfo->Generation = gen; @@ -123,7 +123,7 @@ void TUsersInfoStorage::Parse(const TString& key, const TString& data, const TAc Create( ctx, user, userData.GetReadRuleGeneration(), false, userData.GetSession(), userData.GetPartitionSessionId(), userData.GetGeneration(), userData.GetStep(), offset, - userData.GetOffsetRewindSum(), TInstant::Zero(), {} + userData.GetOffsetRewindSum(), TInstant::Zero(), {}, userData.GetAnyCommits() // savnik ); } else { userInfo->Session = userData.GetSession(); @@ -150,7 +150,7 @@ TUserInfo& TUsersInfoStorage::GetOrCreate(const TString& user, const TActorConte if (it == UsersInfo.end()) { return Create( ctx, user, readRuleGeneration ? *readRuleGeneration : ++CurReadRuleGeneration, false, "", 0, - 0, 0, 0, 0, TInstant::Zero(), {} + 0, 0, 0, 0, TInstant::Zero(), {}, false ); } return it->second; @@ -177,7 +177,7 @@ TUserInfo TUsersInfoStorage::CreateUserInfo(const TActorContext& ctx, const TString& session, ui64 partitionSessionId, ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, - TInstant readFromTimestamp, const TActorId& pipeClient) const + TInstant readFromTimestamp, const TActorId& pipeClient, bool anyCommits) const { TString defaultServiceType = AppData(ctx)->PQConfig.GetDefaultClientServiceType().GetName(); TString userServiceType = ""; @@ -195,7 +195,7 @@ TUserInfo TUsersInfoStorage::CreateUserInfo(const TActorContext& ctx, ctx, StreamCountersSubgroup, user, readRuleGeneration, important, TopicConverter, Partition, session, partitionSessionId, gen, step, offset, readOffsetRewindSum, DCId, readFromTimestamp, DbPath, - meterRead, pipeClient + meterRead, pipeClient, anyCommits }; } @@ -203,16 +203,16 @@ TUserInfoBase TUsersInfoStorage::CreateUserInfo(const TString& user, TMaybe readRuleGeneration) const { return TUserInfoBase{user, readRuleGeneration ? *readRuleGeneration : ++CurReadRuleGeneration, - "", 0, 0, 0, false, {}, 0, {}}; + "", 0, 0, 0, false, false, {}, 0, {}}; } TUserInfo& TUsersInfoStorage::Create( const TActorContext& ctx, const TString& user, const ui64 readRuleGeneration, bool important, const TString& session, ui64 partitionSessionId, ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, - TInstant readFromTimestamp, const TActorId& pipeClient + TInstant readFromTimestamp, const TActorId& pipeClient, bool anyCommits ) { auto userInfo = CreateUserInfo(ctx, user, readRuleGeneration, important, session, partitionSessionId, - gen, step, offset, readOffsetRewindSum, readFromTimestamp, pipeClient); + gen, step, offset, readOffsetRewindSum, readFromTimestamp, pipeClient, anyCommits); auto result = UsersInfo.emplace(user, std::move(userInfo)); Y_ABORT_UNLESS(result.second); return result.first->second; diff --git a/ydb/core/persqueue/user_info.h b/ydb/core/persqueue/user_info.h index d93da603a10a..a9135a096309 100644 --- a/ydb/core/persqueue/user_info.h +++ b/ydb/core/persqueue/user_info.h @@ -45,6 +45,7 @@ struct TUserInfoBase { ui32 Generation = 0; ui32 Step = 0; i64 Offset = 0; + bool AnyCommits = false; // savnik что-то еще надо сделать, чтобы эта штука персистилась? bool Important = false; TInstant ReadFromTimestamp; @@ -170,9 +171,9 @@ struct TUserInfo: public TUserInfoBase { const ui64 readRuleGeneration, const bool important, const NPersQueue::TTopicConverterPtr& topicConverter, const ui32 partition, const TString& session, ui64 partitionSession, ui32 gen, ui32 step, i64 offset, const ui64 readOffsetRewindSum, const TString& dcId, TInstant readFromTimestamp, - const TString& dbPath, bool meterRead, const TActorId& pipeClient + const TString& dbPath, bool meterRead, const TActorId& pipeClient, bool anyCommits ) - : TUserInfoBase{user, readRuleGeneration, session, gen, step, offset, important, + : TUserInfoBase{user, readRuleGeneration, session, gen, step, offset, anyCommits, important, readFromTimestamp, partitionSession, pipeClient} , WriteTimestamp(TAppData::TimeProvider->Now()) , CreateTimestamp(TAppData::TimeProvider->Now()) @@ -388,7 +389,7 @@ class TUsersInfoStorage { TUserInfo& Create( const TActorContext& ctx, const TString& user, const ui64 readRuleGeneration, bool important, const TString& session, ui64 partitionSessionId, ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, - TInstant readFromTimestamp, const TActorId& pipeClient + TInstant readFromTimestamp, const TActorId& pipeClient, bool anyCommits ); void Clear(const TActorContext& ctx); @@ -404,7 +405,7 @@ class TUsersInfoStorage { const TString& session, ui64 partitionSessionId, ui32 gen, ui32 step, i64 offset, ui64 readOffsetRewindSum, - TInstant readFromTimestamp, const TActorId& pipeClient) const; + TInstant readFromTimestamp, const TActorId& pipeClient, bool anyCommits) const; private: THashMap UsersInfo; diff --git a/ydb/core/persqueue/ut/common/autoscaling_ut_common.cpp b/ydb/core/persqueue/ut/common/autoscaling_ut_common.cpp index dc877d041c9e..779bf4776c94 100644 --- a/ydb/core/persqueue/ut/common/autoscaling_ut_common.cpp +++ b/ydb/core/persqueue/ut/common/autoscaling_ut_common.cpp @@ -261,7 +261,6 @@ std::shared_ptr::TSdkReadSession> TTestReadS TString{message.GetData()}, impl->AutoCommit) .WithMsg(new MsgWrapper(message)); - impl->ReceivedMessages.push_back(msg); if (impl->AutoCommit) { @@ -450,7 +449,7 @@ void TTestReadSession::WaitAllMessages() { template void TTestReadSession::Commit() { - Cerr << ">>>>> " << Impl->Name << " Commit all received messages" << Endl << Flush; + Cerr << ">>>>> " << Impl->Name << "Commit all received messages" << Endl << Flush; for (auto& m : Impl->ReceivedMessages) { if (!m.Commited) { m.Msg->Commit(); diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index 1fc252e06257..c0a50542b97d 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -950,6 +950,274 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::BAD_REQUEST); } + Y_UNIT_TEST(PartitionSplit_DistributedTxCommit) { + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(1) + .MaxActivePartitions(100) + .BeginConfigureAutoPartitioningSettings() + .UpUtilizationPercent(2) + .DownUtilizationPercent(1) + .StabilizationWindow(TDuration::Seconds(2)) + .Strategy(EAutoPartitioningStrategy::ScaleUp) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings() + .BeginAddConsumer() + .ConsumerName(TEST_CONSUMER); + client.CreateTopic(TEST_TOPIC, createSettings).Wait(); + + auto msg = TString(1_MB, 'a'); + + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, TEST_TOPIC, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, TEST_TOPIC, false); + + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 1))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 2))); + Sleep(TDuration::Seconds(5)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 1); + } + + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 3))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, 4))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 5))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, 6))); + Sleep(TDuration::Seconds(5)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); + } + + auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, TEST_TOPIC, false); + auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, TEST_TOPIC, false); + + { + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); + UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 8))); + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 9))); + UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 10))); + Sleep(TDuration::Seconds(5)); + auto describe2 = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe2.GetTopicDescription().GetPartitions().size(), 5); + } + + auto reader = client.CreateReadSession( + TReadSessionSettings() + .AutoPartitioningSupport(true) + .AppendTopics(TTopicReadSettings(TEST_TOPIC)) + .ConsumerName(TEST_CONSUMER)); + + TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(15); + auto count = 0; + auto expected = 10; + while(deadlineTime > TInstant::Now()) { + for (auto event : reader->GetEvents(false)) { + if (auto* x = std::get_if(&event)) { + auto& messages = x->GetMessages(); + for (size_t i = 0u; i < messages.size(); ++i) { + count++; + auto& message = messages[i]; + message.Commit(); + } + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* sessionClosedEvent = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT" << x->DebugString() << Endl << Flush; + } else { + Cerr << "SESSION EVENT unhandled \n"; + } + + if (count == expected) { + break; + } + } + Sleep(TDuration::MilliSeconds(250)); + } + Sleep(TDuration::Seconds(5)); + UNIT_ASSERT_EQUAL(count, expected); + + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + UNIT_ASSERT(description.GetPartitions().size() == 5); + + auto stats1 = description.GetPartitions().at(1).GetPartitionConsumerStats(); + UNIT_ASSERT(stats1.Defined()); + UNIT_ASSERT(stats1->GetCommittedOffset() == 4); + } + + Y_UNIT_TEST(PartitionSplit_DistributedTxCommit_ChildFirst) { + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(1) + .MaxActivePartitions(100) + .BeginConfigureAutoPartitioningSettings() + .UpUtilizationPercent(2) + .DownUtilizationPercent(1) + .StabilizationWindow(TDuration::Seconds(2)) + .Strategy(EAutoPartitioningStrategy::ScaleUp) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings() + .BeginAddConsumer() + .ConsumerName(TEST_CONSUMER); + + client.CreateTopic(TEST_TOPIC, createSettings).Wait(); + + auto msg = TString(1_MB, 'a'); + + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, TEST_TOPIC, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, TEST_TOPIC, false); + + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 1))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 2))); + Sleep(TDuration::Seconds(5)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 1); + } + + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 3))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, 4))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 5))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, 6))); + Sleep(TDuration::Seconds(5)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); + } + + auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, TEST_TOPIC, false); + auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, TEST_TOPIC, false); + + { + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); + UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 8))); + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 9))); + UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 10))); + Sleep(TDuration::Seconds(5)); + auto describe2 = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe2.GetTopicDescription().GetPartitions().size(), 5); + } + + auto reader = client.CreateReadSession( + TReadSessionSettings() + .AutoPartitioningSupport(true) + .AppendTopics(TTopicReadSettings(TEST_TOPIC)) + .ConsumerName(TEST_CONSUMER)); + + TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(15); + auto count = 0; + auto expected = 10; + + std::vector partition0Messages; + + while(deadlineTime > TInstant::Now()) { + for (auto event : reader->GetEvents(false)) { + if (auto* x = std::get_if(&event)) { + auto& messages = x->GetMessages(); + for (size_t i = 0u; i < messages.size(); ++i) { + auto& message = messages[i]; + count++; + int partitionId = message.GetPartitionSession()->GetPartitionId(); + + if (partitionId == 1) { + // Commit messages from partition 1 immediately + message.Commit(); + } else if (partitionId == 0) { + // Store messages from partition 0 for later + partition0Messages.push_back(message); + } + } + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* sessionClosedEvent = std::get_if(&event)) { + Cerr << sessionClosedEvent->DebugString() << Endl << Flush; + } else { + Cerr << "SESSION EVENT unhandled \n"; + } + + if (count == expected) { + break; + } + } + if (count == expected) { + break; + } + Sleep(TDuration::MilliSeconds(250)); + } + + UNIT_ASSERT_EQUAL(count, expected); + + Sleep(TDuration::Seconds(5)); + + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + UNIT_ASSERT(description.GetPartitions().size() == 5); + + auto stats1 = description.GetPartitions().at(1).GetPartitionConsumerStats(); + UNIT_ASSERT(stats1.Defined()); + + UNIT_ASSERT(stats1->GetCommittedOffset() == 0); + + for (auto& message : partition0Messages) { + message.Commit(); + } + + Sleep(TDuration::Seconds(5)); + + auto result2 = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description2 = result2.GetConsumerDescription(); + UNIT_ASSERT(description2.GetPartitions().size() == 5); + + stats1 = description2.GetPartitions().at(1).GetPartitionConsumerStats(); + UNIT_ASSERT(stats1.Defined()); + + UNIT_ASSERT(stats1->GetCommittedOffset() == 4); + } + Y_UNIT_TEST(PartitionSplit_AutosplitByLoad) { TTopicSdkTestSetup setup = CreateSetup(); TTopicClient client = setup.MakeClient(); diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto index 9e16ce29a372..ae4fe4ad0e91 100644 --- a/ydb/core/protos/msgbus_pq.proto +++ b/ydb/core/protos/msgbus_pq.proto @@ -472,7 +472,7 @@ message TPersQueuePartitionResponse { optional uint64 CreateTimestampMS = 6; //create Timestamp of record on Offset (next to be readed record); is not set if no such record exists (no lag) optional uint64 SizeLag = 7; optional uint64 WriteTimestampEstimateMS = 8; - + optional bool ClientHasAnyCommits = 9; } message TCmdGetOwnershipResult { diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index a03ff41ba084..b4f734ef26a1 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -904,7 +904,7 @@ message TUserInfo { optional uint64 OffsetRewindSum = 5; optional uint64 ReadRuleGeneration = 6; optional uint64 PartitionSessionId = 7; - + optional bool AnyCommits = 8; } message TPartitionClientInfo { diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index a3671dc30fa3..3be20e7d346a 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -111,7 +111,7 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc std::vector commits; for (auto& parent: partitionNode->HierarhicalParents) { - TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max()}; + TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); } @@ -120,13 +120,13 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc commits.push_back(commit); } - TKqpHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->Getoffset()}; + TKqpHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = Max(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); // savnik if empty database? - Kqp = std::make_unique(Request().GetDatabaseName().GetOrElse(TString()), ClientId, topic, commits); + Kqp = std::make_unique(Request().GetDatabaseName().GetOrElse(TString()), ClientId, topic, commits); // savnik add cookie? - SendDistributedTxOffsets(ctx); + Kqp->SendCreateSessionRequest(ctx); } } @@ -134,11 +134,41 @@ void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev if (!Kqp->Handle(ev, ctx)) { AnswerError("empty list of topics", PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx); // savnik } - Kqp->BeginTransaction(ctx); } -void TCommitOffsetActor::SendDistributedTxOffsets(const TActorContext& ctx) { - Kqp->SendCreateSessionRequest(ctx); +void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record; + if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { // savnik finish this part! + Ydb::Topic::CommitOffsetResult result; + Request().SendResult(result, record.GetYdbStatus()); + Die(ctx); + } + + auto step = Kqp->Handle(ev, ctx); + + if (step == TKqpHelper::ECurrentStep::DONE) { + Ydb::Topic::CommitOffsetResult result; + Request().SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); + } +} + +void TCommitOffsetActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { + if (ev->Get()->Record.GetStatus() != NMsgBusProxy::MSTATUS_OK) { + auto errorCode = ConvertOldCode(ev->Get()->Record.GetErrorCode()); + return AnswerError(ev->Get()->Record.GetErrorReason(), errorCode, ctx); + } + + // Convert to correct response. + + const auto& partitionResult = ev->Get()->Record.GetPartitionResponse(); + Y_ABORT_UNLESS(!partitionResult.HasCmdReadResult()); + + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "CommitOffset, commit done."); + + Ydb::Topic::CommitOffsetResult result; + Request().SendResult(result, Ydb::StatusIds::SUCCESS); + Die(ctx); } void TCommitOffsetActor::SendCommit(const TTopicInitInfo& topic, const Ydb::Topic::CommitOffsetRequest* commitRequest, const TActorContext& ctx) { @@ -176,42 +206,6 @@ void TCommitOffsetActor::SendCommit(const TTopicInitInfo& topic, const Ydb::Topi NTabletPipe::SendData(ctx, PipeClient, req.Release()); } -void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record; - if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { // savnik finish this part! - Ydb::Topic::CommitOffsetResult result; - Request().SendResult(result, record.GetYdbStatus()); - Die(ctx); - } - - auto step = Kqp->Handle(ev, ctx); - - if (step == TKqpHelper::ECurrentStep::DONE) { - Ydb::Topic::CommitOffsetResult result; - Request().SendResult(result, Ydb::StatusIds::SUCCESS); - Die(ctx); - } -} - -void TCommitOffsetActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { - if (ev->Get()->Record.GetStatus() != NMsgBusProxy::MSTATUS_OK) { - auto errorCode = ConvertOldCode(ev->Get()->Record.GetErrorCode()); - return AnswerError(ev->Get()->Record.GetErrorReason(), errorCode, ctx); - } - - // Convert to correct response. - - const auto& partitionResult = ev->Get()->Record.GetPartitionResponse(); - Y_ABORT_UNLESS(!partitionResult.HasCmdReadResult()); - - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "CommitOffset, commit done."); - - Ydb::Topic::CommitOffsetResult result; - Request().SendResult(result, Ydb::StatusIds::SUCCESS); - Die(ctx); -} - - void TCommitOffsetActor::AnswerError(const TString& errorReason, const PersQueue::ErrorCode::ErrorCode errorCode, const NActors::TActorContext& ctx) { Ydb::Topic::CommitOffsetResponse response; response.mutable_operation()->set_ready(true); diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.h b/ydb/services/persqueue_v1/actors/commit_offset_actor.h index f36203e967aa..597802ae0925 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.h +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.h @@ -72,11 +72,9 @@ class TCommitOffsetActor : public TRpcOperationRequestActor { - TEvPartitionStatus(const TPartitionId& partition, const ui64 offset, const ui64 endOffset, const ui64 writeTimestampEstimateMs, ui64 nodeId, ui64 generation, + TEvPartitionStatus(const TPartitionId& partition, const ui64 offset, const ui64 endOffset, const ui64 writeTimestampEstimateMs, ui64 nodeId, ui64 generation, bool clientHasAnyCommits, bool init = true) : Partition(partition) , Offset(offset) , EndOffset(endOffset) + , ClientHasAnyCommits(clientHasAnyCommits) , WriteTimestampEstimateMs(writeTimestampEstimateMs) , NodeId(nodeId) , Generation(generation) @@ -488,6 +489,7 @@ struct TEvPQProxy { TPartitionId Partition; ui64 Offset; ui64 EndOffset; + bool ClientHasAnyCommits; ui64 WriteTimestampEstimateMs; ui64 NodeId; ui64 Generation; diff --git a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp index 3361f092bedf..dc4cd1077740 100644 --- a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp +++ b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp @@ -3,12 +3,13 @@ namespace NKikimr::NGRpcProxy::V1 { -TKqpHelper::TKqpHelper(TString database, TString consumer, TString path, std::vector commits) +TKqpHelper::TKqpHelper(TString database, TString consumer, TString path, std::vector commits, ui64 cookie) : DataBase(database) , Consumer(consumer) , Path(path) , Commits(std::move(commits)) // savnik move , Step(BEGIN_TRANSACTION_SENDED) + , Cookie(cookie) {} TKqpHelper::ECurrentStep TKqpHelper::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { @@ -33,7 +34,7 @@ TKqpHelper::ECurrentStep TKqpHelper::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr void TKqpHelper::SendCreateSessionRequest(const TActorContext& ctx) { auto ev = MakeCreateSessionRequest(); - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release(), 0, Cookie); } void TKqpHelper::BeginTransaction(const NActors::TActorContext& ctx) { @@ -44,10 +45,10 @@ void TKqpHelper::BeginTransaction(const NActors::TActorContext& ctx) { begin->Record.MutableRequest()->SetSessionId(KqpSessionId); begin->Record.MutableRequest()->SetDatabase(DataBase); - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), begin.Release()); + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), begin.Release(), 0, Cookie); } -bool TKqpHelper::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& /*ctx*/) { +bool TKqpHelper::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { @@ -56,14 +57,14 @@ bool TKqpHelper::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const KqpSessionId = record.GetResponse().GetSessionId(); Y_ABORT_UNLESS(!KqpSessionId.empty()); - + BeginTransaction(ctx); return true; } void TKqpHelper::CloseKqpSession(const TActorContext& ctx) { if (KqpSessionId) { auto ev = MakeCloseSessionRequest(); - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release()); + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release(), 0, Cookie); KqpSessionId = ""; } } @@ -81,13 +82,6 @@ THolder TKqpHelper::MakeCloseSessionReques } void TKqpHelper::SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const NActors::TActorContext& ctx) { - // if (!AppData(ctx)->FeatureFlags.GetEnableTopicServiceTx()) { // savnik need this check? - // return Reply(Ydb::StatusIds::UNSUPPORTED, - // "Disabled transaction support for TopicService.", - // NKikimrIssues::TIssuesIds::DEFAULT_ERROR, - // ctx); - // } - auto& record = ev->Get()->Record; TxId = record.GetResponse().GetTxMeta().id(); Y_ABORT_UNLESS(!TxId.empty()); @@ -105,16 +99,17 @@ void TKqpHelper::SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const NAc auto* topic = offsets->Record.MutableRequest()->MutableTopicOperations()->AddTopics(); topic->set_path(Path); - for(auto &c: Commits) { + for(auto &commit: Commits) { auto* partition = topic->add_partitions(); - partition->set_partition_id(c.PartitionId); + partition->set_partition_id(commit.PartitionId); partition->set_force_commit(true); - partition->set_kill_read_session(true); + partition->set_kill_read_session(commit.KillReadSession); + partition->set_only_check_commited_to_finish(commit.OnlyCheckCommitedToFinish); auto* offset = partition->add_partition_offsets(); - offset->set_end(c.Offset); + offset->set_end(commit.Offset); } - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), offsets.Release()); + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), offsets.Release(), 0, Cookie); } void TKqpHelper::CommitTx(const NActors::TActorContext& ctx) { @@ -126,7 +121,7 @@ void TKqpHelper::CommitTx(const NActors::TActorContext& ctx) { commit->Record.MutableRequest()->SetSessionId(KqpSessionId); commit->Record.MutableRequest()->SetDatabase(DataBase); - ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), commit.Release()); + ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), commit.Release(), 0, Cookie); } } // namespace NKikimr::NGRpcProxy::V1 diff --git a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h index 8e701f947bc9..0b16810a5128 100644 --- a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h +++ b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h @@ -21,9 +21,11 @@ class TKqpHelper { struct TCommitInfo { ui64 PartitionId; i64 Offset; + bool KillReadSession; + bool OnlyCheckCommitedToFinish; }; - TKqpHelper(TString database, TString consumer, TString path, std::vector commits); + TKqpHelper(TString database, TString consumer, TString path, std::vector commits, ui64 cookie = 0); ECurrentStep Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx); void SendCreateSessionRequest(const TActorContext& ctx); @@ -43,6 +45,7 @@ class TKqpHelper { TString Path; std::vector Commits; ECurrentStep Step; + ui64 Cookie; TString TxId; TString KqpSessionId; diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index 6fe140ab5fd9..122a1711fc29 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -26,7 +26,7 @@ TPartitionActor::TPartitionActor( const TString& session, const TPartitionId& partition, const ui32 generation, const ui32 step, const ui64 tabletID, const TTopicCounters& counters, bool commitsDisabled, const TString& clientDC, bool rangesMode, const NPersQueue::TTopicConverterPtr& topic, - bool directRead, bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs + bool directRead, bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs, std::set parents ) : ParentId(parentId) , ClientId(clientId) @@ -64,10 +64,12 @@ TPartitionActor::TPartitionActor( , WaitDataCookie(0) , WaitForData(false) , LockCounted(false) + , Parents(parents) , Counters(counters) , CommitsDisabled(commitsDisabled) , CommitCookie(1) , Topic(topic) + , Database(database) , DirectRead(directRead) , UseMigrationProtocol(useMigrationProtocol) , FirstRead(true) @@ -91,7 +93,7 @@ void TPartitionActor::MakeCommit(const TActorContext& ctx) { ClientCommitOffset = offset; ++CommitCookie; - CommitsInfly.push_back(std::pair(CommitCookie, {CommitCookie, offset, ctx.Now()})); + CommitsInfly.emplace_back(CommitCookie, TCommitInfo{CommitCookie, offset, ctx.Now()}); if (Counters.SLITotal) Counters.SLITotal.Inc(); @@ -109,7 +111,7 @@ void TPartitionActor::MakeCommit(const TActorContext& ctx) { ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(Partition.AssignId, 0, 0, CommittedOffset)); } else { ClientCommitOffset = ClientReadOffset; - CommitsInfly.push_back(std::pair(0, {0, ClientReadOffset, ctx.Now()})); + CommitsInfly.emplace_back(0, TCommitInfo{0, ClientReadOffset, ctx.Now()}); if (Counters.SLITotal) Counters.SLITotal.Inc(); if (PipeClient) //if not then pipe will be recreated soon and SendCommit will be done @@ -141,7 +143,7 @@ void TPartitionActor::MakeCommit(const TActorContext& ctx) { Y_ABORT_UNLESS(offset > ClientCommitOffset); ClientCommitOffset = offset; - CommitsInfly.push_back(std::pair(readId, {startReadId, offset, ctx.Now()})); + CommitsInfly.emplace_back(readId, TCommitInfo{startReadId, offset, ctx.Now()}); if (Counters.SLITotal) Counters.SLITotal.Inc(); @@ -158,28 +160,75 @@ void TPartitionActor::Bootstrap(const TActorContext& ctx) { } void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx) { - NKikimrClient::TPersQueueRequest request; - request.MutablePartitionRequest()->SetTopic(Topic->GetPrimaryPath()); - request.MutablePartitionRequest()->SetPartition(Partition.Partition); - request.MutablePartitionRequest()->SetCookie(readId); + if (!ClientHasAnyCommits && Parents.size() != 0) { + std::vector commits; + for (auto& parent: Parents) { + TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true}; + commits.push_back(commit); + } + TKqpHelper::TCommitInfo commit {.PartitionId = Partition.Partition, .Offset = (i64)offset, .KillReadSession = false, .OnlyCheckCommitedToFinish = false}; + commits.push_back(commit); + auto kqp = std::make_shared(Database, ClientId, Topic->GetPrimaryPath(), commits, readId); + Kqps.emplace(readId, kqp); - Y_ABORT_UNLESS(PipeClient); + kqp->SendCreateSessionRequest(ctx); + } else { + NKikimrClient::TPersQueueRequest request; + request.MutablePartitionRequest()->SetTopic(Topic->GetPrimaryPath()); + request.MutablePartitionRequest()->SetPartition(Partition.Partition); + request.MutablePartitionRequest()->SetCookie(readId); - ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient()); - auto commit = request.MutablePartitionRequest()->MutableCmdSetClientOffset(); - commit->SetClientId(ClientId); - commit->SetOffset(offset); - Y_ABORT_UNLESS(!Session.empty()); - commit->SetSessionId(Session); + Y_ABORT_UNLESS(PipeClient); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition - << " committing to position " << offset << " prev " << CommittedOffset - << " end " << EndOffset << " by cookie " << readId); + ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient()); + auto commit = request.MutablePartitionRequest()->MutableCmdSetClientOffset(); + commit->SetClientId(ClientId); + commit->SetOffset(offset); + Y_ABORT_UNLESS(!Session.empty()); + commit->SetSessionId(Session); - TAutoPtr req(new TEvPersQueue::TEvRequest); - req->Record.Swap(&request); + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition + << " committing to position " << offset << " prev " << CommittedOffset + << " end " << EndOffset << " by cookie " << readId); - NTabletPipe::SendData(ctx, PipeClient, req.Release()); + TAutoPtr req(new TEvPersQueue::TEvRequest); + req->Record.Swap(&request); + + NTabletPipe::SendData(ctx, PipeClient, req.Release()); + } +} + +void TPartitionActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const NActors::TActorContext& ctx) { + auto kqpIt = Kqps.find(ev->Cookie); + if (kqpIt == Kqps.end()) { + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("unexpected cookie at KQP create session response", PersQueue::ErrorCode::ERROR)); + return; + } + + if (!kqpIt->second->Handle(ev, ctx)) { + const auto& record = ev->Get()->Record; + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("status is not ok: " + record.GetError(), PersQueue::ErrorCode::ERROR)); // savnik map ydbStatus + } +} + +void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record; + + auto kqpIt = Kqps.find(ev->Cookie); + if (kqpIt == Kqps.end()) { + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("unexpected cookie at KQP query response", PersQueue::ErrorCode::ERROR)); + return; + } + + if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", PersQueue::ErrorCode::ERROR)); // savnik map ydbStatus. Need retry? + return; + } + + auto step = kqpIt->second->Handle(ev, ctx); + if (step == TKqpHelper::ECurrentStep::DONE) { + CommitDone(ev->Cookie, ctx); + } } void TPartitionActor::SendPublishDirectRead(const ui64 directReadId, const TActorContext& ctx) { @@ -232,7 +281,6 @@ void TPartitionActor::SendForgetDirectRead(const ui64 directReadId, const TActor NTabletPipe::SendData(ctx, PipeClient, req.Release()); } - void TPartitionActor::RestartPipe(const TActorContext& ctx, const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode) { if (!PipeClient) return; @@ -256,7 +304,6 @@ void TPartitionActor::RestartPipe(const TActorContext& ctx, const TString& reaso ctx.Schedule(TDuration::MilliSeconds(RESTART_PIPE_DELAY_MS), new TEvPQProxy::TEvRestartPipe()); } - void TPartitionActor::Handle(TEvPQProxy::TEvDirectReadAck::TPtr& ev, const TActorContext& ctx) { auto it = DirectReadResults.find(ev->Get()->DirectReadId); @@ -295,8 +342,6 @@ void TPartitionActor::Handle(TEvPQProxy::TEvDirectReadAck::TPtr& ev, const TActo } - - void TPartitionActor::Handle(const TEvPQProxy::TEvRestartPipe::TPtr&, const TActorContext& ctx) { Y_ABORT_UNLESS(!PipeClient); @@ -341,7 +386,7 @@ void TPartitionActor::ResendRecentRequests() { NTabletPipe::SendData(ctx, PipeClient, event.Release()); } - if (InitDone) { + if (InitDone) { // savnik need resend? for (auto& c : CommitsInfly) { //resend all commits if (c.second.Offset != Max()) SendCommit(c.first, c.second.Offset, ctx); @@ -544,7 +589,6 @@ bool FillBatchedData( return hasData; } - void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { if (ev->Get()->Record.HasErrorCode() && ev->Get()->Record.GetErrorCode() != NPersQueue::NErrorCode::OK) { @@ -614,6 +658,7 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo EndOffset = resp.GetEndOffset(); SizeLag = resp.GetSizeLag(); WriteTimestampEstimateMs = resp.GetWriteTimestampEstimateMS(); + ClientHasAnyCommits = resp.GetClientHasAnyCommits(); ClientCommitOffset = ReadOffset = CommittedOffset = resp.HasOffset() ? resp.GetOffset() : 0; Y_ABORT_UNLESS(EndOffset >= CommittedOffset); @@ -628,7 +673,7 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo if (!StartReading) { - ctx.Send(ParentId, new TEvPQProxy::TEvPartitionStatus(Partition, CommittedOffset, EndOffset, WriteTimestampEstimateMs, NodeId, TabletGeneration)); + ctx.Send(ParentId, new TEvPQProxy::TEvPartitionStatus(Partition, CommittedOffset, EndOffset, WriteTimestampEstimateMs, NodeId, TabletGeneration, ClientHasAnyCommits)); } else { InitStartReading(ctx); } @@ -680,41 +725,7 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo } if (!(result.HasCmdReadResult() || result.HasCmdPrepareReadResult() || result.HasCmdPublishReadResult() || result.HasCmdForgetReadResult())) { //this is commit response - if (CommitsInfly.empty()) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition - << " unwaited commit-response with cookie " << result.GetCookie() << "; waiting for nothing"); - return; - } - ui64 readId = CommitsInfly.front().first; - - if (result.GetCookie() != readId) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition - << " unwaited commit-response with cookie " << result.GetCookie() << "; waiting for " << readId); - return; - } - - Counters.Commits.Inc(); - - ui32 commitDurationMs = (ctx.Now() - CommitsInfly.front().second.StartTime).MilliSeconds(); - if (Counters.CommitLatency) { - Counters.CommitLatency.IncFor(commitDurationMs, 1); - } - - if (Counters.SLIBigLatency && commitDurationMs >= AppData(ctx)->PQConfig.GetCommitLatencyBigMs()) { - Counters.SLIBigLatency.Inc(); - } - - CommittedOffset = CommitsInfly.front().second.Offset; - ui64 startReadId = CommitsInfly.front().second.StartReadId; - ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(Partition.AssignId, startReadId, readId, CommittedOffset)); - - CommitsInfly.pop_front(); - - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition - << " commit done to position " << CommittedOffset << " endOffset " << EndOffset << " with cookie " << readId); - - PipeGeneration = 0; //reset tries counter - all ok - MakeCommit(ctx); + CommitDone(result.GetCookie(), ctx); return; } @@ -881,6 +892,45 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo PipeGeneration = 0; //reset tries counter - all ok } +void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { + if (CommitsInfly.empty()) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition + << " unwaited commit-response with cookie " << cookie << "; waiting for nothing"); + return; + } + ui64 readId = CommitsInfly.front().first; + + if (cookie != readId) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition + << " unwaited commit-response with cookie " << cookie << "; waiting for " << readId); + return; + } + + Counters.Commits.Inc(); + + ui32 commitDurationMs = (ctx.Now() - CommitsInfly.front().second.StartTime).MilliSeconds(); + if (Counters.CommitLatency) { + Counters.CommitLatency.IncFor(commitDurationMs, 1); + } + + if (Counters.SLIBigLatency && commitDurationMs >= AppData(ctx)->PQConfig.GetCommitLatencyBigMs()) { + Counters.SLIBigLatency.Inc(); + } + + CommittedOffset = CommitsInfly.front().second.Offset; + ui64 startReadId = CommitsInfly.front().second.StartReadId; + ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(Partition.AssignId, startReadId, readId, CommittedOffset)); + + Kqps.erase(CommitsInfly.front().first); // savnik то че надо удаляю? + CommitsInfly.pop_front(); + + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition + << " commit done to position " << CommittedOffset << " endOffset " << EndOffset << " with cookie " << readId); + + PipeGeneration = 0; //reset tries counter - all ok + MakeCommit(ctx); +} + void TPartitionActor::SendPartitionReady(const TActorContext& ctx) { LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition << " ready for read with readOffset " << ReadOffset << " endOffset " << EndOffset); @@ -916,13 +966,10 @@ void TPartitionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const RestartPipe(ctx, TStringBuilder() << "pipe to tablet is dead " << ev->Get()->TabletId, NPersQueue::NErrorCode::TABLET_PIPE_DISCONNECTED); } - - void TPartitionActor::Handle(TEvPQProxy::TEvGetStatus::TPtr&, const TActorContext& ctx) { ctx.Send(ParentId, new TEvPQProxy::TEvPartitionStatus(Partition, CommittedOffset, EndOffset, WriteTimestampEstimateMs, NodeId, TabletGeneration, false)); } - void TPartitionActor::Handle(TEvPQProxy::TEvLockPartition::TPtr& ev, const TActorContext& ctx) { ClientReadOffset = ev->Get()->ReadOffset; ClientCommitOffset = ev->Get()->CommitOffset; @@ -996,7 +1043,7 @@ void TPartitionActor::InitStartReading(const TActorContext& ctx) { return; } Y_ABORT_UNLESS(CommitsInfly.empty()); - CommitsInfly.push_back(std::pair(Max(), {Max(), ClientCommitOffset.GetOrElse(0), ctx.Now()})); + CommitsInfly.emplace_back(Max(), TCommitInfo{Max(), ClientCommitOffset.GetOrElse(0), ctx.Now()}); if (Counters.SLITotal) Counters.SLITotal.Inc(); if (PipeClient) //pipe will be recreated soon diff --git a/ydb/services/persqueue_v1/actors/partition_actor.h b/ydb/services/persqueue_v1/actors/partition_actor.h index 1cbbc5bcddf4..3a6b2d73179c 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.h +++ b/ydb/services/persqueue_v1/actors/partition_actor.h @@ -1,14 +1,17 @@ #pragma once +#include #include "events.h" #include "partition_id.h" #include #include +#include "ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h" #include #include #include +#include #include #include @@ -73,7 +76,7 @@ class TPartitionActor : public NActors::TActorBootstrapped { const TString& session, const TPartitionId& partition, ui32 generation, ui32 step, const ui64 tabletID, const TTopicCounters& counters, const bool commitsDisabled, const TString& clientDC, bool rangesMode, const NPersQueue::TTopicConverterPtr& topic, bool directRead, - bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs); + bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs, std::set parents); ~TPartitionActor(); void Bootstrap(const NActors::TActorContext& ctx); @@ -100,6 +103,9 @@ class TPartitionActor : public NActors::TActorBootstrapped { HFunc(TEvTabletPipe::TEvClientConnected, Handle); HFunc(TEvPersQueue::TEvResponse, Handle); HFunc(TEvPersQueue::TEvHasDataInfoResponse, Handle); + + HFunc(NKqp::TEvKqp::TEvCreateSessionResponse, Handle); + HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle); default: break; }; @@ -123,6 +129,9 @@ class TPartitionActor : public NActors::TActorBootstrapped { void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, const NActors::TActorContext& ctx); + void Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const NActors::TActorContext& ctx); + void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx); + void HandlePoison(NActors::TEvents::TEvPoisonPill::TPtr& ev, const NActors::TActorContext& ctx); void HandleWakeup(const NActors::TActorContext& ctx); void DoWakeup(const NActors::TActorContext& ctx); @@ -142,11 +151,13 @@ class TPartitionActor : public NActors::TActorBootstrapped { void SendPublishDirectRead(const ui64 directReadId, const TActorContext& ctx); void SendForgetDirectRead(const ui64 directReadId, const TActorContext& ctx); void SendPartitionReady(const TActorContext& ctx); + void CommitDone(ui64 cookie, const TActorContext& ctx); NKikimrClient::TPersQueueRequest MakeCreateSessionRequest(bool initial) const; NKikimrClient::TPersQueueRequest MakeReadRequest(ui64 readOffset, ui64 lastOffset, ui64 maxCount, ui64 maxSize, ui64 maxTimeLagMs, ui64 readTimestampMs, ui64 directReadId) const; + private: const TActorId ParentId; const TString ClientId; @@ -167,6 +178,7 @@ class TPartitionActor : public NActors::TActorBootstrapped { ui64 ReadOffset; ui64 ClientReadOffset; TMaybe ClientCommitOffset; + bool ClientHasAnyCommits; bool ClientVerifyReadOffset; ui64 CommittedOffset; ui64 WriteTimestampEstimateMs; @@ -209,12 +221,16 @@ class TPartitionActor : public NActors::TActorBootstrapped { }; std::deque> CommitsInfly; //ReadId, Offset + std::unordered_map> Kqps; + + std::set Parents; TTopicCounters Counters; bool CommitsDisabled; ui64 CommitCookie; NPersQueue::TTopicConverterPtr Topic; + TString Database; bool DirectRead = false; diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.cpp b/ydb/services/persqueue_v1/actors/read_session_actor.cpp index 71c4b29852a8..466012af5365 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.cpp +++ b/ydb/services/persqueue_v1/actors/read_session_actor.cpp @@ -434,7 +434,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommitRange: for (const auto& [b, e] : ev->Get()->CommitInfo.Ranges) { if (!RangesMode) { - return CloseSession(PersQueue::ErrorCode::BAD_REQUEST, "commits ranges in cookies commit mode is prohibited", ctx); + return CloseSession(PersQueue::ErrorCode::BAD_REQUEST, "commits ranges in cookies commit mode is prohibited", ctx); // savnik есть другой мод? Надо там тоже чет поддержать? } if (b >= e || it->second.NextRanges.Intersects(b, e) || b < it->second.Offset) { @@ -1196,29 +1196,27 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartit auto& converter = converterIter->second; const auto name = converter->GetInternalName(); - { - auto it = Topics.find(name); - if (it == Topics.end() || (!ReadWithoutConsumer && it->second.PipeClient != ActorIdFromProto(record.GetPipeClient()))) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " ignored ev lock" - << ": path# " << name - << ", reason# " << "topic is unknown"); - return; - } + auto topicIt = Topics.find(name); + if (topicIt == Topics.end() || (!ReadWithoutConsumer && topicIt->second.PipeClient != ActorIdFromProto(record.GetPipeClient()))) { + LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " ignored ev lock" + << ": path# " << name + << ", reason# " << "topic is unknown"); + return; + } - auto& topic = it->second; + auto& topic = topicIt->second; - // TODO: counters - if (NumPartitionsFromTopic[name]++ == 0) { - if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { - SetupTopicCounters(converter, topic.CloudId, topic.DbId, topic.DbPath, topic.IsServerless, topic.FolderId); - } else { - SetupTopicCounters(converter); - } + // TODO: counters + if (NumPartitionsFromTopic[name]++ == 0) { + if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { + SetupTopicCounters(converter, topic.CloudId, topic.DbId, topic.DbPath, topic.IsServerless, topic.FolderId); + } else { + SetupTopicCounters(converter); } - - topic.Partitions.emplace(record.GetPartition(), NGRpcProxy::TPartitionInfo{record.GetTabletId()}); } + topic.Partitions.emplace(record.GetPartition(), NGRpcProxy::TPartitionInfo{record.GetTabletId()}); + // TODO: counters auto it = TopicCounters.find(name); Y_ABORT_UNLESS(it != TopicCounters.end()); @@ -1234,10 +1232,10 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartit return CloseSession(PersQueue::ErrorCode::ERROR, error, ctx); } - const TActorId actorId = ctx.Register(new TPartitionActor( + const TActorId actorId = ctx.Register(new TPartitionActor( // savnik empty topic ctx.SelfID, ClientId, ClientPath, Cookie, Session, partitionId, record.GetGeneration(), record.GetStep(), record.GetTabletId(), it->second, CommitsDisabled, ClientDC, RangesMode, - converterIter->second, DirectRead, UseMigrationProtocol, maxLag, readTimestampMs)); + converterIter->second, DirectRead, UseMigrationProtocol, maxLag, readTimestampMs, topic.PartitionGraph->GetPartition(partitionId.Partition)->HierarhicalParents)); if (SessionsActive) { PartsPerSession.DecFor(Partitions.size(), 1); @@ -1286,6 +1284,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionSta it->second.LockSent = true; it->second.Offset = ev->Get()->Offset; + it->second.ConsumerHasAnyCommits = ev->Get()->ClientHasAnyCommits; if constexpr (UseMigrationProtocol) { result.mutable_assigned()->mutable_topic()->set_path(it->second.Topic->GetFederationPath()); diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.h b/ydb/services/persqueue_v1/actors/read_session_actor.h index 2f0424aca785..134d26e5f0a3 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.h +++ b/ydb/services/persqueue_v1/actors/read_session_actor.h @@ -45,6 +45,7 @@ struct TPartitionActorInfo { TSet NextCommits; TDisjointIntervalTree NextRanges; ui64 Offset; + bool ConsumerHasAnyCommits; TInstant AssignTimestamp; From 1b279d8cacdc418cb542a4731796e8b9f020928b Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 4 Dec 2024 09:29:31 +0000 Subject: [PATCH 08/29] work --- .../kqp/session_actor/kqp_query_state.cpp | 2 +- ydb/core/kqp/topics/kqp_topics.cpp | 34 ++++++--- ydb/core/kqp/topics/kqp_topics.h | 14 +++- ydb/core/persqueue/events/internal.h | 3 +- ydb/core/persqueue/partition.cpp | 43 ++++++----- ydb/core/persqueue/partition_read.cpp | 5 +- .../persqueue/partition_scale_manager.cpp | 2 +- .../persqueue/partition_sourcemanager.cpp | 2 +- ydb/core/persqueue/pq_impl.cpp | 3 +- .../read_balancer__balancing_app.cpp | 4 +- ydb/core/persqueue/transaction.cpp | 8 +- ydb/core/persqueue/user_info.cpp | 4 +- ydb/core/persqueue/user_info.h | 2 +- ydb/core/persqueue/ut/partitiongraph_ut.cpp | 32 ++++---- ydb/core/persqueue/utils.cpp | 38 +++++----- ydb/core/persqueue/utils.h | 8 +- ydb/core/protos/kqp.proto | 1 + ydb/core/protos/pqconfig.proto | 1 + .../actors/commit_offset_actor.cpp | 22 +++--- ydb/services/persqueue_v1/actors/events.h | 9 +++ .../actors/kqp_commit_offset_helper.cpp | 3 +- .../actors/kqp_commit_offset_helper.h | 1 + .../persqueue_v1/actors/partition_actor.cpp | 30 +++++--- .../persqueue_v1/actors/partition_actor.h | 12 ++- .../actors/read_session_actor.cpp | 73 +++++++++++++++---- .../persqueue_v1/actors/read_session_actor.h | 3 + 26 files changed, 235 insertions(+), 124 deletions(-) diff --git a/ydb/core/kqp/session_actor/kqp_query_state.cpp b/ydb/core/kqp/session_actor/kqp_query_state.cpp index ea99c4d12e93..88804678c6ea 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.cpp +++ b/ydb/core/kqp/session_actor/kqp_query_state.cpp @@ -451,7 +451,7 @@ void TKqpQueryState::AddOffsetsToTransaction() { } else { for (auto& range : partition.partition_offsets()) { YQL_ENSURE(consumer.Defined()); - TopicOperations.AddOperation(path, partition.partition_id(), *consumer, range, partition.force_commit(), partition.kill_read_session(), partition.only_check_commited_to_finish()); + TopicOperations.AddOperation(path, partition.partition_id(), *consumer, range, partition.force_commit(), partition.kill_read_session(), partition.only_check_commited_to_finish(), partition.read_session_id()); } } } diff --git a/ydb/core/kqp/topics/kqp_topics.cpp b/ydb/core/kqp/topics/kqp_topics.cpp index 6bd734dcd07a..7dff2f08679c 100644 --- a/ydb/core/kqp/topics/kqp_topics.cpp +++ b/ydb/core/kqp/topics/kqp_topics.cpp @@ -56,15 +56,21 @@ bool TConsumerOperations::GetOnlyCheckCommitedToFinish() const return OnlyCheckCommitedToFinish_; } +TString TConsumerOperations::GetReadSessionId() const +{ + return ReadSessionId_; +} + void TConsumerOperations::AddOperation(const TString& consumer, const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, bool forceCommit, bool killReadSession, - bool onlyCheckCommitedToFinish) + bool onlyCheckCommitedToFinish, + const TString& readSessionId) { Y_ABORT_UNLESS(Consumer_.Empty() || Consumer_ == consumer); - AddOperationImpl(consumer, range.start(), range.end(), forceCommit, killReadSession, onlyCheckCommitedToFinish); + AddOperationImpl(consumer, range.start(), range.end(), forceCommit, killReadSession, onlyCheckCommitedToFinish, readSessionId); } void TConsumerOperations::Merge(const TConsumerOperations& rhs) @@ -72,8 +78,12 @@ void TConsumerOperations::Merge(const TConsumerOperations& rhs) Y_ABORT_UNLESS(rhs.Consumer_.Defined()); Y_ABORT_UNLESS(Consumer_.Empty() || Consumer_ == rhs.Consumer_); - for (auto& range : rhs.Offsets_) { - AddOperationImpl(*rhs.Consumer_, range.first, range.second, rhs.GetForceCommit(), rhs.GetKillReadSession(), rhs.GetOnlyCheckCommitedToFinish()); + if (!rhs.Offsets_.Empty()) { + for (auto& range : rhs.Offsets_) { + AddOperationImpl(*rhs.Consumer_, range.first, range.second, rhs.GetForceCommit(), rhs.GetKillReadSession(), rhs.GetOnlyCheckCommitedToFinish(), rhs.GetReadSessionId()); + } + } else { + AddOperationImpl(*rhs.Consumer_, 0, 0, rhs.GetForceCommit(), rhs.GetKillReadSession(), rhs.GetOnlyCheckCommitedToFinish(), rhs.GetReadSessionId()); } } @@ -82,7 +92,8 @@ void TConsumerOperations::AddOperationImpl(const TString& consumer, ui64 end, bool forceCommit, bool killReadSession, - bool onlyCheckCommitedToFinish) + bool onlyCheckCommitedToFinish, + const TString& readSessionId) { if (Offsets_.Intersects(begin, end)) { ythrow TOffsetsRangeIntersectExpection() << "offset ranges intersect"; @@ -99,6 +110,7 @@ void TConsumerOperations::AddOperationImpl(const TString& consumer, ForceCommit_ = forceCommit; KillReadSession_ = killReadSession; OnlyCheckCommitedToFinish_ = onlyCheckCommitedToFinish; + ReadSessionId_ = readSessionId; } // @@ -116,7 +128,8 @@ void TTopicPartitionOperations::AddOperation(const TString& topic, const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, bool forceCommit, bool killReadSession, - bool onlyCheckCommitedToFinish) + bool onlyCheckCommitedToFinish, + const TString& readSessionId) { Y_ABORT_UNLESS(Topic_.Empty() || Topic_ == topic); Y_ABORT_UNLESS(Partition_.Empty() || Partition_ == partition); @@ -126,7 +139,7 @@ void TTopicPartitionOperations::AddOperation(const TString& topic, Partition_ = partition; } - Operations_[consumer].AddOperation(consumer, range, forceCommit, killReadSession, onlyCheckCommitedToFinish); + Operations_[consumer].AddOperation(consumer, range, forceCommit, killReadSession, onlyCheckCommitedToFinish, readSessionId); } void TTopicPartitionOperations::AddOperation(const TString& topic, ui32 partition, @@ -163,6 +176,7 @@ void TTopicPartitionOperations::BuildTopicTxs(TTopicOperationTransactions& txs) o->SetKillReadSession(operations.GetKillReadSession()); o->SetForceCommit(operations.GetForceCommit()); o->SetOnlyCheckCommitedToFinish(operations.GetOnlyCheckCommitedToFinish()); + o->SetReadSessionId(operations.GetReadSessionId()); } if (HasWriteOperations_) { @@ -303,7 +317,8 @@ void TTopicOperations::AddOperation(const TString& topic, const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, bool forceCommit, bool killReadSession, - bool onlyCheckCommitedToFinish + bool onlyCheckCommitedToFinish, + const TString& readSessionId ) { TTopicPartition key{topic, partition}; @@ -313,7 +328,8 @@ void TTopicOperations::AddOperation(const TString& topic, range, forceCommit, killReadSession, - onlyCheckCommitedToFinish); + onlyCheckCommitedToFinish, + readSessionId); HasReadOperations_ = true; } diff --git a/ydb/core/kqp/topics/kqp_topics.h b/ydb/core/kqp/topics/kqp_topics.h index d17282d149ed..8eff402d1aa6 100644 --- a/ydb/core/kqp/topics/kqp_topics.h +++ b/ydb/core/kqp/topics/kqp_topics.h @@ -35,12 +35,14 @@ class TConsumerOperations { bool GetForceCommit() const; bool GetKillReadSession() const; bool GetOnlyCheckCommitedToFinish() const; + TString GetReadSessionId() const; void AddOperation(const TString& consumer, const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, bool forceCommit = false, bool killReadSession = false, - bool onlyCheckCommitedToFinish = false); + bool onlyCheckCommitedToFinish = false, + const TString& readSessionId = {}); void Merge(const TConsumerOperations& rhs); @@ -50,13 +52,15 @@ class TConsumerOperations { ui64 end, bool forceCommit = false, bool killReadSession = false, - bool onlyCheckCommitedToFinish = false); + bool onlyCheckCommitedToFinish = false, + const TString& readSessionId = {}); TMaybe Consumer_; TDisjointIntervalTree Offsets_; bool ForceCommit_ = false; bool KillReadSession_ = false; bool OnlyCheckCommitedToFinish_ = false; + TString ReadSessionId_; }; struct TTopicOperationTransaction { @@ -76,7 +80,8 @@ class TTopicPartitionOperations { const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, bool forceCommit = false, bool killReadSession = false, - bool onlyCheckCommitedToFinish = false); + bool onlyCheckCommitedToFinish = false, + const TString& readSessionId = {}); void AddOperation(const TString& topic, ui32 partition, TMaybe supportivePartition); @@ -132,7 +137,8 @@ class TTopicOperations { const NKikimrKqp::TTopicOperationsRequest_TopicOffsets_PartitionOffsets_OffsetsRange& range, bool forceCommit, bool killReadSession, - bool onlyCheckCommitedToFinish); + bool onlyCheckCommitedToFinish, + const TString& readSessionId); void AddOperation(const TString& topic, ui32 partition, TMaybe supportivePartition); diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index 574484e8886d..5f8c15036a28 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -811,7 +811,7 @@ struct TEvPQ { { } - void AddOperation(TString consumer, ui64 begin, ui64 end, bool forceCommit = false, bool killReadSession = false, bool onlyCheckCommitedToFinish = false) { + void AddOperation(TString consumer, ui64 begin, ui64 end, bool forceCommit = false, bool killReadSession = false, bool onlyCheckCommitedToFinish = false, TString readSessionId = {}) { NKikimrPQ::TPartitionOperation operation; operation.SetCommitOffsetsBegin(begin); operation.SetCommitOffsetsEnd(end); @@ -819,6 +819,7 @@ struct TEvPQ { operation.SetForceCommit(forceCommit); operation.SetKillReadSession(killReadSession); operation.SetOnlyCheckCommitedToFinish(onlyCheckCommitedToFinish); + operation.SetReadSessionId(readSessionId); Operations.push_back(std::move(operation)); } diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 4c6cbc3dad6f..dc0dbc5bf7d1 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -2296,7 +2296,7 @@ bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, return true; } -TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPredicate& tx, TMaybe& predicate) +TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPredicate& tx, TMaybe& predicateOut) { if (tx.ForcePredicateFalse) { predicate = false; @@ -2304,7 +2304,7 @@ TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPr } THashSet consumers; - bool ok = true; + bool result = true; for (auto& operation : tx.Operations) { const TString& consumer = operation.GetConsumer(); if (TxAffectedConsumers.contains(consumer)) { @@ -2317,49 +2317,55 @@ TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPr if (AffectedUsers.contains(consumer) && !GetPendingUserIfExists(consumer)) { PQ_LOG_D("Partition " << Partition << " Consumer '" << consumer << "' has been removed"); - ok = false; + result = false; break; } if (!UsersInfoStorage->GetIfExists(consumer)) { PQ_LOG_D("Partition " << Partition << " Unknown consumer '" << consumer << "'"); - ok = false; + result = false; break; } - bool isAffectedConsumer = AffectedUsers.contains(consumer); // savnik check + bool isAffectedConsumer = AffectedUsers.contains(consumer); TUserInfoBase& userInfo = GetOrCreatePendingUser(consumer); - if (operation.HasOnlyCheckCommitedToFinish() && operation.GetOnlyCheckCommitedToFinish() && !IsActive()) { + if (!operation.GetReadSessionId().Empty() && operation.GetReadSessionId() != userInfo.Session) { + PQ_LOG_D("Partition " << Partition << + " Consumer '" << consumer << "'" << + " Bad request (session already dead) " << + " SessionId '" << operation.GetReadSessionId() << "'"); + result = false; + } else if (operation.GetOnlyCheckCommitedToFinish() && !IsActive()) { if (static_cast(userInfo.Offset) != EndOffset) { - ok = false; + result = false; } } else { - if (!operation.GetForceCommit() && operation.GetCommitOffsetsBegin() > operation.GetCommitOffsetsEnd()) { // savnik check default + if (!operation.GetForceCommit() && operation.GetCommitOffsetsBegin() > operation.GetCommitOffsetsEnd()) { PQ_LOG_D("Partition " << Partition << " Consumer '" << consumer << "'" << " Bad request (invalid range) " << " Begin " << operation.GetCommitOffsetsBegin() << " End " << operation.GetCommitOffsetsEnd()); - ok = false; + result = false; } else if (!operation.GetForceCommit() && userInfo.Offset != (i64)operation.GetCommitOffsetsBegin()) { PQ_LOG_D("Partition " << Partition << " Consumer '" << consumer << "'" << " Bad request (gap) " << " Offset " << userInfo.Offset << " Begin " << operation.GetCommitOffsetsBegin()); - ok = false; + result = false; } else if (!operation.GetForceCommit() && operation.GetCommitOffsetsEnd() > EndOffset) { PQ_LOG_D("Partition " << Partition << " Consumer '" << consumer << "'" << " Bad request (behind the last offset) " << " EndOffset " << EndOffset << " End " << operation.GetCommitOffsetsEnd()); - ok = false; + result = false; } - if (!ok) { + if (!result) { if (!isAffectedConsumer) { AffectedUsers.erase(consumer); } @@ -2368,10 +2374,10 @@ TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPr consumers.insert(consumer); } } - if (ok) { - TxAffectedConsumers.insert(consumers.begin(), consumers.end()); // savnik check + if (result) { + TxAffectedConsumers.insert(consumers.begin(), consumers.end()); } - predicate = ok; + predicateOut = result; return EProcessResult::Continue; } @@ -2518,9 +2524,13 @@ void TPartition::CommitTransaction(TSimpleSharedPtr& t) Y_ABORT_UNLESS(t->Predicate.Defined() && *t->Predicate); for (auto& operation : t->Tx->Operations) { + if (operation.GetOnlyCheckCommitedToFinish()) { + continue; + } + TUserInfoBase& userInfo = GetOrCreatePendingUser(operation.GetConsumer()); - if (!operation.HasForceCommit() || !operation.GetForceCommit()) { // savnik check default + if (!operation.GetForceCommit()) { Y_ABORT_UNLESS(userInfo.Offset == (i64)operation.GetCommitOffsetsBegin()); } @@ -2534,7 +2544,6 @@ void TPartition::CommitTransaction(TSimpleSharedPtr& t) userInfo.AnyCommits = true; userInfo.Offset = operation.GetCommitOffsetsEnd(); } - if (operation.GetKillReadSession()) { userInfo.Session = ""; userInfo.PartitionSessionId = 0; diff --git a/ydb/core/persqueue/partition_read.cpp b/ydb/core/persqueue/partition_read.cpp index ab94848f6dbb..3fb9d083233f 100644 --- a/ydb/core/persqueue/partition_read.cpp +++ b/ydb/core/persqueue/partition_read.cpp @@ -124,10 +124,10 @@ TAutoPtr TPartition::MakeHasDataInfoRespon ui32 partitionId = Partition.OriginalPartitionId; auto* node = PartitionGraph.GetPartition(partitionId); - for (auto* child : node->Children) { + for (auto* child : node->DirectChildren) { res->Record.AddChildPartitionIds(child->Id); - for (auto* p : child->Parents) { + for (auto* p : child->DirectParents) { if (p->Id != partitionId) { res->Record.AddAdjacentPartitionIds(p->Id); } @@ -776,7 +776,6 @@ void TPartition::Handle(TEvPQ::TEvRead::TPtr& ev, const TActorContext& ctx) { const TString& user = read->ClientId; auto& userInfo = UsersInfoStorage->GetOrCreate(user, ctx); - if (!read->SessionId.empty() && !userInfo.NoConsumer) { if (userInfo.Session != read->SessionId) { TabletCounters.Cumulative()[COUNTER_PQ_READ_ERROR_NO_SESSION].Increment(1); diff --git a/ydb/core/persqueue/partition_scale_manager.cpp b/ydb/core/persqueue/partition_scale_manager.cpp index 77f16ab019ea..3360752d7653 100644 --- a/ydb/core/persqueue/partition_scale_manager.cpp +++ b/ydb/core/persqueue/partition_scale_manager.cpp @@ -74,7 +74,7 @@ std::pair, std::vector> TPartition auto partitionId = PartitionsToSplit.begin(); while (allowedSplitsCount > 0 && partitionId != PartitionsToSplit.end()) { auto* node = PartitionGraph.GetPartition(*partitionId); - if (node->Children.empty()) { + if (node->DirectChildren.empty()) { auto from = node->From; auto to = node->To; auto mid = MiddleOf(from, to); diff --git a/ydb/core/persqueue/partition_sourcemanager.cpp b/ydb/core/persqueue/partition_sourcemanager.cpp index c9214300384a..c2b6d5a0edd4 100644 --- a/ydb/core/persqueue/partition_sourcemanager.cpp +++ b/ydb/core/persqueue/partition_sourcemanager.cpp @@ -50,7 +50,7 @@ TSourceIdStorage& TPartitionSourceManager::GetSourceIdStorage() const { bool TPartitionSourceManager::HasParents() const { auto node = GetPartitionNode(); - return node && !node->Parents.empty(); + return node && !node->DirectParents.empty(); } diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 55ec6f4dbc5f..baae65944a6a 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -3979,7 +3979,8 @@ void TPersQueue::SendEvTxCalcPredicateToPartitions(const TActorContext& ctx, operation.GetCommitOffsetsEnd(), operation.HasForceCommit() ? operation.GetForceCommit() : false, operation.HasKillReadSession() ? operation.GetKillReadSession() : false, - operation.HasOnlyCheckCommitedToFinish() ? operation.GetOnlyCheckCommitedToFinish() : false); + operation.HasOnlyCheckCommitedToFinish() ? operation.GetOnlyCheckCommitedToFinish() : false, + operation.HasReadSessionId() ? operation.GetReadSessionId() : ""); } } diff --git a/ydb/core/persqueue/read_balancer__balancing_app.cpp b/ydb/core/persqueue/read_balancer__balancing_app.cpp index fd565135d162..71c7b72150fb 100644 --- a/ydb/core/persqueue/read_balancer__balancing_app.cpp +++ b/ydb/core/persqueue/read_balancer__balancing_app.cpp @@ -76,7 +76,7 @@ void TBalancer::RenderApp(NApp::TNavigationBar& __navigationBar) const { for (auto& [partitionId, partition] : consumer->Partitions) { const auto* family = consumer->FindFamily(partitionId); const auto* node = consumer->GetPartitionGraph().GetPartition(partitionId); - TString style = node && node->Children.empty() ? "text-success" : "text-muted"; + TString style = node && node->DirectChildren.empty() ? "text-success" : "text-muted"; auto* partitionInfo = GetPartitionInfo(partitionId); TABLER() { @@ -110,7 +110,7 @@ void TBalancer::RenderApp(NApp::TNavigationBar& __navigationBar) const { } TABLED() { if (node) { - for (auto* parent : node->Parents) { + for (auto* parent : node->DirectParents) { HREF("#" + partitionAnchor(parent->Id)) { __stream << parent->Id; } __stream << ", "; } diff --git a/ydb/core/persqueue/transaction.cpp b/ydb/core/persqueue/transaction.cpp index 9042afed69bd..25d5ee46f01f 100644 --- a/ydb/core/persqueue/transaction.cpp +++ b/ydb/core/persqueue/transaction.cpp @@ -185,16 +185,16 @@ void TDistributedTransaction::OnProposeTransaction(const NKikimrPQ::TConfigTrans continue; } - if (node->Children.empty()) { - for (const auto* r : node->Parents) { + if (node->DirectChildren.empty()) { + for (const auto* r : node->DirectParents) { if (extractTabletId != r->TabletId) { PredicatesReceived[r->TabletId].SetTabletId(r->TabletId); } } } - for (const auto* r : node->Children) { - if (r->Children.empty()) { + for (const auto* r : node->DirectChildren) { + if (r->DirectChildren.empty()) { if (extractTabletId != r->TabletId) { PredicateRecipients[r->TabletId] = false; } diff --git a/ydb/core/persqueue/user_info.cpp b/ydb/core/persqueue/user_info.cpp index 0d75bbabc876..9b8560576756 100644 --- a/ydb/core/persqueue/user_info.cpp +++ b/ydb/core/persqueue/user_info.cpp @@ -95,7 +95,7 @@ void TUsersInfoStorage::ParseDeprecated(const TString& key, const TString& data, Y_ABORT_UNLESS(offset <= (ui64)Max(), "Offset is too big: %" PRIu64, offset); if (!userInfo) { - Create(ctx, user, 0, false, session, 0, gen, step, static_cast(offset), 0, TInstant::Zero(), {}, false); //savnik check anyCommits + Create(ctx, user, 0, false, session, 0, gen, step, static_cast(offset), 0, TInstant::Zero(), {}, false); } else { userInfo->Session = session; userInfo->Generation = gen; @@ -123,7 +123,7 @@ void TUsersInfoStorage::Parse(const TString& key, const TString& data, const TAc Create( ctx, user, userData.GetReadRuleGeneration(), false, userData.GetSession(), userData.GetPartitionSessionId(), userData.GetGeneration(), userData.GetStep(), offset, - userData.GetOffsetRewindSum(), TInstant::Zero(), {}, userData.GetAnyCommits() // savnik + userData.GetOffsetRewindSum(), TInstant::Zero(), {}, userData.GetAnyCommits() ); } else { userInfo->Session = userData.GetSession(); diff --git a/ydb/core/persqueue/user_info.h b/ydb/core/persqueue/user_info.h index a9135a096309..5aaad167d9f0 100644 --- a/ydb/core/persqueue/user_info.h +++ b/ydb/core/persqueue/user_info.h @@ -45,7 +45,7 @@ struct TUserInfoBase { ui32 Generation = 0; ui32 Step = 0; i64 Offset = 0; - bool AnyCommits = false; // savnik что-то еще надо сделать, чтобы эта штука персистилась? + bool AnyCommits = false; bool Important = false; TInstant ReadFromTimestamp; diff --git a/ydb/core/persqueue/ut/partitiongraph_ut.cpp b/ydb/core/persqueue/ut/partitiongraph_ut.cpp index eb8d1cff01cf..43603f312ef8 100644 --- a/ydb/core/persqueue/ut/partitiongraph_ut.cpp +++ b/ydb/core/persqueue/ut/partitiongraph_ut.cpp @@ -59,22 +59,22 @@ Y_UNIT_TEST_SUITE(TPartitionGraphTest) { UNIT_ASSERT(n4); UNIT_ASSERT(n5); - UNIT_ASSERT_VALUES_EQUAL(n0->Parents.size(), 0); - UNIT_ASSERT_VALUES_EQUAL(n0->Children.size(), 0); - UNIT_ASSERT_VALUES_EQUAL(n0->HierarhicalParents.size(), 0); - - UNIT_ASSERT_VALUES_EQUAL(n1->Parents.size(), 0); - UNIT_ASSERT_VALUES_EQUAL(n1->Children.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(n1->HierarhicalParents.size(), 0); - - UNIT_ASSERT_VALUES_EQUAL(n5->Parents.size(), 2); - UNIT_ASSERT_VALUES_EQUAL(n5->Children.size(), 0u); - UNIT_ASSERT_VALUES_EQUAL(n5->HierarhicalParents.size(), 4); - UNIT_ASSERT(std::find(n5->HierarhicalParents.cbegin(), n5->HierarhicalParents.cend(), n0) == n5->HierarhicalParents.end()); - UNIT_ASSERT(std::find(n5->HierarhicalParents.cbegin(), n5->HierarhicalParents.cend(), n1) != n5->HierarhicalParents.end()); - UNIT_ASSERT(std::find(n5->HierarhicalParents.cbegin(), n5->HierarhicalParents.cend(), n2) != n5->HierarhicalParents.end()); - UNIT_ASSERT(std::find(n5->HierarhicalParents.cbegin(), n5->HierarhicalParents.cend(), n3) != n5->HierarhicalParents.end()); - UNIT_ASSERT(std::find(n5->HierarhicalParents.cbegin(), n5->HierarhicalParents.cend(), n4) != n5->HierarhicalParents.end()); + UNIT_ASSERT_VALUES_EQUAL(n0->DirectParents.size(), 0); + UNIT_ASSERT_VALUES_EQUAL(n0->DirectChildren.size(), 0); + UNIT_ASSERT_VALUES_EQUAL(n0->AllParents.size(), 0); + + UNIT_ASSERT_VALUES_EQUAL(n1->DirectParents.size(), 0); + UNIT_ASSERT_VALUES_EQUAL(n1->DirectChildren.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(n1->AllParents.size(), 0); + + UNIT_ASSERT_VALUES_EQUAL(n5->DirectParents.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(n5->DirectChildren.size(), 0u); + UNIT_ASSERT_VALUES_EQUAL(n5->AllParents.size(), 4); + UNIT_ASSERT(std::find(n5->AllParents.cbegin(), n5->AllParents.cend(), n0) == n5->AllParents.end()); + UNIT_ASSERT(std::find(n5->AllParents.cbegin(), n5->AllParents.cend(), n1) != n5->AllParents.end()); + UNIT_ASSERT(std::find(n5->AllParents.cbegin(), n5->AllParents.cend(), n2) != n5->AllParents.end()); + UNIT_ASSERT(std::find(n5->AllParents.cbegin(), n5->AllParents.cend(), n3) != n5->AllParents.end()); + UNIT_ASSERT(std::find(n5->AllParents.cbegin(), n5->AllParents.cend(), n4) != n5->AllParents.end()); { std::set traversedNodes; diff --git a/ydb/core/persqueue/utils.cpp b/ydb/core/persqueue/utils.cpp index f2743180665c..ccdc3ffe95c2 100644 --- a/ydb/core/persqueue/utils.cpp +++ b/ydb/core/persqueue/utils.cpp @@ -170,10 +170,10 @@ std::set TPartitionGraph::GetActiveChildren(ui32 id) const { const auto* n = queue.front(); queue.pop_front(); - if (n->Children.empty()) { + if (n->DirectChildren.empty()) { result.emplace(n->Id); } else { - queue.insert(queue.end(), n->Children.begin(), n->Children.end()); + queue.insert(queue.end(), n->DirectChildren.begin(), n->DirectChildren.end()); } } @@ -186,7 +186,7 @@ void Travers0(std::deque& queue, const std::functi queue.pop_front(); if (func(node->Id)) { - queue.insert(queue.end(), node->Children.begin(), node->Children.end()); + queue.insert(queue.end(), node->DirectChildren.begin(), node->DirectChildren.end()); } } } @@ -203,7 +203,7 @@ void TPartitionGraph::Travers(const std::function& func) const { continue; } - queue.insert(queue.end(), n.Children.begin(), n.Children.end()); + queue.insert(queue.end(), n.DirectChildren.begin(), n.DirectChildren.end()); } Travers0(queue, func); @@ -220,7 +220,7 @@ void TPartitionGraph::Travers(ui32 id, const std::function& func } std::deque queue; - queue.insert(queue.end(), n->Children.begin(), n->Children.end()); + queue.insert(queue.end(), n->DirectChildren.begin(), n->DirectChildren.end()); Travers0(queue, func); } @@ -270,14 +270,14 @@ std::unordered_map BuildGraph(const TCollection& pa for (const auto& p : partitions) { auto& node = result[GetPartitionId(p)]; - node.Children.reserve(p.ChildPartitionIdsSize()); + node.DirectChildren.reserve(p.ChildPartitionIdsSize()); for (auto id : p.GetChildPartitionIds()) { - node.Children.push_back(&result[id]); + node.DirectChildren.push_back(&result[id]); } - node.Parents.reserve(p.ParentPartitionIdsSize()); + node.DirectParents.reserve(p.ParentPartitionIdsSize()); for (auto id : p.GetParentPartitionIds()) { - node.Parents.push_back(&result[id]); + node.DirectParents.push_back(&result[id]); } if (p.GetParentPartitionIds().empty()) { @@ -290,19 +290,19 @@ std::unordered_map BuildGraph(const TCollection& pa queue.pop_front(); bool allCompleted = true; - for (auto* c : n->Parents) { - if (c->HierarhicalParents.empty() && !c->Parents.empty()) { + for (auto* c : n->DirectParents) { + if (c->AllParents.empty() && !c->DirectParents.empty()) { allCompleted = false; break; } } if (allCompleted) { - for (auto* c : n->Parents) { - n->HierarhicalParents.insert(c->HierarhicalParents.begin(), c->HierarhicalParents.end()); - n->HierarhicalParents.insert(c); + for (auto* c : n->DirectParents) { + n->AllParents.insert(c->AllParents.begin(), c->AllParents.end()); + n->AllParents.insert(c); } - queue.insert(queue.end(), n->Children.begin(), n->Children.end()); + queue.insert(queue.end(), n->DirectChildren.begin(), n->DirectChildren.end()); } } @@ -313,8 +313,8 @@ std::unordered_map BuildGraph(const TCollection& pa auto* current = queue.front(); queue.pop_front(); - for (auto* child : current->Children) { - if (node.HierarhicalChildren.insert(child).second) { + for (auto* child : current->DirectChildren) { + if (node.AllChildren.insert(child).second) { queue.push_back(child); } } @@ -332,11 +332,11 @@ TPartitionGraph::Node::Node(ui32 id, ui64 tabletId, const TString& from, const T } bool TPartitionGraph::Node::IsRoot() const { - return Parents.empty(); + return DirectParents.empty(); } bool TPartitionGraph::Node::IsParent(ui32 partitionId) const { - return AnyOf(Parents, [=](const auto& p) { + return AnyOf(DirectParents, [=](const auto& p) { return p->Id == partitionId; }); } diff --git a/ydb/core/persqueue/utils.h b/ydb/core/persqueue/utils.h index 6b7b5157a8c7..43eb519f85ef 100644 --- a/ydb/core/persqueue/utils.h +++ b/ydb/core/persqueue/utils.h @@ -44,13 +44,13 @@ class TPartitionGraph { TString To; // Direct parents of this node - std::vector Parents; + std::vector DirectParents; // Direct children of this node - std::vector Children; + std::vector DirectChildren; // All parents include parents of parents and so on - std::set HierarhicalParents; // savnik rename to Ancestors? + std::set AllParents; // All children include children of children and so on - std::set HierarhicalChildren; + std::set AllChildren; bool IsRoot() const; bool IsParent(ui32 partitionId) const; diff --git a/ydb/core/protos/kqp.proto b/ydb/core/protos/kqp.proto index 397b10e10085..b858719b5a47 100644 --- a/ydb/core/protos/kqp.proto +++ b/ydb/core/protos/kqp.proto @@ -89,6 +89,7 @@ message TTopicOperationsRequest { optional bool force_commit = 3; optional bool kill_read_session = 4; optional bool only_check_commited_to_finish = 5; + optional string read_session_id = 6; message OffsetsRange { optional int64 start = 1; diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index b4f734ef26a1..2f27923c359e 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -960,6 +960,7 @@ message TPartitionOperation { optional bool ForceCommit = 7; optional bool KillReadSession = 8; optional bool OnlyCheckCommitedToFinish = 9; + optional string ReadSessionId = 10; }; message TWriteId { diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index 3be20e7d346a..ba771940442f 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -105,22 +105,22 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc auto* partitionNode = topicInitInfo.PartitionGraph->GetPartition(commitRequest->partition_id()); - if (partitionNode->HierarhicalParents.size() == 0 && partitionNode->Children.size() == 0) { + if (partitionNode->AllParents.size() == 0 && partitionNode->DirectChildren.size() == 0) { SendCommit(topicInitInfo, commitRequest, ctx); } else { std::vector commits; - for (auto& parent: partitionNode->HierarhicalParents) { + for (auto& parent: partitionNode->AllParents) { TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); } - for (auto& child: partitionNode->HierarhicalChildren) { - TKqpHelper::TCommitInfo commit {.PartitionId = child->Id, .Offset = 0}; + for (auto& child: partitionNode->AllChildren) { + TKqpHelper::TCommitInfo commit {.PartitionId = child->Id, .Offset = 0, .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); } - TKqpHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = Max(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; + TKqpHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->offset(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); // savnik if empty database? @@ -132,16 +132,19 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const NActors::TActorContext& ctx) { if (!Kqp->Handle(ev, ctx)) { - AnswerError("empty list of topics", PersQueue::ErrorCode::UNKNOWN_TOPIC, ctx); // savnik + AnswerError(ev->Get()->Record.GetError(), PersQueue::ErrorCode::ERROR, ctx); } } void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { auto& record = ev->Get()->Record; - if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { // savnik finish this part! - Ydb::Topic::CommitOffsetResult result; + if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "strict CommitOffset failed. Kqp error: " << ev->Get()->Record); + + Ydb::Topic::CommitOffsetResult result; // savnik: how to return exception? Request().SendResult(result, record.GetYdbStatus()); Die(ctx); + return; } auto step = Kqp->Handle(ev, ctx); @@ -150,6 +153,7 @@ void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const Ydb::Topic::CommitOffsetResult result; Request().SendResult(result, Ydb::StatusIds::SUCCESS); Die(ctx); + return; } } @@ -183,7 +187,7 @@ void TCommitOffsetActor::SendCommit(const TTopicInitInfo& topic, const Ydb::Topi .DoFirstRetryInstantly = true }; - PipeClient = ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); // savnik close only in that case + PipeClient = ctx.Register(NTabletPipe::CreateClient(ctx.SelfID, tabletId, clientConfig)); NKikimrClient::TPersQueueRequest request; request.MutablePartitionRequest()->SetTopic(topic.TopicNameConverter->GetPrimaryPath()); diff --git a/ydb/services/persqueue_v1/actors/events.h b/ydb/services/persqueue_v1/actors/events.h index 5aadfd485724..5705be99a0da 100644 --- a/ydb/services/persqueue_v1/actors/events.h +++ b/ydb/services/persqueue_v1/actors/events.h @@ -89,6 +89,7 @@ struct TEvPQProxy { EvReadingStarted, EvReadingFinished, EvAlterTopicResponse, + EvParentCommitedToFinish, EvEnd }; @@ -420,6 +421,14 @@ struct TEvPQProxy { ui64 Offset; }; + struct TEvParentCommitedToFinish : public NActors::TEventLocal { + explicit TEvParentCommitedToFinish(ui64 parentPartitionId) + : ParentPartitionId(parentPartitionId) + { } + + ui64 ParentPartitionId; + }; + struct TEvReleasePartition : public NActors::TEventLocal { TEvReleasePartition() { } diff --git a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp index dc4cd1077740..5f01010525ed 100644 --- a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp +++ b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp @@ -94,8 +94,6 @@ void TKqpHelper::SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const NAc offsets->Record.MutableRequest()->MutableTxControl()->set_tx_id(TxId); offsets->Record.MutableRequest()->MutableTopicOperations()->SetConsumer(Consumer); - // savnik need set something else? - auto* topic = offsets->Record.MutableRequest()->MutableTopicOperations()->AddTopics(); topic->set_path(Path); @@ -105,6 +103,7 @@ void TKqpHelper::SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const NAc partition->set_force_commit(true); partition->set_kill_read_session(commit.KillReadSession); partition->set_only_check_commited_to_finish(commit.OnlyCheckCommitedToFinish); + partition->set_read_session_id(commit.ReadSessionId); auto* offset = partition->add_partition_offsets(); offset->set_end(commit.Offset); } diff --git a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h index 0b16810a5128..09640c01581a 100644 --- a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h +++ b/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h @@ -23,6 +23,7 @@ class TKqpHelper { i64 Offset; bool KillReadSession; bool OnlyCheckCommitedToFinish; + TString ReadSessionId; }; TKqpHelper(TString database, TString consumer, TString path, std::vector commits, ui64 cookie = 0); diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index 122a1711fc29..f325943f74d1 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -26,7 +26,8 @@ TPartitionActor::TPartitionActor( const TString& session, const TPartitionId& partition, const ui32 generation, const ui32 step, const ui64 tabletID, const TTopicCounters& counters, bool commitsDisabled, const TString& clientDC, bool rangesMode, const NPersQueue::TTopicConverterPtr& topic, - bool directRead, bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs, std::set parents + bool directRead, bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs, std::set parents, + std::unordered_set notCommitedToFinishParents ) : ParentId(parentId) , ClientId(clientId) @@ -74,22 +75,21 @@ TPartitionActor::TPartitionActor( , UseMigrationProtocol(useMigrationProtocol) , FirstRead(true) , ReadingFinishedSent(false) + , NotCommitedToFinishParents(notCommitedToFinishParents) { } void TPartitionActor::MakeCommit(const TActorContext& ctx) { ui64 offset = ClientReadOffset; - if (CommitsDisabled) - return; - if (CommitsInfly.size() > MAX_COMMITS_INFLY) + if (CommitsDisabled || NotCommitedToFinishParents.size() != 0 || CommitsInfly.size() >= MAX_COMMITS_INFLY) return; //Ranges mode if (!NextRanges.Empty() && NextRanges.Min() == ClientCommitOffset) { - auto first = NextRanges.begin(); - offset = first->second; - NextRanges.EraseInterval(first->first, first->second); + auto firstRange = NextRanges.begin(); + offset = firstRange->second; + NextRanges.EraseInterval(firstRange->first, firstRange->second); ClientCommitOffset = offset; ++CommitCookie; @@ -163,10 +163,10 @@ void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TAc if (!ClientHasAnyCommits && Parents.size() != 0) { std::vector commits; for (auto& parent: Parents) { - TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true}; + TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true, .ReadSessionId = Session}; commits.push_back(commit); } - TKqpHelper::TCommitInfo commit {.PartitionId = Partition.Partition, .Offset = (i64)offset, .KillReadSession = false, .OnlyCheckCommitedToFinish = false}; + TKqpHelper::TCommitInfo commit {.PartitionId = Partition.Partition, .Offset = (i64)offset, .KillReadSession = false, .OnlyCheckCommitedToFinish = false, .ReadSessionId = Session}; commits.push_back(commit); auto kqp = std::make_shared(Database, ClientId, Topic->GetPrimaryPath(), commits, readId); Kqps.emplace(readId, kqp); @@ -207,7 +207,7 @@ void TPartitionActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, c if (!kqpIt->second->Handle(ev, ctx)) { const auto& record = ev->Get()->Record; - ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("status is not ok: " + record.GetError(), PersQueue::ErrorCode::ERROR)); // savnik map ydbStatus + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("status is not ok: " + record.GetError(), PersQueue::ErrorCode::ERROR)); } } @@ -221,12 +221,13 @@ void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TAc } if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { - ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", PersQueue::ErrorCode::ERROR)); // savnik map ydbStatus. Need retry? + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", PersQueue::ErrorCode::ERROR)); // savnik: retry? return; } auto step = kqpIt->second->Handle(ev, ctx); if (step == TKqpHelper::ECurrentStep::DONE) { + ClientHasAnyCommits = true; CommitDone(ev->Cookie, ctx); } } @@ -589,6 +590,11 @@ bool FillBatchedData( return hasData; } +void TPartitionActor::Handle(TEvPQProxy::TEvParentCommitedToFinish::TPtr& ev, const TActorContext& ctx) { + NotCommitedToFinishParents.erase(ev->Get()->ParentPartitionId); + MakeCommit(ctx); +} + void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorContext& ctx) { if (ev->Get()->Record.HasErrorCode() && ev->Get()->Record.GetErrorCode() != NPersQueue::NErrorCode::OK) { @@ -921,7 +927,7 @@ void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { ui64 startReadId = CommitsInfly.front().second.StartReadId; ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(Partition.AssignId, startReadId, readId, CommittedOffset)); - Kqps.erase(CommitsInfly.front().first); // savnik то че надо удаляю? + Kqps.erase(CommitsInfly.front().first); CommitsInfly.pop_front(); LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Partition diff --git a/ydb/services/persqueue_v1/actors/partition_actor.h b/ydb/services/persqueue_v1/actors/partition_actor.h index 3a6b2d73179c..8153cc0d3964 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.h +++ b/ydb/services/persqueue_v1/actors/partition_actor.h @@ -68,7 +68,7 @@ class TPartitionActor : public NActors::TActorBootstrapped { static constexpr ui32 MAX_PIPE_RESTARTS = 100; //after 100 restarts without progress kill session static constexpr ui32 RESTART_PIPE_DELAY_MS = 100; - static constexpr ui32 MAX_COMMITS_INFLY = 3; + static constexpr ui32 MAX_COMMITS_INFLY = 1; public: @@ -76,7 +76,8 @@ class TPartitionActor : public NActors::TActorBootstrapped { const TString& session, const TPartitionId& partition, ui32 generation, ui32 step, const ui64 tabletID, const TTopicCounters& counters, const bool commitsDisabled, const TString& clientDC, bool rangesMode, const NPersQueue::TTopicConverterPtr& topic, bool directRead, - bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs, std::set parents); + bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs, std::set parents, + std::unordered_set notCommitedToFinishParents); ~TPartitionActor(); void Bootstrap(const NActors::TActorContext& ctx); @@ -106,6 +107,9 @@ class TPartitionActor : public NActors::TActorBootstrapped { HFunc(NKqp::TEvKqp::TEvCreateSessionResponse, Handle); HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle); + + HFunc(TEvPQProxy::TEvParentCommitedToFinish, Handle); + default: break; }; @@ -132,6 +136,8 @@ class TPartitionActor : public NActors::TActorBootstrapped { void Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const NActors::TActorContext& ctx); void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPQProxy::TEvParentCommitedToFinish::TPtr& ev, const TActorContext& ctx); + void HandlePoison(NActors::TEvents::TEvPoisonPill::TPtr& ev, const NActors::TActorContext& ctx); void HandleWakeup(const NActors::TActorContext& ctx); void DoWakeup(const NActors::TActorContext& ctx); @@ -256,6 +262,8 @@ class TPartitionActor : public NActors::TActorBootstrapped { bool FirstRead; bool ReadingFinishedSent; + + std::unordered_set NotCommitedToFinishParents; }; diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.cpp b/ydb/services/persqueue_v1/actors/read_session_actor.cpp index 466012af5365..378c8eee2ac8 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.cpp +++ b/ydb/services/persqueue_v1/actors/read_session_actor.cpp @@ -434,7 +434,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommitRange: for (const auto& [b, e] : ev->Get()->CommitInfo.Ranges) { if (!RangesMode) { - return CloseSession(PersQueue::ErrorCode::BAD_REQUEST, "commits ranges in cookies commit mode is prohibited", ctx); // savnik есть другой мод? Надо там тоже чет поддержать? + return CloseSession(PersQueue::ErrorCode::BAD_REQUEST, "commits ranges in cookies commit mode is prohibited", ctx); } if (b >= e || it->second.NextRanges.Intersects(b, e) || b < it->second.Offset) { @@ -458,7 +458,6 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuth::TPtr& } } - template void TReadSessionActor::Handle(TEvPQProxy::TEvDirectReadAck::TPtr& ev, const TActorContext& ctx) { auto it = Partitions.find(ev->Get()->AssignId); @@ -487,6 +486,14 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvDirectReadAc it->second.MaxProcessedDirectReadId = directReadId; it->second.PendingDirectReadAcks.push(directReadId); ProcessDirectReads(it, ctx); + it->second.MaxProcessedDirectReadId = ev->Get()->DirectReadId; + + BytesInflight_ -= drIt->second.ByteSize; + if (BytesInflight) { + (*BytesInflight) -= drIt->second.ByteSize; + } + it->second.DirectReads.erase(drIt); + ProcessReads(ctx); } @@ -642,14 +649,14 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommitDone:: } auto assignId = ev->Get()->AssignId; - auto it = Partitions.find(assignId); - if (it == Partitions.end()) { + auto partitionIt = Partitions.find(assignId); + if (partitionIt == Partitions.end()) { return CloseSession(PersQueue::ErrorCode::BAD_REQUEST, TStringBuilder() << "unknown partition_session_id " << assignId << " #01", ctx); } - Y_ABORT_UNLESS(it->second.Offset < ev->Get()->Offset); - it->second.NextRanges.EraseInterval(it->second.Offset, ev->Get()->Offset); + Y_ABORT_UNLESS(partitionIt->second.Offset < ev->Get()->Offset); + partitionIt->second.NextRanges.EraseInterval(partitionIt->second.Offset, ev->Get()->Offset); if (ev->Get()->StartCookie == Max()) { // means commit at start return; @@ -664,8 +671,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommitDone:: auto c = result.mutable_committed()->add_cookies(); c->set_partition_cookie(i); c->set_assign_id(ev->Get()->AssignId); - it->second.NextCommits.erase(i); - it->second.ReadIdCommitted = i; + partitionIt->second.NextCommits.erase(i); + partitionIt->second.ReadIdCommitted = i; } } else { // commit on cookies not supported in this case Y_ABORT_UNLESS(false); @@ -674,7 +681,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommitDone:: if constexpr (UseMigrationProtocol) { auto c = result.mutable_committed()->add_offset_ranges(); c->set_assign_id(ev->Get()->AssignId); - c->set_start_offset(it->second.Offset); + c->set_start_offset(partitionIt->second.Offset); c->set_end_offset(ev->Get()->Offset); } else { auto c = result.mutable_commit_offset_response()->add_partitions_committed_offsets(); @@ -683,14 +690,35 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommitDone:: } } - it->second.Offset = ev->Get()->Offset; + partitionIt->second.Offset = ev->Get()->Offset; LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " replying for commits" << ": assignId# " << ev->Get()->AssignId << ", from# " << ev->Get()->StartCookie << ", to# " << ev->Get()->LastCookie - << ", offset# " << it->second.Offset); + << ", offset# " << partitionIt->second.Offset); WriteToStreamOrDie(ctx, std::move(result)); + + if (ev->Get()->Offset == partitionIt->second.EndOffset && partitionIt->second.HasChildren) { + partitionIt->second.CommitedToFinish = true; + + auto topicName = partitionIt->second.Topic->GetInternalName(); + auto topicIt = Topics.find(partitionIt->second.Topic->GetInternalName()); + if (topicIt == Topics.end()) { + return CloseSession(PersQueue::ErrorCode::BAD_REQUEST, TStringBuilder() + << "unknown topic partition_session_id: " << assignId, ctx); + } + + auto& topic = topicIt->second; + for (auto& child: topic.PartitionGraph->GetPartition(partitionIt->second.Partition.Partition)->DirectChildren) { + for (auto& otherPartitions: Partitions) { + if (otherPartitions.second.Partition.Partition == child->Id) { + ctx.Send(otherPartitions.second.Actor, new TEvPQProxy::TEvParentCommitedToFinish(partitionIt->second.Partition.Partition)); + } + } + } + + } } template @@ -1232,10 +1260,19 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartit return CloseSession(PersQueue::ErrorCode::ERROR, error, ctx); } - const TActorId actorId = ctx.Register(new TPartitionActor( // savnik empty topic + std::unordered_set notCommitedToFinishParents; // savnik: если сначала придет лок на детей, все сломается. Плохо? + for (auto& parent: topic.PartitionGraph->GetPartition(record.GetPartition())->DirectParents) { + for (auto& otherPartitions: Partitions) { // savnik: to map + if (otherPartitions.second.Partition.Partition == parent->Id && !otherPartitions.second.CommitedToFinish) { + notCommitedToFinishParents.emplace(otherPartitions.second.Partition.Partition); + } + } + } + + const TActorId actorId = ctx.Register(new TPartitionActor( ctx.SelfID, ClientId, ClientPath, Cookie, Session, partitionId, record.GetGeneration(), record.GetStep(), record.GetTabletId(), it->second, CommitsDisabled, ClientDC, RangesMode, - converterIter->second, DirectRead, UseMigrationProtocol, maxLag, readTimestampMs, topic.PartitionGraph->GetPartition(partitionId.Partition)->HierarhicalParents)); + converterIter->second, DirectRead, UseMigrationProtocol, maxLag, readTimestampMs, topic.PartitionGraph->GetPartition(partitionId.Partition)->HierarhicalParents, notCommitedToFinishParents)); if (SessionsActive) { PartsPerSession.DecFor(Partitions.size(), 1); @@ -1273,6 +1310,14 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionSta << "unknown partition_session_id " << assignId << " #02", ctx); } + auto topicName = it->second.Topic->GetInternalName(); + auto topicIt = Topics.find(it->second.Topic->GetInternalName()); + if (topicIt == Topics.end()) { + return CloseSession(PersQueue::ErrorCode::BAD_REQUEST, TStringBuilder() + << "unknown topic partition_session_id: " << assignId, ctx); + } + + auto& topic = topicIt->second; TServerMessage result; result.set_status(Ydb::StatusIds::SUCCESS); @@ -1285,6 +1330,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionSta it->second.LockSent = true; it->second.Offset = ev->Get()->Offset; it->second.ConsumerHasAnyCommits = ev->Get()->ClientHasAnyCommits; + it->second.HasChildren = topic.PartitionGraph->GetPartition(it->second.Partition.Partition)->DirectChildren.size() > 0; + it->second.EndOffset = ev->Get()->EndOffset; if constexpr (UseMigrationProtocol) { result.mutable_assigned()->mutable_topic()->set_path(it->second.Topic->GetFederationPath()); diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.h b/ydb/services/persqueue_v1/actors/read_session_actor.h index 134d26e5f0a3..a67478a923bb 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.h +++ b/ydb/services/persqueue_v1/actors/read_session_actor.h @@ -46,11 +46,14 @@ struct TPartitionActorInfo { TDisjointIntervalTree NextRanges; ui64 Offset; bool ConsumerHasAnyCommits; + bool HasChildren; + bool CommitedToFinish; TInstant AssignTimestamp; ui64 Generation; ui64 NodeId; + ui64 EndOffset; struct TDirectReadInfo { From c117ecee23494521219094738291a8d765accb9f Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 4 Dec 2024 09:42:20 +0000 Subject: [PATCH 09/29] ut --- .../persqueue/read_balancer__balancing.cpp | 10 +- .../ut/ut_with_sdk/autoscaling_ut.cpp | 107 ++++++++++++++++++ 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/ydb/core/persqueue/read_balancer__balancing.cpp b/ydb/core/persqueue/read_balancer__balancing.cpp index 79218cdff6b9..f8f97a4c4795 100644 --- a/ydb/core/persqueue/read_balancer__balancing.cpp +++ b/ydb/core/persqueue/read_balancer__balancing.cpp @@ -681,7 +681,7 @@ bool IsRoot(const TPartitionGraph::Node* node, const std::unordered_set& p if (node->IsRoot()) { return true; } - for (auto* p : node->Parents) { + for (auto* p : node->DirectParents) { if (partitions.contains(p->Id)) { return false; } @@ -966,10 +966,10 @@ bool TConsumer::IsReadable(ui32 partitionId) { } if (Partitions.empty()) { - return node->Parents.empty(); + return node->DirectParents.empty(); } - for(auto* parent : node->HierarhicalParents) { + for(auto* parent : node->AllParents) { if (!IsInactive(parent->Id)) { return false; } @@ -1035,9 +1035,9 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons if (family->CanAttach(std::vector{id})) { auto* node = GetPartitionGraph().GetPartition(id); bool allParentsMerged = true; - if (node->Parents.size() > 1) { + if (node->DirectParents.size() > 1) { // The partition was obtained as a result of the merge. - for (auto* c : node->Parents) { + for (auto* c : node->DirectParents) { auto* other = FindFamily(c->Id); if (!other) { allParentsMerged = false; diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index c0a50542b97d..25aeaa428f35 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -1218,6 +1218,113 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(stats1->GetCommittedOffset() == 4); } + Y_UNIT_TEST(PartitionSplit_DistributedTxCommit_CheckSessionReset) { + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(1) + .MaxActivePartitions(100) + .BeginConfigureAutoPartitioningSettings() + .UpUtilizationPercent(2) + .DownUtilizationPercent(1) + .StabilizationWindow(TDuration::Seconds(2)) + .Strategy(EAutoPartitioningStrategy::ScaleUp) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings() + .BeginAddConsumer() + .ConsumerName(TEST_CONSUMER); + + client.CreateTopic(TEST_TOPIC, createSettings).Wait(); + + auto msg = TString(1_MB, 'a'); + + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, TEST_TOPIC, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, TEST_TOPIC, false); + auto seqNo = 1; + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + Sleep(TDuration::Seconds(5)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 1); + } + + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, seqNo++))); + Sleep(TDuration::Seconds(5)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); + } + + auto writeSession_3 = CreateWriteSession(client, "producer-2", 1, TEST_TOPIC, false); + UNIT_ASSERT(writeSession_3->Write(Msg("message", seqNo++))); + UNIT_ASSERT(writeSession_3->Write(Msg("message", seqNo++))); + + auto reader = client.CreateReadSession( + TReadSessionSettings() + .AutoPartitioningSupport(true) + .AppendTopics(TTopicReadSettings(TEST_TOPIC)) + .ConsumerName(TEST_CONSUMER)); + + TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(15); + + auto commitSent = false; + + while(deadlineTime > TInstant::Now()) { + for (auto event : reader->GetEvents(false)) { + if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + auto& messages = x->GetMessages(); + for (size_t i = 0u; i < messages.size(); ++i) { + auto& message = messages[i]; + message.Commit(); + + if (message.GetSeqNo() == 8) { + if (!commitSent) { + commitSent = true; + Sleep(TDuration::MilliSeconds(300)); + auto status = client.CommitOffset(TEST_TOPIC, 1, TEST_CONSUMER, 0).GetValueSync(); + UNIT_ASSERT(status.IsSuccess()); + } else { + return; + } + } + } + UNIT_ASSERT(writeSession_3->Write(Msg(TStringBuilder() << "message-" << seqNo, seqNo))); + seqNo++; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* sessionClosedEvent = std::get_if(&event)) { + Cerr << sessionClosedEvent->DebugString() << Endl << Flush; + } else { + Cerr << "SESSION EVENT unhandled \n"; + } + } + Sleep(TDuration::MilliSeconds(250)); + } + + UNIT_ASSERT(false); + } + Y_UNIT_TEST(PartitionSplit_AutosplitByLoad) { TTopicSdkTestSetup setup = CreateSetup(); TTopicClient client = setup.MakeClient(); From 9b185acd446c025270933fc1347fe9ad6cbe81ad Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Thu, 5 Dec 2024 12:28:52 +0000 Subject: [PATCH 10/29] fix after merge --- ydb/core/persqueue/partition.cpp | 2 +- ydb/core/persqueue/read_balancer_app.cpp | 8 ++++---- ydb/services/persqueue_v1/actors/partition_actor.cpp | 2 +- ydb/services/persqueue_v1/actors/partition_actor.h | 2 +- ydb/services/persqueue_v1/actors/read_session_actor.cpp | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index dc0dbc5bf7d1..fc906a44cce0 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -2331,7 +2331,7 @@ TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPr bool isAffectedConsumer = AffectedUsers.contains(consumer); TUserInfoBase& userInfo = GetOrCreatePendingUser(consumer); - if (!operation.GetReadSessionId().Empty() && operation.GetReadSessionId() != userInfo.Session) { + if (!operation.GetReadSessionId().empty() && operation.GetReadSessionId() != userInfo.Session) { PQ_LOG_D("Partition " << Partition << " Consumer '" << consumer << "'" << " Bad request (session already dead) " << diff --git a/ydb/core/persqueue/read_balancer_app.cpp b/ydb/core/persqueue/read_balancer_app.cpp index d4f9f8abf137..9ca5c425258b 100644 --- a/ydb/core/persqueue/read_balancer_app.cpp +++ b/ydb/core/persqueue/read_balancer_app.cpp @@ -77,7 +77,7 @@ TString TPersQueueReadBalancer::GenerateStat() { for (auto& [partitionId, partitionInfo] : PartitionsInfo) { const auto& stats = AggregatedStats.Stats[partitionId]; const auto* node = PartitionGraph.GetPartition(partitionId); - TString style = node && node->Children.empty() ? "text-success" : "text-muted"; + TString style = node && node->DirectChildren.empty() ? "text-success" : "text-muted"; TABLER() { TABLED() { @@ -87,7 +87,7 @@ TString TPersQueueReadBalancer::GenerateStat() { } TABLED() { if (node) { - str << (node->Children.empty() ? "Active" : "Inactive"); + str << (node->DirectChildren.empty() ? "Active" : "Inactive"); if (node->IsRoot()) { str << " (root)"; } @@ -96,7 +96,7 @@ TString TPersQueueReadBalancer::GenerateStat() { TABLED() { HREF(TStringBuilder() << "?TabletID=" << partitionInfo.TabletId) { str << partitionInfo.TabletId; } } TABLED() { if (node) { - for (auto* parent : node->Parents) { + for (auto* parent : node->DirectParents) { HREF("#" + partitionAnchor(parent->Id)) { str << parent->Id; } str << ", "; } @@ -104,7 +104,7 @@ TString TPersQueueReadBalancer::GenerateStat() { } TABLED() { if (node) { - for (auto* child : node->Children) { + for (auto* child : node->DirectChildren) { HREF("#" + partitionAnchor(child->Id)) { str << child->Id; } str << ", "; } diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index f325943f74d1..fee8f5983120 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -25,7 +25,7 @@ TPartitionActor::TPartitionActor( const TActorId& parentId, const TString& clientId, const TString& clientPath, const ui64 cookie, const TString& session, const TPartitionId& partition, const ui32 generation, const ui32 step, const ui64 tabletID, const TTopicCounters& counters, bool commitsDisabled, - const TString& clientDC, bool rangesMode, const NPersQueue::TTopicConverterPtr& topic, + const TString& clientDC, bool rangesMode, const NPersQueue::TTopicConverterPtr& topic, const TString& database, bool directRead, bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs, std::set parents, std::unordered_set notCommitedToFinishParents ) diff --git a/ydb/services/persqueue_v1/actors/partition_actor.h b/ydb/services/persqueue_v1/actors/partition_actor.h index 8153cc0d3964..7ac9ae9cd01c 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.h +++ b/ydb/services/persqueue_v1/actors/partition_actor.h @@ -75,7 +75,7 @@ class TPartitionActor : public NActors::TActorBootstrapped { TPartitionActor(const TActorId& parentId, const TString& clientId, const TString& clientPath, const ui64 cookie, const TString& session, const TPartitionId& partition, ui32 generation, ui32 step, const ui64 tabletID, const TTopicCounters& counters, const bool commitsDisabled, - const TString& clientDC, bool rangesMode, const NPersQueue::TTopicConverterPtr& topic, bool directRead, + const TString& clientDC, bool rangesMode, const NPersQueue::TTopicConverterPtr& topic, const TString& database, bool directRead, bool useMigrationProtocol, ui32 maxTimeLagMs, ui64 readTimestampMs, std::set parents, std::unordered_set notCommitedToFinishParents); ~TPartitionActor(); diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.cpp b/ydb/services/persqueue_v1/actors/read_session_actor.cpp index 378c8eee2ac8..1b8d027f085a 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.cpp +++ b/ydb/services/persqueue_v1/actors/read_session_actor.cpp @@ -1272,7 +1272,7 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartit const TActorId actorId = ctx.Register(new TPartitionActor( ctx.SelfID, ClientId, ClientPath, Cookie, Session, partitionId, record.GetGeneration(), record.GetStep(), record.GetTabletId(), it->second, CommitsDisabled, ClientDC, RangesMode, - converterIter->second, DirectRead, UseMigrationProtocol, maxLag, readTimestampMs, topic.PartitionGraph->GetPartition(partitionId.Partition)->HierarhicalParents, notCommitedToFinishParents)); + converterIter->second, Request->GetDatabaseName().GetOrElse(AppData(ctx)->PQConfig.GetDatabase()), DirectRead, UseMigrationProtocol, maxLag, readTimestampMs, topic.PartitionGraph->GetPartition(partitionId.Partition)->AllParents, notCommitedToFinishParents)); if (SessionsActive) { PartsPerSession.DecFor(Partitions.size(), 1); From 45553c26ce67b060909284549d5bd5fa9c319ad1 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Mon, 9 Dec 2024 21:57:32 +0000 Subject: [PATCH 11/29] fix uts --- .../actors/read_session_actor.cpp | 43 ++++++++++--------- .../persqueue_v1/actors/read_session_actor.h | 1 + 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.cpp b/ydb/services/persqueue_v1/actors/read_session_actor.cpp index 1b8d027f085a..b694fab5e5ae 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.cpp +++ b/ydb/services/persqueue_v1/actors/read_session_actor.cpp @@ -699,26 +699,6 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommitDone:: << ", offset# " << partitionIt->second.Offset); WriteToStreamOrDie(ctx, std::move(result)); - if (ev->Get()->Offset == partitionIt->second.EndOffset && partitionIt->second.HasChildren) { - partitionIt->second.CommitedToFinish = true; - - auto topicName = partitionIt->second.Topic->GetInternalName(); - auto topicIt = Topics.find(partitionIt->second.Topic->GetInternalName()); - if (topicIt == Topics.end()) { - return CloseSession(PersQueue::ErrorCode::BAD_REQUEST, TStringBuilder() - << "unknown topic partition_session_id: " << assignId, ctx); - } - - auto& topic = topicIt->second; - for (auto& child: topic.PartitionGraph->GetPartition(partitionIt->second.Partition.Partition)->DirectChildren) { - for (auto& otherPartitions: Partitions) { - if (otherPartitions.second.Partition.Partition == child->Id) { - ctx.Send(otherPartitions.second.Actor, new TEvPQProxy::TEvParentCommitedToFinish(partitionIt->second.Partition.Partition)); - } - } - } - - } } template @@ -1114,8 +1094,14 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvAuthResultOk return CloseSession(PersQueue::ErrorCode::OVERLOAD, TStringBuilder() << "metering mode of topic: " << name << " has been changed", ctx); } + it->second.PartitionGraph = t.PartitionGraph; } } + + while (!Locks.empty()) { + ctx.Send(ctx.SelfID, std::move(Locks.front())); + Locks.pop_front(); + } } template @@ -1234,6 +1220,14 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartit auto& topic = topicIt->second; + auto* partitionNode = topic.PartitionGraph->GetPartition(record.GetPartition()); + if (!partitionNode) { + Locks.push_back(ev->Release()); + ForceACLCheck = true; + RecheckACL(ctx); + return; + } + // TODO: counters if (NumPartitionsFromTopic[name]++ == 0) { if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { @@ -2396,6 +2390,15 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadingFinis auto& topic = it->second; NTabletPipe::SendData(ctx, topic.PipeClient, new TEvPersQueue::TEvReadingPartitionFinishedRequest(ClientId, msg->PartitionId, AutoPartitioningSupport, msg->FirstMessage)); + std::unordered_set childs(msg->ChildPartitionIds.begin(), msg->ChildPartitionIds.end()); + for (auto& partition: Partitions) { + if (partition.second.Partition.Partition == msg->PartitionId) { + partition.second.CommitedToFinish = true; + } else if (childs.contains(partition.second.Partition.Partition)) { + ctx.Send(partition.second.Actor, new TEvPQProxy::TEvParentCommitedToFinish(msg->PartitionId)); + } + } + if constexpr (!UseMigrationProtocol) { if (AutoPartitioningSupport) { TPartitionActorInfo* partitionInfo = nullptr; diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.h b/ydb/services/persqueue_v1/actors/read_session_actor.h index a67478a923bb..f93953c2976e 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.h +++ b/ydb/services/persqueue_v1/actors/read_session_actor.h @@ -423,6 +423,7 @@ class TReadSessionActor TMap PartitionToControlMessages; std::deque> Reads; + std::deque> Locks; ui64 Cookie; From c7dd0fc9a0a61dcf3bbdab3ab895c9d1bcb42d38 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Tue, 10 Dec 2024 20:34:39 +0000 Subject: [PATCH 12/29] fix --- .../ut/ut_with_sdk/autoscaling_ut.cpp | 2 +- ydb/services/persqueue_v1/actors/events.h | 4 +- .../persqueue_v1/actors/partition_actor.cpp | 4 +- .../actors/read_session_actor.cpp | 39 ++++++++++++------- .../persqueue_v1/actors/read_session_actor.h | 2 - 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index 25aeaa428f35..5d8f428301df 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -729,7 +729,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer can commit at the end of the inactive partition."); status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::BAD_REQUEST, status.GetStatus(), "The consumer cannot commit an offset for inactive, read-to-the-end partitions."); + UNIT_ASSERT_VALUES_EQUAL_C(NYdb::EStatus::SUCCESS, status.GetStatus(), "The consumer can commit an offset for inactive, read-to-the-end partitions."); } Y_UNIT_TEST(ControlPlane_CreateAlterDescribe) { diff --git a/ydb/services/persqueue_v1/actors/events.h b/ydb/services/persqueue_v1/actors/events.h index 5705be99a0da..bf12ade4567c 100644 --- a/ydb/services/persqueue_v1/actors/events.h +++ b/ydb/services/persqueue_v1/actors/events.h @@ -408,17 +408,19 @@ struct TEvPQProxy { struct TEvCommitDone : public NActors::TEventLocal { - explicit TEvCommitDone(const ui64 assignId, const ui64 startCookie, const ui64 lastCookie, const ui64 offset) + explicit TEvCommitDone(const ui64 assignId, const ui64 startCookie, const ui64 lastCookie, const ui64 offset, const ui64 endOffset) : AssignId(assignId) , StartCookie(startCookie) , LastCookie(lastCookie) , Offset(offset) + , EndOffset(endOffset) { } ui64 AssignId; ui64 StartCookie; ui64 LastCookie; ui64 Offset; + ui64 EndOffset; }; struct TEvParentCommitedToFinish : public NActors::TEventLocal { diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index fee8f5983120..ec7b4673e2a3 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -108,7 +108,7 @@ void TPartitionActor::MakeCommit(const TActorContext& ctx) { if (it != NextCommits.end() && *it == 0) { //commit of readed in prev session data NextCommits.erase(NextCommits.begin()); if (ClientReadOffset <= ClientCommitOffset) { - ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(Partition.AssignId, 0, 0, CommittedOffset)); + ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(Partition.AssignId, 0, 0, CommittedOffset, EndOffset)); } else { ClientCommitOffset = ClientReadOffset; CommitsInfly.emplace_back(0, TCommitInfo{0, ClientReadOffset, ctx.Now()}); @@ -925,7 +925,7 @@ void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { CommittedOffset = CommitsInfly.front().second.Offset; ui64 startReadId = CommitsInfly.front().second.StartReadId; - ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(Partition.AssignId, startReadId, readId, CommittedOffset)); + ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(Partition.AssignId, startReadId, readId, CommittedOffset, EndOffset)); Kqps.erase(CommitsInfly.front().first); CommitsInfly.pop_front(); diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.cpp b/ydb/services/persqueue_v1/actors/read_session_actor.cpp index b694fab5e5ae..5b2fa0678354 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.cpp +++ b/ydb/services/persqueue_v1/actors/read_session_actor.cpp @@ -691,6 +691,7 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommitDone:: } partitionIt->second.Offset = ev->Get()->Offset; + partitionIt->second.EndOffset = ev->Get()->EndOffset; LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " replying for commits" << ": assignId# " << ev->Get()->AssignId @@ -699,6 +700,25 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommitDone:: << ", offset# " << partitionIt->second.Offset); WriteToStreamOrDie(ctx, std::move(result)); + if (ev->Get()->Offset == ev->Get()->EndOffset) { + auto topicName = partitionIt->second.Topic->GetInternalName(); + auto topicIt = Topics.find(partitionIt->second.Topic->GetInternalName()); + if (topicIt == Topics.end()) { + return CloseSession(PersQueue::ErrorCode::BAD_REQUEST, TStringBuilder() + << "unknown topic partition_session_id: " << assignId, ctx); + } + + auto& topic = topicIt->second; + for (auto& child: topic.PartitionGraph->GetPartition(partitionIt->second.Partition.Partition)->DirectChildren) { + for (auto& otherPartitions: Partitions) { + if (otherPartitions.second.Partition.Partition == child->Id) { + ctx.Send(otherPartitions.second.Actor, new TEvPQProxy::TEvParentCommitedToFinish(partitionIt->second.Partition.Partition)); + } + } + } + + } + } template @@ -1223,8 +1243,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartit auto* partitionNode = topic.PartitionGraph->GetPartition(record.GetPartition()); if (!partitionNode) { Locks.push_back(ev->Release()); - ForceACLCheck = true; - RecheckACL(ctx); + if (!AuthInitActor) { + RunAuthActor(ctx); + } return; } @@ -1257,7 +1278,7 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartit std::unordered_set notCommitedToFinishParents; // savnik: если сначала придет лок на детей, все сломается. Плохо? for (auto& parent: topic.PartitionGraph->GetPartition(record.GetPartition())->DirectParents) { for (auto& otherPartitions: Partitions) { // savnik: to map - if (otherPartitions.second.Partition.Partition == parent->Id && !otherPartitions.second.CommitedToFinish) { + if (otherPartitions.second.Partition.Partition == parent->Id && otherPartitions.second.Offset != otherPartitions.second.EndOffset) { notCommitedToFinishParents.emplace(otherPartitions.second.Partition.Partition); } } @@ -1311,8 +1332,6 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionSta << "unknown topic partition_session_id: " << assignId, ctx); } - auto& topic = topicIt->second; - TServerMessage result; result.set_status(Ydb::StatusIds::SUCCESS); if (ev->Get()->Init) { @@ -1324,7 +1343,6 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionSta it->second.LockSent = true; it->second.Offset = ev->Get()->Offset; it->second.ConsumerHasAnyCommits = ev->Get()->ClientHasAnyCommits; - it->second.HasChildren = topic.PartitionGraph->GetPartition(it->second.Partition.Partition)->DirectChildren.size() > 0; it->second.EndOffset = ev->Get()->EndOffset; if constexpr (UseMigrationProtocol) { @@ -2390,15 +2408,6 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadingFinis auto& topic = it->second; NTabletPipe::SendData(ctx, topic.PipeClient, new TEvPersQueue::TEvReadingPartitionFinishedRequest(ClientId, msg->PartitionId, AutoPartitioningSupport, msg->FirstMessage)); - std::unordered_set childs(msg->ChildPartitionIds.begin(), msg->ChildPartitionIds.end()); - for (auto& partition: Partitions) { - if (partition.second.Partition.Partition == msg->PartitionId) { - partition.second.CommitedToFinish = true; - } else if (childs.contains(partition.second.Partition.Partition)) { - ctx.Send(partition.second.Actor, new TEvPQProxy::TEvParentCommitedToFinish(msg->PartitionId)); - } - } - if constexpr (!UseMigrationProtocol) { if (AutoPartitioningSupport) { TPartitionActorInfo* partitionInfo = nullptr; diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.h b/ydb/services/persqueue_v1/actors/read_session_actor.h index f93953c2976e..8dc4080a5469 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.h +++ b/ydb/services/persqueue_v1/actors/read_session_actor.h @@ -46,8 +46,6 @@ struct TPartitionActorInfo { TDisjointIntervalTree NextRanges; ui64 Offset; bool ConsumerHasAnyCommits; - bool HasChildren; - bool CommitedToFinish; TInstant AssignTimestamp; From 3f898d97146483b64b69d2629a9280b103418a98 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 11 Dec 2024 16:44:50 +0000 Subject: [PATCH 13/29] fix --- ydb/core/persqueue/partition.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index fc906a44cce0..74bf7630d62c 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -3180,7 +3180,7 @@ void TPartition::EmulatePostProcessUserAct(const TEvPQ::TEvSetClientInfo& act, ScheduleReplyGetClientOffsetOk(act.Cookie, offset, - ts.first, ts.second, ui->AnyCommits); + ts.first, ts.second, ui ? ui->AnyCommits : false); } else { ScheduleReplyOk(act.Cookie); } From 9ac46b61733179a3eb5180f6504952c1c695df1a Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 11 Dec 2024 19:37:25 +0000 Subject: [PATCH 14/29] fix --- ydb/services/persqueue_v1/actors/partition_actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index ec7b4673e2a3..11e296990335 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -973,7 +973,7 @@ void TPartitionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const } void TPartitionActor::Handle(TEvPQProxy::TEvGetStatus::TPtr&, const TActorContext& ctx) { - ctx.Send(ParentId, new TEvPQProxy::TEvPartitionStatus(Partition, CommittedOffset, EndOffset, WriteTimestampEstimateMs, NodeId, TabletGeneration, false)); + ctx.Send(ParentId, new TEvPQProxy::TEvPartitionStatus(Partition, CommittedOffset, EndOffset, WriteTimestampEstimateMs, NodeId, TabletGeneration, false, ClientHasAnyCommits)); } void TPartitionActor::Handle(TEvPQProxy::TEvLockPartition::TPtr& ev, const TActorContext& ctx) { From 67fdf36a4be4d5c81bc489bacf0fa211a65a928a Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 12 Mar 2025 13:05:40 +0000 Subject: [PATCH 15/29] fix after rebase --- ydb/core/persqueue/partition.cpp | 2 +- .../ut/ut_with_sdk/autoscaling_ut.cpp | 42 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 74bf7630d62c..53904aa291a6 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -2299,7 +2299,7 @@ bool TPartition::ExecUserActionOrTransaction(TSimpleSharedPtr& t, TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPredicate& tx, TMaybe& predicateOut) { if (tx.ForcePredicateFalse) { - predicate = false; + predicateOut = false; return EProcessResult::Continue; } diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index 5d8f428301df..f8d3b8e605fc 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -646,8 +646,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto msg = TString(1_MB, 'a'); - auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, TEST_TOPIC, false); - auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, TEST_TOPIC, false); + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, std::string{TEST_TOPIC}, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, std::string{TEST_TOPIC}, false); { UNIT_ASSERT(writeSession_1->Write(Msg(msg, 1))); @@ -667,8 +667,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } - auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, TEST_TOPIC, false); - auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, TEST_TOPIC, false); + auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, std::string{TEST_TOPIC}, false); + auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, std::string{TEST_TOPIC}, false); { UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); @@ -691,15 +691,15 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(description.GetPartitions().size() == 5); auto stats0 = description.GetPartitions().at(0).GetPartitionConsumerStats(); - UNIT_ASSERT(stats0.Defined()); + UNIT_ASSERT(stats0); UNIT_ASSERT(stats0->GetCommittedOffset() == 6); auto stats1 = description.GetPartitions().at(1).GetPartitionConsumerStats(); - UNIT_ASSERT(stats1.Defined()); + UNIT_ASSERT(stats1); UNIT_ASSERT(stats1->GetCommittedOffset() == 2); auto stats3 = description.GetPartitions().at(3).GetPartitionConsumerStats(); - UNIT_ASSERT(stats3.Defined()); + UNIT_ASSERT(stats3); UNIT_ASSERT(stats3->GetCommittedOffset() == 0); } @@ -972,8 +972,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto msg = TString(1_MB, 'a'); - auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, TEST_TOPIC, false); - auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, TEST_TOPIC, false); + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, std::string{TEST_TOPIC}, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, std::string{TEST_TOPIC}, false); { UNIT_ASSERT(writeSession_1->Write(Msg(msg, 1))); @@ -993,8 +993,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } - auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, TEST_TOPIC, false); - auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, TEST_TOPIC, false); + auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, std::string{TEST_TOPIC}, false); + auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, std::string{TEST_TOPIC}, false); { UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); @@ -1063,7 +1063,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(description.GetPartitions().size() == 5); auto stats1 = description.GetPartitions().at(1).GetPartitionConsumerStats(); - UNIT_ASSERT(stats1.Defined()); + UNIT_ASSERT(stats1); UNIT_ASSERT(stats1->GetCommittedOffset() == 4); } @@ -1090,8 +1090,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto msg = TString(1_MB, 'a'); - auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, TEST_TOPIC, false); - auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, TEST_TOPIC, false); + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, std::string{TEST_TOPIC}, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, std::string{TEST_TOPIC}, false); { UNIT_ASSERT(writeSession_1->Write(Msg(msg, 1))); @@ -1111,8 +1111,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } - auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, TEST_TOPIC, false); - auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, TEST_TOPIC, false); + auto writeSession2_1 = CreateWriteSession(client, "producer-1", 1, std::string{TEST_TOPIC}, false); + auto writeSession2_2 = CreateWriteSession(client, "producer-2", 1, std::string{TEST_TOPIC}, false); { UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); @@ -1196,7 +1196,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(description.GetPartitions().size() == 5); auto stats1 = description.GetPartitions().at(1).GetPartitionConsumerStats(); - UNIT_ASSERT(stats1.Defined()); + UNIT_ASSERT(stats1); UNIT_ASSERT(stats1->GetCommittedOffset() == 0); @@ -1213,7 +1213,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(description2.GetPartitions().size() == 5); stats1 = description2.GetPartitions().at(1).GetPartitionConsumerStats(); - UNIT_ASSERT(stats1.Defined()); + UNIT_ASSERT(stats1); UNIT_ASSERT(stats1->GetCommittedOffset() == 4); } @@ -1241,8 +1241,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto msg = TString(1_MB, 'a'); - auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, TEST_TOPIC, false); - auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, TEST_TOPIC, false); + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, std::string{TEST_TOPIC}, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, std::string{TEST_TOPIC}, false); auto seqNo = 1; { UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); @@ -1262,7 +1262,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } - auto writeSession_3 = CreateWriteSession(client, "producer-2", 1, TEST_TOPIC, false); + auto writeSession_3 = CreateWriteSession(client, "producer-2", 1, std::string{TEST_TOPIC}, false); UNIT_ASSERT(writeSession_3->Write(Msg("message", seqNo++))); UNIT_ASSERT(writeSession_3->Write(Msg("message", seqNo++))); From a5f62e09f8aefac8a4de92990bc0577fadb5724f Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Tue, 25 Mar 2025 18:53:19 +0000 Subject: [PATCH 16/29] fix --- ydb/core/persqueue/events/internal.h | 4 +++- ydb/core/persqueue/partition.cpp | 4 ++-- ydb/core/protos/msgbus_pq.proto | 1 + ydb/services/persqueue_v1/actors/partition_actor.cpp | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index 5f8c15036a28..ba3278254f65 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -362,7 +362,7 @@ struct TEvPQ { TEvSetClientInfo(const ui64 cookie, const TString& clientId, const ui64 offset, const TString& sessionId, const ui64 partitionSessionId, const ui32 generation, const ui32 step, const TActorId& pipeClient, - ESetClientInfoType type = ESCI_OFFSET, ui64 readRuleGeneration = 0, bool strict = false) + ESetClientInfoType type = ESCI_OFFSET, ui64 readRuleGeneration = 0, bool strict = false, bool killReadSession = false) : Cookie(cookie) , ClientId(clientId) , Offset(offset) @@ -373,6 +373,7 @@ struct TEvPQ { , Type(type) , ReadRuleGeneration(readRuleGeneration) , Strict(strict) + , KillReadSession(killReadSession) , PipeClient(pipeClient) { } @@ -387,6 +388,7 @@ struct TEvPQ { ESetClientInfoType Type; ui64 ReadRuleGeneration; bool Strict; + bool KillReadSession; TActorId PipeClient; }; diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 53904aa291a6..b229819c9c75 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -2337,8 +2337,8 @@ TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPr " Bad request (session already dead) " << " SessionId '" << operation.GetReadSessionId() << "'"); result = false; - } else if (operation.GetOnlyCheckCommitedToFinish() && !IsActive()) { - if (static_cast(userInfo.Offset) != EndOffset) { + } else if (operation.GetOnlyCheckCommitedToFinish()) { + if (IsActive() || static_cast(userInfo.Offset) != EndOffset) { result = false; } } else { diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto index ae4fe4ad0e91..922c55e8f653 100644 --- a/ydb/core/protos/msgbus_pq.proto +++ b/ydb/core/protos/msgbus_pq.proto @@ -69,6 +69,7 @@ message TPersQueuePartitionRequest { optional bool MirrorerRequest = 10 [default = false]; optional bool Strict = 11 [default = false]; + optional bool KillReadSession = 12 [default = false]; } message TCmdGetClientOffset { diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index 11e296990335..976b4c94a1ab 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -160,7 +160,7 @@ void TPartitionActor::Bootstrap(const TActorContext& ctx) { } void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx) { - if (!ClientHasAnyCommits && Parents.size() != 0) { + if (!ClientHasAnyCommits && Parents.size() != 0) { // savnik проверка на то что включены транзакции? std::vector commits; for (auto& parent: Parents) { TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true, .ReadSessionId = Session}; From a96154fd684eb4217f3f857c02794c26586fa10b Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 26 Mar 2025 11:11:08 +0000 Subject: [PATCH 17/29] fix --- ydb/services/persqueue_v1/actors/partition_actor.cpp | 2 +- ydb/services/persqueue_v1/actors/partition_actor.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index 976b4c94a1ab..0ec35cae0ba1 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -973,7 +973,7 @@ void TPartitionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const } void TPartitionActor::Handle(TEvPQProxy::TEvGetStatus::TPtr&, const TActorContext& ctx) { - ctx.Send(ParentId, new TEvPQProxy::TEvPartitionStatus(Partition, CommittedOffset, EndOffset, WriteTimestampEstimateMs, NodeId, TabletGeneration, false, ClientHasAnyCommits)); + ctx.Send(ParentId, new TEvPQProxy::TEvPartitionStatus(Partition, CommittedOffset, EndOffset, WriteTimestampEstimateMs, NodeId, TabletGeneration, ClientHasAnyCommits, false)); } void TPartitionActor::Handle(TEvPQProxy::TEvLockPartition::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/services/persqueue_v1/actors/partition_actor.h b/ydb/services/persqueue_v1/actors/partition_actor.h index 7ac9ae9cd01c..f9ab74e10f9b 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.h +++ b/ydb/services/persqueue_v1/actors/partition_actor.h @@ -6,7 +6,7 @@ #include #include -#include "ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h" +#include #include #include From 2a1a3ad387ef8fff59610dd0f66400786bcbb8e9 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 26 Mar 2025 15:53:51 +0000 Subject: [PATCH 18/29] fix --- ydb/services/persqueue_v1/actors/partition_actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index 0ec35cae0ba1..be22cff0bff7 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -227,7 +227,6 @@ void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TAc auto step = kqpIt->second->Handle(ev, ctx); if (step == TKqpHelper::ECurrentStep::DONE) { - ClientHasAnyCommits = true; CommitDone(ev->Cookie, ctx); } } @@ -730,7 +729,7 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo return; } - if (!(result.HasCmdReadResult() || result.HasCmdPrepareReadResult() || result.HasCmdPublishReadResult() || result.HasCmdForgetReadResult())) { //this is commit response + if (!(result.HasCmdReadResult() || result.HasCmdPrepareReadResult() || result.HasCmdPublishReadResult() || result.HasCmdForgetReadResult())) { // this is commit response CommitDone(result.GetCookie(), ctx); return; } @@ -913,6 +912,7 @@ void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { } Counters.Commits.Inc(); + ClientHasAnyCommits = true; ui32 commitDurationMs = (ctx.Now() - CommitsInfly.front().second.StartTime).MilliSeconds(); if (Counters.CommitLatency) { From 2e4b1c09bb0185fd4b652a927784420dc2a37cf6 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 26 Mar 2025 16:00:29 +0000 Subject: [PATCH 19/29] PQ_V0 comits to autopartitioned topics --- .../persqueue_v0/grpc_pq_read_actor.cpp | 206 ++++++++++++------ 1 file changed, 137 insertions(+), 69 deletions(-) diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp index 43f335ac67f0..0456a00b1103 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -144,8 +145,8 @@ TString PartitionResponseToLog(const NKikimrClient::TPersQueuePartitionResponse& class TPartitionActor : public NActors::TActorBootstrapped { public: TPartitionActor(const TActorId& parentId, const TString& clientId, const ui64 cookie, const TString& session, const ui32 generation, - const ui32 step, const NPersQueue::TTopicConverterPtr& topic, const ui32 partition, const ui64 tabletID, - const TReadSessionActor::TTopicCounters& counters, const TString& clientDC); + const ui32 step, const NPersQueue::TTopicConverterPtr& topic, const TString& database, const ui32 partition, const ui64 tabletID, + const TReadSessionActor::TTopicCounters& counters, const TString& clientDC, std::set parents); ~TPartitionActor(); void Bootstrap(const NActors::TActorContext& ctx); @@ -156,21 +157,24 @@ class TPartitionActor : public NActors::TActorBootstrapped { private: STFUNC(StateFunc) { switch (ev->GetTypeRewrite()) { - CFunc(NActors::TEvents::TSystem::Wakeup, HandleWakeup) - HFunc(TEvPQProxy::TEvDeadlineExceeded, Handle) + CFunc(NActors::TEvents::TSystem::Wakeup, HandleWakeup); + HFunc(TEvPQProxy::TEvDeadlineExceeded, Handle); - HFunc(NActors::TEvents::TEvPoisonPill, HandlePoison) - HFunc(TEvPQProxy::TEvRead, Handle) - HFunc(TEvPQProxy::TEvCommit, Handle) - HFunc(TEvPQProxy::TEvReleasePartition, Handle) - HFunc(TEvPQProxy::TEvLockPartition, Handle) - HFunc(TEvPQProxy::TEvGetStatus, Handle) - HFunc(TEvPQProxy::TEvRestartPipe, Handle) + HFunc(NActors::TEvents::TEvPoisonPill, HandlePoison); + HFunc(TEvPQProxy::TEvRead, Handle); + HFunc(TEvPQProxy::TEvCommit, Handle); + HFunc(TEvPQProxy::TEvReleasePartition, Handle); + HFunc(TEvPQProxy::TEvLockPartition, Handle); + HFunc(TEvPQProxy::TEvGetStatus, Handle); + HFunc(TEvPQProxy::TEvRestartPipe, Handle); HFunc(TEvTabletPipe::TEvClientDestroyed, Handle); HFunc(TEvTabletPipe::TEvClientConnected, Handle); HFunc(TEvPersQueue::TEvResponse, Handle); HFunc(TEvPersQueue::TEvHasDataInfoResponse, Handle); + + HFunc(NKqp::TEvKqp::TEvCreateSessionResponse, Handle); + HFunc(NKqp::TEvKqp::TEvQueryResponse, Handle); default: break; }; @@ -192,6 +196,9 @@ class TPartitionActor : public NActors::TActorBootstrapped { void Handle(TEvPersQueue::TEvResponse::TPtr& ev, const NActors::TActorContext& ctx); void Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, const NActors::TActorContext& ctx); + void Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const NActors::TActorContext& ctx); + void Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx); + void HandlePoison(NActors::TEvents::TEvPoisonPill::TPtr& ev, const NActors::TActorContext& ctx); void HandleWakeup(const NActors::TActorContext& ctx); @@ -202,6 +209,7 @@ class TPartitionActor : public NActors::TActorBootstrapped { void RestartPipe(const NActors::TActorContext& ctx, const TString& reason, const NPersQueue::NErrorCode::EErrorCode errorCode); void WaitDataInPartition(const NActors::TActorContext& ctx); void SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx); + void CommitDone(ui64 cookie, const TActorContext& ctx); void SendPartitionReady(const TActorContext& ctx); private: @@ -214,6 +222,7 @@ class TPartitionActor : public NActors::TActorBootstrapped { const ui32 Step; NPersQueue::TTopicConverterPtr Topic; + TString Database; const ui32 Partition; const ui64 TabletID; @@ -237,6 +246,7 @@ class TPartitionActor : public NActors::TActorBootstrapped { ui64 EndOffset; ui64 SizeLag; + bool ClientHasAnyCommits = false; TString ReadGuid; // empty if not reading @@ -255,6 +265,9 @@ class TPartitionActor : public NActors::TActorBootstrapped { bool FirstRead; bool ReadingFinishedSent; + + std::unordered_map> Kqps; + std::set Parents; }; @@ -1057,8 +1070,8 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T IActor* partitionActor = new TPartitionActor( ctx.SelfID, InternalClientId, Cookie, Session, record.GetGeneration(), - record.GetStep(), jt->second.FullConverter, record.GetPartition(), record.GetTabletId(), it->second, - ClientDC + record.GetStep(), jt->second.FullConverter, Database.empty() ? NKikimr::NPQ::GetDatabaseFromConfig(AppData(ctx)->PQConfig) : Database, record.GetPartition(), record.GetTabletId(), it->second, + ClientDC, jt->second.PartitionGraph->GetPartition(record.GetPartition())->AllParents ); TActorId actorId = ctx.Register(partitionActor); @@ -1819,8 +1832,8 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadingFinished::TPtr& ev, const T TPartitionActor::TPartitionActor( const TActorId& parentId, const TString& internalClientId, const ui64 cookie, const TString& session, - const ui32 generation, const ui32 step, const NPersQueue::TTopicConverterPtr& topic, const ui32 partition, - const ui64 tabletID, const TReadSessionActor::TTopicCounters& counters, const TString& clientDC + const ui32 generation, const ui32 step, const NPersQueue::TTopicConverterPtr& topic, const TString& database, const ui32 partition, + const ui64 tabletID, const TReadSessionActor::TTopicCounters& counters, const TString& clientDC, std::set parents ) : ParentId(parentId) , InternalClientId(internalClientId) @@ -1830,6 +1843,7 @@ TPartitionActor::TPartitionActor( , Generation(generation) , Step(step) , Topic(topic) + , Database(database) , Partition(partition) , TabletID(tabletID) , ReadOffset(0) @@ -1856,6 +1870,7 @@ TPartitionActor::TPartitionActor( , Counters(counters) , FirstRead(true) , ReadingFinishedSent(false) + , Parents(parents) { } @@ -1888,28 +1903,82 @@ void TPartitionActor::CheckRelease(const TActorContext& ctx) { void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx) { - NKikimrClient::TPersQueueRequest request; - request.MutablePartitionRequest()->SetTopic(Topic->GetClientsideName()); - request.MutablePartitionRequest()->SetPartition(Partition); - request.MutablePartitionRequest()->SetCookie(readId); + if (!ClientHasAnyCommits && Parents.size() != 0) { + std::vector commits; + for (auto& parent: Parents) { + NKikimr::NGRpcProxy::V1::TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true, .ReadSessionId = Session}; + commits.push_back(commit); + } + NKikimr::NGRpcProxy::V1::TKqpHelper::TCommitInfo commit {.PartitionId = Partition, .Offset = (i64)offset, .KillReadSession = false, .OnlyCheckCommitedToFinish = false, .ReadSessionId = Session}; + commits.push_back(commit); + auto kqp = std::make_shared(Database, InternalClientId, Topic->GetPrimaryPath(), commits, readId); + Kqps.emplace(readId, kqp); - Y_ABORT_UNLESS(PipeClient); + kqp->SendCreateSessionRequest(ctx); + } else { + NKikimrClient::TPersQueueRequest request; + request.MutablePartitionRequest()->SetTopic(Topic->GetClientsideName()); + request.MutablePartitionRequest()->SetPartition(Partition); + request.MutablePartitionRequest()->SetCookie(readId); - ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient()); - auto commit = request.MutablePartitionRequest()->MutableCmdSetClientOffset(); - commit->SetClientId(InternalClientId); - commit->SetOffset(offset); - Y_ABORT_UNLESS(!Session.empty()); - commit->SetSessionId(Session); + Y_ABORT_UNLESS(PipeClient); + + ActorIdToProto(PipeClient, request.MutablePartitionRequest()->MutablePipeClient()); + auto commit = request.MutablePartitionRequest()->MutableCmdSetClientOffset(); + commit->SetClientId(InternalClientId); + commit->SetOffset(offset); + Y_ABORT_UNLESS(!Session.empty()); + commit->SetSessionId(Session); + + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" + << Partition << " committing to position " << offset << " prev " << CommittedOffset + << " end " << EndOffset << " by cookie " << readId); + + TAutoPtr req(new TEvPersQueue::TEvRequest); + req->Record.Swap(&request); + + NTabletPipe::SendData(ctx, PipeClient, req.Release()); + } +} + +void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { + if (CommitsInfly.empty()) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() + << " partition:" << Partition + << " unwaited commit-response with cookie " << cookie << "; waiting for nothing"); + return; + } + ui64 readId = CommitsInfly.front().first; + + if (cookie != readId) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() + << " partition:" << Partition + << " unwaited commit-response with cookie " << cookie << "; waiting for " << readId); + return; + } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" - << Partition << " committing to position " << offset << " prev " << CommittedOffset - << " end " << EndOffset << " by cookie " << readId); + Counters.Commits.Inc(); + ClientHasAnyCommits = true; + + CommittedOffset = CommitsInfly.front().second; + CommitsInfly.pop_front(); + if (readId != Max()) //this readId is reserved for upcommits on client skipping with ClientCommitOffset + ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() + << " partition:" << Partition + << " commit done to position " << CommittedOffset << " endOffset " << EndOffset << " with cookie " << readId); - TAutoPtr req(new TEvPersQueue::TEvRequest); - req->Record.Swap(&request); + while (!CommitsInfly.empty() && CommitsInfly.front().second == Max()) { //this is cookies that have no effect on this partition + readId = CommitsInfly.front().first; + CommitsInfly.pop_front(); + ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); + LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() + << "partition :" << Partition + << " commit done with no effect with cookie " << readId); + } - NTabletPipe::SendData(ctx, PipeClient, req.Release()); + CheckRelease(ctx); + PipeGeneration = 0; //reset tries counter - all ok } void TPartitionActor::SendPartitionReady(const TActorContext& ctx) { @@ -2050,6 +2119,7 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo if (resp.HasWriteTimestampMS()) WTime = resp.GetWriteTimestampMS(); WriteTimestampEstimateMs = resp.GetWriteTimestampEstimateMS(); + ClientHasAnyCommits = resp.GetClientHasAnyCommits(); InitDone = true; PipeGeneration = 0; //reset tries counter - all ok LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " INIT DONE " << Topic->GetPrintableString() @@ -2065,43 +2135,8 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo return; } - if (!result.HasCmdReadResult()) { //this is commit response - if (CommitsInfly.empty()) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited commit-response with cookie " << result.GetCookie() << "; waiting for nothing"); - return; - } - ui64 readId = CommitsInfly.front().first; - - if (result.GetCookie() != readId) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited commit-response with cookie " << result.GetCookie() << "; waiting for " << readId); - return; - } - - Counters.Commits.Inc(); - - CommittedOffset = CommitsInfly.front().second; - CommitsInfly.pop_front(); - if (readId != Max()) //this readId is reserved for upcommits on client skipping with ClientCommitOffset - ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " commit done to position " << CommittedOffset << " endOffset " << EndOffset << " with cookie " << readId); - - while (!CommitsInfly.empty() && CommitsInfly.front().second == Max()) { //this is cookies that have no effect on this partition - readId = CommitsInfly.front().first; - CommitsInfly.pop_front(); - ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << "partition :" << Partition - << " commit done with no effect with cookie " << readId); - } - - CheckRelease(ctx); - PipeGeneration = 0; //reset tries counter - all ok + if (!result.HasCmdReadResult()) { // this is commit response + CommitDone(result.GetCookie(), ctx); return; } @@ -2316,6 +2351,39 @@ void TPartitionActor::Handle(TEvPQProxy::TEvLockPartition::TPtr& ev, const TActo InitLockPartition(ctx); } +void TPartitionActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const NActors::TActorContext& ctx) { + auto kqpIt = Kqps.find(ev->Cookie); + if (kqpIt == Kqps.end()) { + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("unexpected cookie at KQP create session response", NPersQueue::NErrorCode::ERROR)); + return; + } + + if (!kqpIt->second->Handle(ev, ctx)) { + const auto& record = ev->Get()->Record; + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("status is not ok: " + record.GetError(), NPersQueue::NErrorCode::ERROR)); + } +} + +void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { + auto& record = ev->Get()->Record; + + auto kqpIt = Kqps.find(ev->Cookie); + if (kqpIt == Kqps.end()) { + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("unexpected cookie at KQP query response", NPersQueue::NErrorCode::ERROR)); + return; + } + + if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", NPersQueue::NErrorCode::ERROR)); // savnik: retry? + return; + } + + auto step = kqpIt->second->Handle(ev, ctx); + if (step == NKikimr::NGRpcProxy::V1::TKqpHelper::ECurrentStep::DONE) { + CommitDone(ev->Cookie, ctx); + } +} + void TPartitionActor::InitStartReading(const TActorContext& ctx) { Y_ABORT_UNLESS(AllPrepareInited); From f7d192b1b53619c7feb7893ea21752f696fd7dc9 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 26 Mar 2025 17:33:05 +0000 Subject: [PATCH 20/29] rename KQP helper --- .../ut/ut_with_sdk/autoscaling_ut.cpp | 105 +++++++++++------- .../persqueue_v0/grpc_pq_read_actor.cpp | 14 +-- .../actors/commit_offset_actor.cpp | 12 +- .../persqueue_v1/actors/commit_offset_actor.h | 4 +- ...lper.cpp => distributed_commit_helper.cpp} | 22 ++-- ...t_helper.h => distributed_commit_helper.h} | 4 +- .../persqueue_v1/actors/partition_actor.cpp | 10 +- .../persqueue_v1/actors/partition_actor.h | 4 +- ydb/services/persqueue_v1/actors/ya.make | 4 +- 9 files changed, 101 insertions(+), 78 deletions(-) rename ydb/services/persqueue_v1/actors/{kqp_commit_offset_helper.cpp => distributed_commit_helper.cpp} (78%) rename ydb/services/persqueue_v1/actors/{kqp_commit_offset_helper.h => distributed_commit_helper.h} (90%) diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index f8d3b8e605fc..c0cf7f53d508 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -652,17 +652,17 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { { UNIT_ASSERT(writeSession_1->Write(Msg(msg, 1))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, 2))); - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 1); } { UNIT_ASSERT(writeSession_1->Write(Msg(msg, 3))); - UNIT_ASSERT(writeSession_2->Write(Msg(msg, 4))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 4))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, 5))); UNIT_ASSERT(writeSession_2->Write(Msg(msg, 6))); - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } @@ -672,10 +672,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { { UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); - UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 8))); + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 8))); UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 9))); UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 10))); - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); auto describe2 = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe2.GetTopicDescription().GetPartitions().size(), 5); } @@ -690,17 +690,40 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto description = result.GetConsumerDescription(); UNIT_ASSERT(description.GetPartitions().size() == 5); - auto stats0 = description.GetPartitions().at(0).GetPartitionConsumerStats(); - UNIT_ASSERT(stats0); - UNIT_ASSERT(stats0->GetCommittedOffset() == 6); + auto stats_part_0_try_1 = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats_part_0_try_1); + UNIT_ASSERT(stats_part_0_try_1->GetCommittedOffset() == 6); - auto stats1 = description.GetPartitions().at(1).GetPartitionConsumerStats(); - UNIT_ASSERT(stats1); - UNIT_ASSERT(stats1->GetCommittedOffset() == 2); + auto stats_part_1_try_1 = description.GetPartitions().at(1).GetPartitionConsumerStats(); + UNIT_ASSERT(stats_part_1_try_1); + UNIT_ASSERT(stats_part_1_try_1->GetCommittedOffset() == 2); + + auto stats_part_3_try_1 = description.GetPartitions().at(3).GetPartitionConsumerStats(); + UNIT_ASSERT(stats_part_3_try_1); + UNIT_ASSERT(stats_part_3_try_1->GetCommittedOffset() == 0); + + + + auto status2 = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); + UNIT_ASSERT(status2.IsSuccess()); + + auto result2 = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result2.IsSuccess()); + + auto description2 = result2.GetConsumerDescription(); + UNIT_ASSERT(description2.GetPartitions().size() == 5); - auto stats3 = description.GetPartitions().at(3).GetPartitionConsumerStats(); - UNIT_ASSERT(stats3); - UNIT_ASSERT(stats3->GetCommittedOffset() == 0); + auto stats_part_0_try_2 = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats_part_0_try_2); + UNIT_ASSERT(stats_part_0_try_2->GetCommittedOffset() == 6); + + auto stats_part_1_try_2 = description.GetPartitions().at(1).GetPartitionConsumerStats(); + UNIT_ASSERT(stats_part_1_try_2); + UNIT_ASSERT(stats_part_1_try_2->GetCommittedOffset() == 2); + + auto stats_part_3_try_2 = description.GetPartitions().at(3).GetPartitionConsumerStats(); + UNIT_ASSERT(stats_part_3_try_2); + UNIT_ASSERT(stats_part_3_try_2->GetCommittedOffset() == 0); } Y_UNIT_TEST(CommitTopPast_BeforeAutoscaleAwareSDK) { @@ -985,10 +1008,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { { UNIT_ASSERT(writeSession_1->Write(Msg(msg, 3))); - UNIT_ASSERT(writeSession_2->Write(Msg(msg, 4))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 4))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, 5))); UNIT_ASSERT(writeSession_2->Write(Msg(msg, 6))); - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } @@ -998,10 +1021,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { { UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); - UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 8))); + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 8))); UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 9))); UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 10))); - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); auto describe2 = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe2.GetTopicDescription().GetPartitions().size(), 5); } @@ -1012,10 +1035,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { .AppendTopics(TTopicReadSettings(TEST_TOPIC)) .ConsumerName(TEST_CONSUMER)); - TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(15); + TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(5); auto count = 0; auto expected = 10; - while(deadlineTime > TInstant::Now()) { + while (deadlineTime > TInstant::Now()) { for (auto event : reader->GetEvents(false)) { if (auto* x = std::get_if(&event)) { auto& messages = x->GetMessages(); @@ -1023,6 +1046,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { count++; auto& message = messages[i]; message.Commit(); + Cerr << "SESSION EVENT read message: " << count << " from partition: " << message.GetPartitionSession()->GetPartitionId() << Endl << Flush; } } else if (auto* x = std::get_if(&event)) { x->Confirm(); @@ -1103,10 +1127,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { { UNIT_ASSERT(writeSession_1->Write(Msg(msg, 3))); - UNIT_ASSERT(writeSession_2->Write(Msg(msg, 4))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 4))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, 5))); UNIT_ASSERT(writeSession_2->Write(Msg(msg, 6))); - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } @@ -1116,10 +1140,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { { UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); - UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 8))); + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 8))); UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 9))); UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 10))); - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); auto describe2 = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe2.GetTopicDescription().GetPartitions().size(), 5); } @@ -1130,7 +1154,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { .AppendTopics(TTopicReadSettings(TEST_TOPIC)) .ConsumerName(TEST_CONSUMER)); - TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(15); + TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(5); auto count = 0; auto expected = 10; @@ -1144,7 +1168,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto& message = messages[i]; count++; int partitionId = message.GetPartitionSession()->GetPartitionId(); - + Cerr << "SESSION EVENT read message: " << count << " from partition: " << partitionId << Endl << Flush; if (partitionId == 1) { // Commit messages from partition 1 immediately message.Commit(); @@ -1218,7 +1242,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(stats1->GetCommittedOffset() == 4); } - Y_UNIT_TEST(PartitionSplit_DistributedTxCommit_CheckSessionReset) { + Y_UNIT_TEST(PartitionSplit_DistributedTxCommit_CheckSessionResetAfterCommit) { TTopicSdkTestSetup setup = CreateSetup(); TTopicClient client = setup.MakeClient(); @@ -1254,17 +1278,17 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { { UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); - UNIT_ASSERT(writeSession_2->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); UNIT_ASSERT(writeSession_2->Write(Msg(msg, seqNo++))); - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } auto writeSession_3 = CreateWriteSession(client, "producer-2", 1, std::string{TEST_TOPIC}, false); - UNIT_ASSERT(writeSession_3->Write(Msg("message", seqNo++))); - UNIT_ASSERT(writeSession_3->Write(Msg("message", seqNo++))); + UNIT_ASSERT(writeSession_3->Write(Msg(TStringBuilder() << "message-" << seqNo, seqNo++))); + UNIT_ASSERT(writeSession_3->Write(Msg(TStringBuilder() << "message-" << seqNo, seqNo++))); auto reader = client.CreateReadSession( TReadSessionSettings() @@ -1272,10 +1296,9 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { .AppendTopics(TTopicReadSettings(TEST_TOPIC)) .ConsumerName(TEST_CONSUMER)); - TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(15); + TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(5); auto commitSent = false; - while(deadlineTime > TInstant::Now()) { for (auto event : reader->GetEvents(false)) { if (auto* x = std::get_if(&event)) { @@ -1284,20 +1307,20 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { for (size_t i = 0u; i < messages.size(); ++i) { auto& message = messages[i]; message.Commit(); - - if (message.GetSeqNo() == 8) { + Cerr << "SESSION EVENT READ SeqNo: " << message.GetSeqNo() << Endl << Flush; + // check we get this SeqNo two times + if (message.GetSeqNo() == 6) { if (!commitSent) { commitSent = true; Sleep(TDuration::MilliSeconds(300)); - auto status = client.CommitOffset(TEST_TOPIC, 1, TEST_CONSUMER, 0).GetValueSync(); + auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0).GetValueSync(); UNIT_ASSERT(status.IsSuccess()); } else { return; } } } - UNIT_ASSERT(writeSession_3->Write(Msg(TStringBuilder() << "message-" << seqNo, seqNo))); - seqNo++; + UNIT_ASSERT(writeSession_3->Write(Msg(TStringBuilder() << "message-" << seqNo, seqNo++))); } else if (auto* x = std::get_if(&event)) { x->Confirm(); Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; @@ -1358,10 +1381,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { { UNIT_ASSERT(writeSession_1->Write(Msg(msg, 3))); - UNIT_ASSERT(writeSession_2->Write(Msg(msg, 4))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, 4))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, 5))); UNIT_ASSERT(writeSession_2->Write(Msg(msg, 6))); - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); } @@ -1371,7 +1394,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { { UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 7))); - UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 8))); + UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 8))); UNIT_ASSERT(writeSession2_1->Write(Msg(msg, 9))); UNIT_ASSERT(writeSession2_2->Write(Msg(msg, 10))); Sleep(TDuration::Seconds(5)); diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp index 0456a00b1103..c66fda87ff82 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include @@ -266,7 +266,7 @@ class TPartitionActor : public NActors::TActorBootstrapped { bool FirstRead; bool ReadingFinishedSent; - std::unordered_map> Kqps; + std::unordered_map> Kqps; std::set Parents; }; @@ -1904,14 +1904,14 @@ void TPartitionActor::CheckRelease(const TActorContext& ctx) { void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx) { if (!ClientHasAnyCommits && Parents.size() != 0) { - std::vector commits; + std::vector commits; for (auto& parent: Parents) { - NKikimr::NGRpcProxy::V1::TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true, .ReadSessionId = Session}; + NKikimr::NGRpcProxy::V1::TDistributedCommitHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true, .ReadSessionId = Session}; commits.push_back(commit); } - NKikimr::NGRpcProxy::V1::TKqpHelper::TCommitInfo commit {.PartitionId = Partition, .Offset = (i64)offset, .KillReadSession = false, .OnlyCheckCommitedToFinish = false, .ReadSessionId = Session}; + NKikimr::NGRpcProxy::V1::TDistributedCommitHelper::TCommitInfo commit {.PartitionId = Partition, .Offset = (i64)offset, .KillReadSession = false, .OnlyCheckCommitedToFinish = false, .ReadSessionId = Session}; commits.push_back(commit); - auto kqp = std::make_shared(Database, InternalClientId, Topic->GetPrimaryPath(), commits, readId); + auto kqp = std::make_shared(Database, InternalClientId, Topic->GetPrimaryPath(), commits, readId); Kqps.emplace(readId, kqp); kqp->SendCreateSessionRequest(ctx); @@ -2379,7 +2379,7 @@ void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TAc } auto step = kqpIt->second->Handle(ev, ctx); - if (step == NKikimr::NGRpcProxy::V1::TKqpHelper::ECurrentStep::DONE) { + if (step == NKikimr::NGRpcProxy::V1::TDistributedCommitHelper::ECurrentStep::DONE) { CommitDone(ev->Cookie, ctx); } } diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index ba771940442f..d977ae56f61e 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -108,23 +108,23 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc if (partitionNode->AllParents.size() == 0 && partitionNode->DirectChildren.size() == 0) { SendCommit(topicInitInfo, commitRequest, ctx); } else { - std::vector commits; + std::vector commits; for (auto& parent: partitionNode->AllParents) { - TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; + TDistributedCommitHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); } for (auto& child: partitionNode->AllChildren) { - TKqpHelper::TCommitInfo commit {.PartitionId = child->Id, .Offset = 0, .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; + TDistributedCommitHelper::TCommitInfo commit {.PartitionId = child->Id, .Offset = 0, .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); } - TKqpHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->offset(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; + TDistributedCommitHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->offset(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); // savnik if empty database? - Kqp = std::make_unique(Request().GetDatabaseName().GetOrElse(TString()), ClientId, topic, commits); // savnik add cookie? + Kqp = std::make_unique(Request().GetDatabaseName().GetOrElse(TString()), ClientId, topic, commits); // savnik add cookie? Kqp->SendCreateSessionRequest(ctx); } @@ -149,7 +149,7 @@ void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const auto step = Kqp->Handle(ev, ctx); - if (step == TKqpHelper::ECurrentStep::DONE) { + if (step == TDistributedCommitHelper::ECurrentStep::DONE) { Ydb::Topic::CommitOffsetResult result; Request().SendResult(result, Ydb::StatusIds::SUCCESS); Die(ctx); diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.h b/ydb/services/persqueue_v1/actors/commit_offset_actor.h index 597802ae0925..6daa6df3c54d 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.h +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.h @@ -1,7 +1,7 @@ #pragma once #include "events.h" -#include "kqp_commit_offset_helper.h" +#include "distributed_commit_helper.h" #include @@ -96,7 +96,7 @@ class TCommitOffsetActor : public TRpcOperationRequestActor Kqp; + std::unique_ptr Kqp; }; } diff --git a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp b/ydb/services/persqueue_v1/actors/distributed_commit_helper.cpp similarity index 78% rename from ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp rename to ydb/services/persqueue_v1/actors/distributed_commit_helper.cpp index 5f01010525ed..9034ae151252 100644 --- a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.cpp +++ b/ydb/services/persqueue_v1/actors/distributed_commit_helper.cpp @@ -1,9 +1,9 @@ -#include "kqp_commit_offset_helper.h" +#include "distributed_commit_helper.h" #include "ydb/core/kqp/common/simple/services.h" namespace NKikimr::NGRpcProxy::V1 { -TKqpHelper::TKqpHelper(TString database, TString consumer, TString path, std::vector commits, ui64 cookie) +TDistributedCommitHelper::TDistributedCommitHelper(TString database, TString consumer, TString path, std::vector commits, ui64 cookie) : DataBase(database) , Consumer(consumer) , Path(path) @@ -12,7 +12,7 @@ TKqpHelper::TKqpHelper(TString database, TString consumer, TString path, std::ve , Cookie(cookie) {} -TKqpHelper::ECurrentStep TKqpHelper::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { +TDistributedCommitHelper::ECurrentStep TDistributedCommitHelper::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { switch (Step) { case BEGIN_TRANSACTION_SENDED: Step = OFFSETS_SENDED; @@ -32,12 +32,12 @@ TKqpHelper::ECurrentStep TKqpHelper::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr return Step; } -void TKqpHelper::SendCreateSessionRequest(const TActorContext& ctx) { +void TDistributedCommitHelper::SendCreateSessionRequest(const TActorContext& ctx) { auto ev = MakeCreateSessionRequest(); ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release(), 0, Cookie); } -void TKqpHelper::BeginTransaction(const NActors::TActorContext& ctx) { +void TDistributedCommitHelper::BeginTransaction(const NActors::TActorContext& ctx) { auto begin = MakeHolder(); begin->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_BEGIN_TX); @@ -48,7 +48,7 @@ void TKqpHelper::BeginTransaction(const NActors::TActorContext& ctx) { ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), begin.Release(), 0, Cookie); } -bool TKqpHelper::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { +bool TDistributedCommitHelper::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { @@ -61,7 +61,7 @@ bool TKqpHelper::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const return true; } -void TKqpHelper::CloseKqpSession(const TActorContext& ctx) { +void TDistributedCommitHelper::CloseKqpSession(const TActorContext& ctx) { if (KqpSessionId) { auto ev = MakeCloseSessionRequest(); ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), ev.Release(), 0, Cookie); @@ -69,19 +69,19 @@ void TKqpHelper::CloseKqpSession(const TActorContext& ctx) { } } -THolder TKqpHelper::MakeCreateSessionRequest() { +THolder TDistributedCommitHelper::MakeCreateSessionRequest() { auto ev = MakeHolder(); ev->Record.MutableRequest()->SetDatabase(DataBase); return ev; } -THolder TKqpHelper::MakeCloseSessionRequest() { +THolder TDistributedCommitHelper::MakeCloseSessionRequest() { auto ev = MakeHolder(); ev->Record.MutableRequest()->SetSessionId(KqpSessionId); return ev; } -void TKqpHelper::SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const NActors::TActorContext& ctx) { +void TDistributedCommitHelper::SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const NActors::TActorContext& ctx) { auto& record = ev->Get()->Record; TxId = record.GetResponse().GetTxMeta().id(); Y_ABORT_UNLESS(!TxId.empty()); @@ -111,7 +111,7 @@ void TKqpHelper::SendCommits(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const NAc ctx.Send(NKqp::MakeKqpProxyID(ctx.SelfID.NodeId()), offsets.Release(), 0, Cookie); } -void TKqpHelper::CommitTx(const NActors::TActorContext& ctx) { +void TDistributedCommitHelper::CommitTx(const NActors::TActorContext& ctx) { auto commit = MakeHolder(); commit->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_COMMIT_TX); diff --git a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h b/ydb/services/persqueue_v1/actors/distributed_commit_helper.h similarity index 90% rename from ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h rename to ydb/services/persqueue_v1/actors/distributed_commit_helper.h index 09640c01581a..7b14d47c8e90 100644 --- a/ydb/services/persqueue_v1/actors/kqp_commit_offset_helper.h +++ b/ydb/services/persqueue_v1/actors/distributed_commit_helper.h @@ -9,7 +9,7 @@ namespace NKikimr::NGRpcProxy::V1 { using namespace NKikimr::NGRpcService; -class TKqpHelper { +class TDistributedCommitHelper { public: enum ECurrentStep { BEGIN_TRANSACTION_SENDED, @@ -26,7 +26,7 @@ class TKqpHelper { TString ReadSessionId; }; - TKqpHelper(TString database, TString consumer, TString path, std::vector commits, ui64 cookie = 0); + TDistributedCommitHelper(TString database, TString consumer, TString path, std::vector commits, ui64 cookie = 0); ECurrentStep Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx); void SendCreateSessionRequest(const TActorContext& ctx); diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index be22cff0bff7..5c5c9b8f6bad 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -161,14 +161,14 @@ void TPartitionActor::Bootstrap(const TActorContext& ctx) { void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx) { if (!ClientHasAnyCommits && Parents.size() != 0) { // savnik проверка на то что включены транзакции? - std::vector commits; + std::vector commits; for (auto& parent: Parents) { - TKqpHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true, .ReadSessionId = Session}; + TDistributedCommitHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true, .ReadSessionId = Session}; commits.push_back(commit); } - TKqpHelper::TCommitInfo commit {.PartitionId = Partition.Partition, .Offset = (i64)offset, .KillReadSession = false, .OnlyCheckCommitedToFinish = false, .ReadSessionId = Session}; + TDistributedCommitHelper::TCommitInfo commit {.PartitionId = Partition.Partition, .Offset = (i64)offset, .KillReadSession = false, .OnlyCheckCommitedToFinish = false, .ReadSessionId = Session}; commits.push_back(commit); - auto kqp = std::make_shared(Database, ClientId, Topic->GetPrimaryPath(), commits, readId); + auto kqp = std::make_shared(Database, ClientId, Topic->GetPrimaryPath(), commits, readId); Kqps.emplace(readId, kqp); kqp->SendCreateSessionRequest(ctx); @@ -226,7 +226,7 @@ void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TAc } auto step = kqpIt->second->Handle(ev, ctx); - if (step == TKqpHelper::ECurrentStep::DONE) { + if (step == TDistributedCommitHelper::ECurrentStep::DONE) { CommitDone(ev->Cookie, ctx); } } diff --git a/ydb/services/persqueue_v1/actors/partition_actor.h b/ydb/services/persqueue_v1/actors/partition_actor.h index f9ab74e10f9b..b2eec72172fc 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.h +++ b/ydb/services/persqueue_v1/actors/partition_actor.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include @@ -227,7 +227,7 @@ class TPartitionActor : public NActors::TActorBootstrapped { }; std::deque> CommitsInfly; //ReadId, Offset - std::unordered_map> Kqps; + std::unordered_map> Kqps; std::set Parents; diff --git a/ydb/services/persqueue_v1/actors/ya.make b/ydb/services/persqueue_v1/actors/ya.make index 6de67668912b..57adbeb98975 100644 --- a/ydb/services/persqueue_v1/actors/ya.make +++ b/ydb/services/persqueue_v1/actors/ya.make @@ -28,8 +28,8 @@ SRCS( codecs.cpp commit_offset_actor.h commit_offset_actor.cpp - kqp_commit_offset_helper.h - kqp_commit_offset_helper.cpp + distributed_commit_helper.h + distributed_commit_helper.cpp events.h persqueue_utils.h persqueue_utils.cpp From dc751e8d40d28cd0d28da1493263a6514a390dc1 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 26 Mar 2025 17:39:54 +0000 Subject: [PATCH 21/29] fix after rebase --- ydb/services/persqueue_v1/actors/read_session_actor.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.cpp b/ydb/services/persqueue_v1/actors/read_session_actor.cpp index 5b2fa0678354..6e0fd45bb4ed 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.cpp +++ b/ydb/services/persqueue_v1/actors/read_session_actor.cpp @@ -486,14 +486,6 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvDirectReadAc it->second.MaxProcessedDirectReadId = directReadId; it->second.PendingDirectReadAcks.push(directReadId); ProcessDirectReads(it, ctx); - it->second.MaxProcessedDirectReadId = ev->Get()->DirectReadId; - - BytesInflight_ -= drIt->second.ByteSize; - if (BytesInflight) { - (*BytesInflight) -= drIt->second.ByteSize; - } - it->second.DirectReads.erase(drIt); - ProcessReads(ctx); } From ed54e010ded124e40cd74a31d8cdb7b290744f85 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 26 Mar 2025 17:50:05 +0000 Subject: [PATCH 22/29] clean TODOs --- ydb/core/persqueue/partition.cpp | 2 +- .../deprecated/persqueue_v0/grpc_pq_read_actor.cpp | 2 +- ydb/services/persqueue_v1/actors/commit_offset_actor.cpp | 6 ++---- .../persqueue_v1/actors/distributed_commit_helper.cpp | 4 ++-- ydb/services/persqueue_v1/actors/partition_actor.cpp | 8 ++++---- ydb/services/persqueue_v1/actors/read_session_actor.cpp | 4 ++-- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index b229819c9c75..319fbfde78d0 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -2853,7 +2853,7 @@ TPartition::EProcessResult TPartition::PreProcessImmediateTx(const NKikimrPQ::TE return EProcessResult::Continue; } -void TPartition::ExecImmediateTx(TTransaction& t) // savnik +void TPartition::ExecImmediateTx(TTransaction& t) { --ImmediateTxCount; const auto& record = t.ProposeTransaction->GetRecord(); diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp index c66fda87ff82..db871d866592 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp @@ -2374,7 +2374,7 @@ void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TAc } if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { - ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", NPersQueue::NErrorCode::ERROR)); // savnik: retry? + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", NPersQueue::NErrorCode::ERROR)); return; } diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index d977ae56f61e..c5a9f018dbf1 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -123,9 +123,7 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc TDistributedCommitHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->offset(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); - // savnik if empty database? - Kqp = std::make_unique(Request().GetDatabaseName().GetOrElse(TString()), ClientId, topic, commits); // savnik add cookie? - + Kqp = std::make_unique(Request().GetDatabaseName().GetOrElse(TString()), ClientId, topic, commits); Kqp->SendCreateSessionRequest(ctx); } } @@ -141,7 +139,7 @@ void TCommitOffsetActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "strict CommitOffset failed. Kqp error: " << ev->Get()->Record); - Ydb::Topic::CommitOffsetResult result; // savnik: how to return exception? + Ydb::Topic::CommitOffsetResult result; Request().SendResult(result, record.GetYdbStatus()); Die(ctx); return; diff --git a/ydb/services/persqueue_v1/actors/distributed_commit_helper.cpp b/ydb/services/persqueue_v1/actors/distributed_commit_helper.cpp index 9034ae151252..eb6410dc55b2 100644 --- a/ydb/services/persqueue_v1/actors/distributed_commit_helper.cpp +++ b/ydb/services/persqueue_v1/actors/distributed_commit_helper.cpp @@ -7,7 +7,7 @@ TDistributedCommitHelper::TDistributedCommitHelper(TString database, TString con : DataBase(database) , Consumer(consumer) , Path(path) - , Commits(std::move(commits)) // savnik move + , Commits(std::move(commits)) , Step(BEGIN_TRANSACTION_SENDED) , Cookie(cookie) {} @@ -24,7 +24,7 @@ TDistributedCommitHelper::ECurrentStep TDistributedCommitHelper::Handle(NKqp::TE break; case COMMIT_SENDED: Step = DONE; - CloseKqpSession(ctx); // savnik wait? + CloseKqpSession(ctx); break; case DONE: break; diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index 5c5c9b8f6bad..fe5fcc104dc9 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -160,7 +160,7 @@ void TPartitionActor::Bootstrap(const TActorContext& ctx) { } void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TActorContext& ctx) { - if (!ClientHasAnyCommits && Parents.size() != 0) { // savnik проверка на то что включены транзакции? + if (!ClientHasAnyCommits && Parents.size() != 0) { std::vector commits; for (auto& parent: Parents) { TDistributedCommitHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = false, .OnlyCheckCommitedToFinish = true, .ReadSessionId = Session}; @@ -221,7 +221,7 @@ void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TAc } if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { - ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", PersQueue::ErrorCode::ERROR)); // savnik: retry? + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", PersQueue::ErrorCode::ERROR)); return; } @@ -386,7 +386,7 @@ void TPartitionActor::ResendRecentRequests() { NTabletPipe::SendData(ctx, PipeClient, event.Release()); } - if (InitDone) { // savnik need resend? + if (InitDone) { for (auto& c : CommitsInfly) { //resend all commits if (c.second.Offset != Max()) SendCommit(c.first, c.second.Offset, ctx); @@ -1373,7 +1373,7 @@ void TPartitionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContext& auto request = MakeReadRequest(ReadOffset, 0, req->MaxCount, req->MaxSize, req->MaxTimeLagMs, req->ReadTimestampMs, DirectReadId); RequestInfly = true; CurrentRequest = request; - + if (!PipeClient) //Pipe will be recreated soon return; diff --git a/ydb/services/persqueue_v1/actors/read_session_actor.cpp b/ydb/services/persqueue_v1/actors/read_session_actor.cpp index 6e0fd45bb4ed..4e838f22d710 100644 --- a/ydb/services/persqueue_v1/actors/read_session_actor.cpp +++ b/ydb/services/persqueue_v1/actors/read_session_actor.cpp @@ -1267,9 +1267,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartit return CloseSession(PersQueue::ErrorCode::ERROR, error, ctx); } - std::unordered_set notCommitedToFinishParents; // savnik: если сначала придет лок на детей, все сломается. Плохо? + std::unordered_set notCommitedToFinishParents; for (auto& parent: topic.PartitionGraph->GetPartition(record.GetPartition())->DirectParents) { - for (auto& otherPartitions: Partitions) { // savnik: to map + for (auto& otherPartitions: Partitions) { // TODO: map if (otherPartitions.second.Partition.Partition == parent->Id && otherPartitions.second.Offset != otherPartitions.second.EndOffset) { notCommitedToFinishParents.emplace(otherPartitions.second.Partition.Partition); } From 728f5e26533b3f03775f950fc1b8eff504faaaea Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Thu, 27 Mar 2025 19:52:27 +0000 Subject: [PATCH 23/29] pq_v0 fixes --- ydb/core/persqueue/partition.cpp | 8 ++--- .../deprecated/persqueue_v0/grpc_pq_actor.h | 2 ++ .../persqueue_v0/grpc_pq_read_actor.cpp | 33 +++++++++++++++++-- .../persqueue_v1/actors/partition_actor.cpp | 8 ++++- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 319fbfde78d0..e78aa8b835bd 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -840,7 +840,7 @@ void TPartition::Handle(TEvPQ::TEvPartitionStatus::TPtr& ev, const TActorContext } } } - result.SetScaleStatus(SplitMergeEnabled(TabletConfig) ? ScaleStatus :NKikimrPQ::EScaleStatus::NORMAL); + result.SetScaleStatus(SplitMergeEnabled(TabletConfig) ? ScaleStatus : NKikimrPQ::EScaleStatus::NORMAL); ctx.Send(ev->Get()->Sender, new TEvPQ::TEvPartitionStatusResponse(result, Partition)); } @@ -2887,7 +2887,7 @@ void TPartition::ExecImmediateTx(TTransaction& t) } TUserInfoBase& pendingUserInfo = GetOrCreatePendingUser(user); - if (operation.GetCommitOffsetsBegin() > operation.GetCommitOffsetsEnd()) { + if (!operation.GetForceCommit() && operation.GetCommitOffsetsBegin() > operation.GetCommitOffsetsEnd()) { ScheduleReplyPropose(record, NKikimrPQ::TEvProposeTransactionResult::BAD_REQUEST, NKikimrPQ::TError::BAD_REQUEST, @@ -2895,7 +2895,7 @@ void TPartition::ExecImmediateTx(TTransaction& t) return; } - if (pendingUserInfo.Offset != (i64)operation.GetCommitOffsetsBegin()) { + if (!operation.GetForceCommit() && pendingUserInfo.Offset != (i64)operation.GetCommitOffsetsBegin()) { ScheduleReplyPropose(record, NKikimrPQ::TEvProposeTransactionResult::ABORTED, NKikimrPQ::TError::BAD_REQUEST, @@ -2903,7 +2903,7 @@ void TPartition::ExecImmediateTx(TTransaction& t) return; } - if (operation.GetCommitOffsetsEnd() > EndOffset) { + if (!operation.GetForceCommit() && operation.GetCommitOffsetsEnd() > EndOffset) { ScheduleReplyPropose(record, NKikimrPQ::TEvProposeTransactionResult::BAD_REQUEST, NKikimrPQ::TError::BAD_REQUEST, diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_actor.h b/ydb/services/deprecated/persqueue_v0/grpc_pq_actor.h index ece96617d623..80fa87a54102 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_actor.h +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_actor.h @@ -943,6 +943,8 @@ class TReadSessionActor : public TActorBootstrapped { NPersQueue::TTopicsListController TopicsHandler; NPersQueue::TTopicsToConverter TopicsList; + + std::deque> Locks; }; } diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp index db871d866592..df203a86c673 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp @@ -987,6 +987,7 @@ void TReadSessionActor::Handle(V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const topicHolder.IsServerless = t.IsServerless; topicHolder.FolderId = t.FolderId; topicHolder.FullConverter = t.TopicNameConverter; + topicHolder.PartitionGraph = t.PartitionGraph; FullPathToConverter[t.TopicNameConverter->GetPrimaryPath()] = t.TopicNameConverter; const auto& second = t.TopicNameConverter->GetSecondaryPath(); if (!second.empty()) { @@ -1008,14 +1009,21 @@ void TReadSessionActor::Handle(V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const ctx.Schedule(Min(CommitInterval, CHECK_ACL_DELAY), new TEvents::TEvWakeup()); } else { for (auto& [name, t] : ev->Get()->TopicAndTablets) { - if (Topics.find(t.TopicNameConverter->GetInternalName()) == Topics.end()) { + auto it = Topics.find(t.TopicNameConverter->GetInternalName()); + if (it == Topics.end()) { CloseSession(TStringBuilder() << "list of topics changed - new topic '" << t.TopicNameConverter->GetInternalName() << "' found", NPersQueue::NErrorCode::BAD_REQUEST, ctx); return; } + it->second.PartitionGraph = t.PartitionGraph; } } + + while (!Locks.empty()) { + ctx.Send(ctx.SelfID, std::move(Locks.front())); + Locks.pop_front(); + } } @@ -1056,6 +1064,20 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T ); return; } + + auto* partitionNode = jt->second.PartitionGraph->GetPartition(record.GetPartition()); + if (!partitionNode) { + LOG_DEBUG_S( + ctx, NKikimrServices::PQ_READ_PROXY, + PQ_LOG_PREFIX << " lock for unknown partition = " << record.GetPartition() + ); + Locks.push_back(ev->Release()); + if (!AuthInflight) { + SendAuthRequest(ctx); + } + return; + } + // ToDo[counters] if (NumPartitionsFromTopic[intName]++ == 0) { if (AppData(ctx)->PQConfig.GetTopicsAreFirstClassCitizen()) { @@ -2374,7 +2396,14 @@ void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TAc } if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { - ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", NPersQueue::NErrorCode::ERROR)); + + auto kqpQueryError = TStringBuilder() << "Kqp error. Status# " << record.GetYdbStatus() << ", "; + + NYql::TIssues issues; + NYql::IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues); + kqpQueryError << issues.ToString(); + + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession(kqpQueryError, NPersQueue::NErrorCode::ERROR)); return; } diff --git a/ydb/services/persqueue_v1/actors/partition_actor.cpp b/ydb/services/persqueue_v1/actors/partition_actor.cpp index fe5fcc104dc9..22863b38308b 100644 --- a/ydb/services/persqueue_v1/actors/partition_actor.cpp +++ b/ydb/services/persqueue_v1/actors/partition_actor.cpp @@ -221,7 +221,13 @@ void TPartitionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TAc } if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { - ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession("KQP query response status is not ok", PersQueue::ErrorCode::ERROR)); + auto kqpQueryError = TStringBuilder() << "Kqp error. Status# " << record.GetYdbStatus() << ", "; + + NYql::TIssues issues; + NYql::IssuesFromMessage(record.GetResponse().GetQueryIssues(), issues); + kqpQueryError << issues.ToString(); + + ctx.Send(ParentId, new TEvPQProxy::TEvCloseSession(kqpQueryError, PersQueue::ErrorCode::ERROR)); return; } From 32b821f62142d7496c099eb3eb4cc029452774ef Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Fri, 28 Mar 2025 00:26:34 +0000 Subject: [PATCH 24/29] ReadSessionId parameter for CommitOffset API --- ydb/core/persqueue/partition.cpp | 4 +- .../ut/ut_with_sdk/autoscaling_ut.cpp | 113 ++++++++++++++++++ ydb/public/api/protos/ydb_topic.proto | 3 +- .../ydb-cpp-sdk/client/topic/control_plane.h | 4 +- .../cpp/src/client/topic/impl/topic_impl.h | 4 +- .../actors/commit_offset_actor.cpp | 16 ++- 6 files changed, 135 insertions(+), 9 deletions(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index e78aa8b835bd..44cb4bcd5f2d 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -2335,7 +2335,9 @@ TPartition::EProcessResult TPartition::BeginTransaction(const TEvPQ::TEvTxCalcPr PQ_LOG_D("Partition " << Partition << " Consumer '" << consumer << "'" << " Bad request (session already dead) " << - " SessionId '" << operation.GetReadSessionId() << "'"); + " RequestSessionId '" << operation.GetReadSessionId() << + " CurrentSessionId '" << userInfo.Session << + "'"); result = false; } else if (operation.GetOnlyCheckCommitedToFinish()) { if (IsActive() || static_cast(userInfo.Offset) != EndOffset) { diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index c0cf7f53d508..d80a49b45477 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -1348,6 +1348,119 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(false); } + Y_UNIT_TEST(PartitionSplit_DistributedTxCommit_CheckSessionNotResetedAfterCommitWithSessionId) { + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(1) + .MaxActivePartitions(100) + .BeginConfigureAutoPartitioningSettings() + .UpUtilizationPercent(2) + .DownUtilizationPercent(1) + .StabilizationWindow(TDuration::Seconds(2)) + .Strategy(EAutoPartitioningStrategy::ScaleUp) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings() + .BeginAddConsumer() + .ConsumerName(TEST_CONSUMER); + + client.CreateTopic(TEST_TOPIC, createSettings).Wait(); + + auto msg = TString(1_MB, 'a'); + + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, std::string{TEST_TOPIC}, false); + auto writeSession_2 = CreateWriteSession(client, "producer-2", 0, std::string{TEST_TOPIC}, false); + auto seqNo = 1; + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + Sleep(TDuration::Seconds(5)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 1); + } + + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, seqNo++))); + Sleep(TDuration::Seconds(15)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); + } + + auto writeSession_3 = CreateWriteSession(client, "producer-2", 1, std::string{TEST_TOPIC}, false); + UNIT_ASSERT(writeSession_3->Write(Msg(TStringBuilder() << "message-" << seqNo, seqNo++))); + UNIT_ASSERT(writeSession_3->Write(Msg(TStringBuilder() << "message-" << seqNo, seqNo++))); + + auto reader = client.CreateReadSession( + TReadSessionSettings() + .AutoPartitioningSupport(true) + .AppendTopics(TTopicReadSettings(TEST_TOPIC)) + .ConsumerName(TEST_CONSUMER)); + + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + + auto describeConsumerResult = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(describeConsumerResult.IsSuccess()); + + auto consumerDescription = describeConsumerResult.GetConsumerDescription(); + + TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(5); + + auto commitSent = false; + while(deadlineTime > TInstant::Now()) { + for (auto event : reader->GetEvents(false)) { + if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + auto& messages = x->GetMessages(); + for (size_t i = 0u; i < messages.size(); ++i) { + auto& message = messages[i]; + message.Commit(); + Cerr << "SESSION EVENT READ SeqNo: " << message.GetSeqNo() << Endl << Flush; + // check we NOT get this SeqNo two times + if (message.GetSeqNo() == 6) { + if (!commitSent) { + commitSent = true; + Sleep(TDuration::MilliSeconds(300)); + TCommitOffsetSettings commitSettings {.ReadSessionId_ = consumerDescription.GetPartitions().at(0).GetPartitionConsumerStats()->GetReadSessionId()}; + auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0, commitSettings).GetValueSync(); + UNIT_ASSERT(status.IsSuccess()); + } else { + UNIT_ASSERT(false); + } + } + } + UNIT_ASSERT(writeSession_3->Write(Msg(TStringBuilder() << "message-" << seqNo, seqNo++))); + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* sessionClosedEvent = std::get_if(&event)) { + Cerr << sessionClosedEvent->DebugString() << Endl << Flush; + } else { + Cerr << "SESSION EVENT unhandled \n"; + } + } + Sleep(TDuration::MilliSeconds(250)); + } + + } + Y_UNIT_TEST(PartitionSplit_AutosplitByLoad) { TTopicSdkTestSetup setup = CreateSetup(); TTopicClient client = setup.MakeClient(); diff --git a/ydb/public/api/protos/ydb_topic.proto b/ydb/public/api/protos/ydb_topic.proto index a46af6cdf2bd..afc765aaf29e 100644 --- a/ydb/public/api/protos/ydb_topic.proto +++ b/ydb/public/api/protos/ydb_topic.proto @@ -752,9 +752,10 @@ message CommitOffsetRequest { int64 partition_id = 3; // Path of consumer. string consumer = 4; - // Processed offset. int64 offset = 5; + // Read session identifier. + optional string read_session_id = 6; } // Commit offset response sent from server to client. diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h index 37ee274b22f3..c776e3f5b45a 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h @@ -766,6 +766,8 @@ struct TDescribePartitionSettings: public TOperationRequestSettings {}; +struct TCommitOffsetSettings : public TOperationRequestSettings { + FLUENT_SETTING_OPTIONAL(std::string, ReadSessionId); +}; } // namespace NYdb::NTopic diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h b/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h index 810c647c0786..5c2d927ea2fe 100644 --- a/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h +++ b/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h @@ -282,7 +282,9 @@ class TTopicClient::TImpl : public TClientImplCommon { request.set_partition_id(partitionId); request.set_consumer(TStringType{consumerName}); request.set_offset(offset); - + if (settings.ReadSessionId_) { + request.set_read_session_id(*settings.ReadSessionId_); + } return RunSimple( std::move(request), &Ydb::Topic::V1::TopicService::Stub::AsyncCommitOffset, diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index c5a9f018dbf1..bb393b093a65 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -29,7 +29,6 @@ TCommitOffsetActor::TCommitOffsetActor( } - TCommitOffsetActor::~TCommitOffsetActor() = default; @@ -76,7 +75,6 @@ void TCommitOffsetActor::Bootstrap(const TActorContext& ctx) { )); } - void TCommitOffsetActor::Die(const TActorContext& ctx) { if (PipeClient) NTabletPipe::CloseClient(ctx, PipeClient); @@ -108,19 +106,24 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc if (partitionNode->AllParents.size() == 0 && partitionNode->DirectChildren.size() == 0) { SendCommit(topicInitInfo, commitRequest, ctx); } else { + auto killReadSession = !commitRequest->has_read_session_id(); std::vector commits; for (auto& parent: partitionNode->AllParents) { - TDistributedCommitHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; + TDistributedCommitHelper::TCommitInfo commit {.PartitionId = parent->Id, .Offset = Max(), .KillReadSession = killReadSession, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); } for (auto& child: partitionNode->AllChildren) { - TDistributedCommitHelper::TCommitInfo commit {.PartitionId = child->Id, .Offset = 0, .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; + TDistributedCommitHelper::TCommitInfo commit {.PartitionId = child->Id, .Offset = 0, .KillReadSession = killReadSession, .OnlyCheckCommitedToFinish = false}; commits.push_back(commit); } - TDistributedCommitHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->offset(), .KillReadSession = true, .OnlyCheckCommitedToFinish = false}; + TDistributedCommitHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->offset(), .KillReadSession = killReadSession, .OnlyCheckCommitedToFinish = false}; + + if (commitRequest->has_read_session_id()) { + commit.ReadSessionId = commitRequest->read_session_id(); + } commits.push_back(commit); Kqp = std::make_unique(Request().GetDatabaseName().GetOrElse(TString()), ClientId, topic, commits); @@ -197,6 +200,9 @@ void TCommitOffsetActor::SendCommit(const TTopicInitInfo& topic, const Ydb::Topi commit->SetClientId(ClientId); commit->SetOffset(commitRequest->offset()); commit->SetStrict(true); + if (commitRequest->has_read_session_id()) { + commit->SetSessionId(commitRequest->read_session_id()); + } LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "strict CommitOffset, partition " << commitRequest->partition_id() << " committing to position " << commitRequest->offset() /*<< " prev " << CommittedOffset From 187497655e6c6ae1e247e6d2523e60812b4c9d33 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Fri, 28 Mar 2025 13:52:29 +0000 Subject: [PATCH 25/29] add readSessionId to TPartitionSession --- ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp | 9 +-------- .../cpp/include/ydb-cpp-sdk/client/topic/read_events.h | 6 ++++++ .../sdk/cpp/src/client/topic/impl/read_session_impl.h | 3 +++ .../cpp/src/client/topic/impl/read_session_impl.ipp | 10 ++++++++-- .../persqueue_v1/actors/commit_offset_actor.cpp | 6 +++--- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index d80a49b45477..18be6e39dd2b 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -1402,13 +1402,6 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { .AppendTopics(TTopicReadSettings(TEST_TOPIC)) .ConsumerName(TEST_CONSUMER)); - auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); - - auto describeConsumerResult = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); - UNIT_ASSERT(describeConsumerResult.IsSuccess()); - - auto consumerDescription = describeConsumerResult.GetConsumerDescription(); - TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(5); auto commitSent = false; @@ -1426,7 +1419,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { if (!commitSent) { commitSent = true; Sleep(TDuration::MilliSeconds(300)); - TCommitOffsetSettings commitSettings {.ReadSessionId_ = consumerDescription.GetPartitions().at(0).GetPartitionConsumerStats()->GetReadSessionId()}; + TCommitOffsetSettings commitSettings {.ReadSessionId_ = message.GetPartitionSession()->GetReadSessionId()}; auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0, commitSettings).GetValueSync(); UNIT_ASSERT(status.IsSuccess()); } else { diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_events.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_events.h index 9f4ab84f98c9..b619abeb5dc3 100644 --- a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_events.h +++ b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/read_events.h @@ -31,6 +31,11 @@ struct TPartitionSession: public TThrRefBase, public TPrintable { template > TPartitionStreamImpl(ui64 partitionStreamId, std::string topicPath, + std::string readSessionId, i64 partitionId, i64 assignId, i64 readOffset, @@ -617,6 +618,7 @@ class TPartitionStreamImpl : public TAPartitionStream { { TAPartitionStream::PartitionSessionId = partitionStreamId; TAPartitionStream::TopicPath = std::move(topicPath); + TAPartitionStream::ReadSessionId = std::move(readSessionId); TAPartitionStream::PartitionId = static_cast(partitionId); MaxCommittedOffset = static_cast(readOffset); } @@ -1333,6 +1335,7 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext::OnReadDoneImpl( LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Server session id: " << msg.session_id()); RetryState = nullptr; + ReadSessionId = msg.session_id(); // Successful init. Do nothing. ContinueReadingDataImpl(); @@ -1219,6 +1220,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( Y_UNUSED(deferred); RetryState = nullptr; + ReadSessionId = msg.session_id(); LOG_LAZY(Log, TLOG_INFO, GetLogPrefix() << "Server session id: " << msg.session_id()); @@ -1318,8 +1320,12 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( Y_ABORT_UNLESS(Lock.IsLocked()); auto partitionStream = MakeIntrusive>( - NextPartitionStreamId, msg.partition_session().path(), msg.partition_session().partition_id(), - msg.partition_session().partition_session_id(), msg.committed_offset(), + NextPartitionStreamId, + msg.partition_session().path(), + ReadSessionId, + msg.partition_session().partition_id(), + msg.partition_session().partition_session_id(), + msg.committed_offset(), SelfContext); NextPartitionStreamId += PartitionStreamIdStep; diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index bb393b093a65..b8646a72f68d 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -106,7 +106,7 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc if (partitionNode->AllParents.size() == 0 && partitionNode->DirectChildren.size() == 0) { SendCommit(topicInitInfo, commitRequest, ctx); } else { - auto killReadSession = !commitRequest->has_read_session_id(); + auto killReadSession = !(commitRequest->has_read_session_id() && !commitRequest->read_session_id().empty()); std::vector commits; for (auto& parent: partitionNode->AllParents) { @@ -121,7 +121,7 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc TDistributedCommitHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->offset(), .KillReadSession = killReadSession, .OnlyCheckCommitedToFinish = false}; - if (commitRequest->has_read_session_id()) { + if (commitRequest->has_read_session_id() && !commitRequest->read_session_id().empty()) { commit.ReadSessionId = commitRequest->read_session_id(); } commits.push_back(commit); @@ -200,7 +200,7 @@ void TCommitOffsetActor::SendCommit(const TTopicInitInfo& topic, const Ydb::Topi commit->SetClientId(ClientId); commit->SetOffset(commitRequest->offset()); commit->SetStrict(true); - if (commitRequest->has_read_session_id()) { + if (commitRequest->has_read_session_id() && !commitRequest->read_session_id().empty()) { commit->SetSessionId(commitRequest->read_session_id()); } From e07d7bfd4ce6c39ebbe621d030854bc622748a3d Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Tue, 1 Apr 2025 23:45:47 +0000 Subject: [PATCH 26/29] fix for commit to past --- ydb/core/persqueue/partition.cpp | 5 + .../ut/ut_with_sdk/autoscaling_ut.cpp | 246 +++++++++++++++++- 2 files changed, 248 insertions(+), 3 deletions(-) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 44cb4bcd5f2d..74263e2cb98c 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -2536,6 +2536,10 @@ void TPartition::CommitTransaction(TSimpleSharedPtr& t) Y_ABORT_UNLESS(userInfo.Offset == (i64)operation.GetCommitOffsetsBegin()); } + if ((i64)operation.GetCommitOffsetsEnd() < userInfo.Offset && !operation.GetReadSessionId().empty()) { + continue; // this is stale request, answer ok for it + } + if (operation.GetCommitOffsetsEnd() <= StartOffset) { userInfo.AnyCommits = false; userInfo.Offset = StartOffset; @@ -2546,6 +2550,7 @@ void TPartition::CommitTransaction(TSimpleSharedPtr& t) userInfo.AnyCommits = true; userInfo.Offset = operation.GetCommitOffsetsEnd(); } + if (operation.GetKillReadSession()) { userInfo.Session = ""; userInfo.PartitionSessionId = 0; diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index 18be6e39dd2b..12d537289d7b 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -1348,7 +1348,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(false); } - Y_UNIT_TEST(PartitionSplit_DistributedTxCommit_CheckSessionNotResetedAfterCommitWithSessionId) { + Y_UNIT_TEST(PartitionSplit_DistributedTxCommit_CheckOffsetCommitForDifferentCases_SplitedTopic) { TTopicSdkTestSetup setup = CreateSetup(); TTopicClient client = setup.MakeClient(); @@ -1383,6 +1383,8 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { } { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); @@ -1405,6 +1407,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(5); auto commitSent = false; + TString readSessionId = ""; while(deadlineTime > TInstant::Now()) { for (auto event : reader->GetEvents(false)) { if (auto* x = std::get_if(&event)) { @@ -1412,19 +1415,89 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto& messages = x->GetMessages(); for (size_t i = 0u; i < messages.size(); ++i) { auto& message = messages[i]; - message.Commit(); Cerr << "SESSION EVENT READ SeqNo: " << message.GetSeqNo() << Endl << Flush; + + if (commitSent) { + // read session not changed + UNIT_ASSERT_EQUAL(readSessionId, message.GetPartitionSession()->GetReadSessionId()); + } + // check we NOT get this SeqNo two times if (message.GetSeqNo() == 6) { if (!commitSent) { commitSent = true; Sleep(TDuration::MilliSeconds(300)); + + { + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + + auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats); + UNIT_ASSERT(stats->GetCommittedOffset() >= 5); + } + + readSessionId = message.GetPartitionSession()->GetReadSessionId(); TCommitOffsetSettings commitSettings {.ReadSessionId_ = message.GetPartitionSession()->GetReadSessionId()}; - auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0, commitSettings).GetValueSync(); + auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 8, commitSettings).GetValueSync(); UNIT_ASSERT(status.IsSuccess()); + + { + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + + auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats); + + UNIT_ASSERT(stats->GetCommittedOffset() == 8); + } + + // must be ignored, because commit to past + TCommitOffsetSettings commitToPastSettings {.ReadSessionId_ = message.GetPartitionSession()->GetReadSessionId()}; + auto commitToPastStatus = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0, commitToPastSettings).GetValueSync(); + UNIT_ASSERT(commitToPastStatus.IsSuccess()); + + { + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + + auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats); + + UNIT_ASSERT(stats->GetCommittedOffset() == 8); + } + + TCommitOffsetSettings commitSettingsWrongSession {.ReadSessionId_ = "random_session"}; + auto statusWrongSession = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0, commitSettingsWrongSession).GetValueSync(); + UNIT_ASSERT(!statusWrongSession.IsSuccess()); + + { + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + + auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats); + + UNIT_ASSERT(stats->GetCommittedOffset() == 8); + } + } else { UNIT_ASSERT(false); } + } else { + message.Commit(); } } UNIT_ASSERT(writeSession_3->Write(Msg(TStringBuilder() << "message-" << seqNo, seqNo++))); @@ -1451,9 +1524,176 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { } Sleep(TDuration::MilliSeconds(250)); } + } + + Y_UNIT_TEST(PartitionSplit_DistributedTxCommit_CheckOffsetCommitForDifferentCases_NotSplitedTopic) { + TTopicSdkTestSetup setup = CreateSetup(); + TTopicClient client = setup.MakeClient(); + + TCreateTopicSettings createSettings; + createSettings + .BeginConfigurePartitioningSettings() + .MinActivePartitions(1) + .MaxActivePartitions(100) + .BeginConfigureAutoPartitioningSettings() + .UpUtilizationPercent(2) + .DownUtilizationPercent(1) + .StabilizationWindow(TDuration::Seconds(2)) + .Strategy(EAutoPartitioningStrategy::ScaleUp) + .EndConfigureAutoPartitioningSettings() + .EndConfigurePartitioningSettings() + .BeginAddConsumer() + .ConsumerName(TEST_CONSUMER); + + client.CreateTopic(TEST_TOPIC, createSettings).Wait(); + + auto msg = TString(1_MB, 'a'); + + auto writeSession_1 = CreateWriteSession(client, "producer-1", 0, std::string{TEST_TOPIC}, false); + auto seqNo = 1; + { + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + Sleep(TDuration::Seconds(15)); + auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); + UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 1); + } + + auto reader = client.CreateReadSession( + TReadSessionSettings() + .AutoPartitioningSupport(true) + .AppendTopics(TTopicReadSettings(TEST_TOPIC)) + .ConsumerName(TEST_CONSUMER)); + + TInstant deadlineTime = TInstant::Now() + TDuration::Seconds(5); + + auto commitSent = false; + TString readSessionId = ""; + while(deadlineTime > TInstant::Now()) { + for (auto event : reader->GetEvents(false)) { + if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + auto& messages = x->GetMessages(); + for (size_t i = 0u; i < messages.size(); ++i) { + auto& message = messages[i]; + + if (commitSent) { + // read session not changed + UNIT_ASSERT_EQUAL(readSessionId, message.GetPartitionSession()->GetReadSessionId()); + } + + // check we NOT get this SeqNo two times + if (message.GetSeqNo() == 6) { + if (!commitSent) { + commitSent = true; + Sleep(TDuration::MilliSeconds(1000)); + + { + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + + auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats); + UNIT_ASSERT(stats->GetCommittedOffset() == 5); + } + + readSessionId = message.GetPartitionSession()->GetReadSessionId(); + TCommitOffsetSettings commitSettings {.ReadSessionId_ = message.GetPartitionSession()->GetReadSessionId()}; + auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 8, commitSettings).GetValueSync(); + UNIT_ASSERT(status.IsSuccess()); + + { + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + + auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats); + + UNIT_ASSERT(stats->GetCommittedOffset() == 8); + } + + // must be ignored, because commit to past + TCommitOffsetSettings commitToPastSettings {.ReadSessionId_ = message.GetPartitionSession()->GetReadSessionId()}; + auto commitToPastStatus = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0, commitToPastSettings).GetValueSync(); + UNIT_ASSERT(commitToPastStatus.IsSuccess()); + + { + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + + auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats); + + UNIT_ASSERT(stats->GetCommittedOffset() == 8); + } + + TCommitOffsetSettings commitSettingsWrongSession {.ReadSessionId_ = "random_session"}; + auto statusWrongSession = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 0, commitSettingsWrongSession).GetValueSync(); + UNIT_ASSERT(!statusWrongSession.IsSuccess()); + { + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); + UNIT_ASSERT(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + + auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); + UNIT_ASSERT(stats); + + UNIT_ASSERT(stats->GetCommittedOffset() == 8); + } + + } else { + UNIT_ASSERT(false); + } + } else { + message.Commit(); + } + } + UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* x = std::get_if(&event)) { + x->Confirm(); + Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; + } else if (auto* sessionClosedEvent = std::get_if(&event)) { + Cerr << sessionClosedEvent->DebugString() << Endl << Flush; + } else { + Cerr << "SESSION EVENT unhandled \n"; + } + } + Sleep(TDuration::MilliSeconds(250)); + } } + + Y_UNIT_TEST(PartitionSplit_AutosplitByLoad) { TTopicSdkTestSetup setup = CreateSetup(); TTopicClient client = setup.MakeClient(); From 69e99103b869153a9548b2066949f50e5d13d19f Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 2 Apr 2025 00:22:47 +0000 Subject: [PATCH 27/29] remove optional from read_session_id field --- ydb/public/api/protos/ydb_topic.proto | 4 ++-- ydb/services/persqueue_v1/actors/commit_offset_actor.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ydb/public/api/protos/ydb_topic.proto b/ydb/public/api/protos/ydb_topic.proto index afc765aaf29e..a45a8c673140 100644 --- a/ydb/public/api/protos/ydb_topic.proto +++ b/ydb/public/api/protos/ydb_topic.proto @@ -754,8 +754,8 @@ message CommitOffsetRequest { string consumer = 4; // Processed offset. int64 offset = 5; - // Read session identifier. - optional string read_session_id = 6; + // Read session identifier from StreamRead RPC. + string read_session_id = 6; } // Commit offset response sent from server to client. diff --git a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp index b8646a72f68d..471162c5f9ae 100644 --- a/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp +++ b/ydb/services/persqueue_v1/actors/commit_offset_actor.cpp @@ -106,7 +106,7 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc if (partitionNode->AllParents.size() == 0 && partitionNode->DirectChildren.size() == 0) { SendCommit(topicInitInfo, commitRequest, ctx); } else { - auto killReadSession = !(commitRequest->has_read_session_id() && !commitRequest->read_session_id().empty()); + auto killReadSession = commitRequest->read_session_id().empty(); std::vector commits; for (auto& parent: partitionNode->AllParents) { @@ -121,7 +121,7 @@ void TCommitOffsetActor::Handle(TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TAc TDistributedCommitHelper::TCommitInfo commit {.PartitionId = partitionNode->Id, .Offset = commitRequest->offset(), .KillReadSession = killReadSession, .OnlyCheckCommitedToFinish = false}; - if (commitRequest->has_read_session_id() && !commitRequest->read_session_id().empty()) { + if (!commitRequest->read_session_id().empty()) { commit.ReadSessionId = commitRequest->read_session_id(); } commits.push_back(commit); @@ -200,7 +200,7 @@ void TCommitOffsetActor::SendCommit(const TTopicInitInfo& topic, const Ydb::Topi commit->SetClientId(ClientId); commit->SetOffset(commitRequest->offset()); commit->SetStrict(true); - if (commitRequest->has_read_session_id() && !commitRequest->read_session_id().empty()) { + if (!commitRequest->read_session_id().empty()) { commit->SetSessionId(commitRequest->read_session_id()); } From 3ea8dd482a8d4e383672acac98339634b2a28c1c Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 2 Apr 2025 17:30:25 +0000 Subject: [PATCH 28/29] fix uts --- .../ut/ut_with_sdk/autoscaling_ut.cpp | 38 ++++--------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp index 12d537289d7b..018e6b30331d 100644 --- a/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp +++ b/ydb/core/persqueue/ut/ut_with_sdk/autoscaling_ut.cpp @@ -1388,7 +1388,10 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + writeSession_1->Close(); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, seqNo++))); + writeSession_2->Close(); Sleep(TDuration::Seconds(15)); auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 3); @@ -1428,18 +1431,6 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { commitSent = true; Sleep(TDuration::MilliSeconds(300)); - { - auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); - auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - auto description = result.GetConsumerDescription(); - - auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); - UNIT_ASSERT(stats); - UNIT_ASSERT(stats->GetCommittedOffset() >= 5); - } - readSessionId = message.GetPartitionSession()->GetReadSessionId(); TCommitOffsetSettings commitSettings {.ReadSessionId_ = message.GetPartitionSession()->GetReadSessionId()}; auto status = client.CommitOffset(TEST_TOPIC, 0, TEST_CONSUMER, 8, commitSettings).GetValueSync(); @@ -1560,11 +1551,13 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + writeSession_1->Close(); Sleep(TDuration::Seconds(15)); auto describe = client.DescribeTopic(TEST_TOPIC).GetValueSync(); UNIT_ASSERT_EQUAL(describe.GetTopicDescription().GetPartitions().size(), 1); } + auto writeSession_2 = CreateWriteSession(client, "producer-1", 0, std::string{TEST_TOPIC}, false); auto reader = client.CreateReadSession( TReadSessionSettings() .AutoPartitioningSupport(true) @@ -1592,19 +1585,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { if (message.GetSeqNo() == 6) { if (!commitSent) { commitSent = true; - Sleep(TDuration::MilliSeconds(1000)); - - { - auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); - auto result = client.DescribeConsumer(TEST_TOPIC, TEST_CONSUMER, describeConsumerSettings).GetValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - auto description = result.GetConsumerDescription(); - - auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); - UNIT_ASSERT(stats); - UNIT_ASSERT(stats->GetCommittedOffset() == 5); - } + Sleep(TDuration::MilliSeconds(300)); readSessionId = message.GetPartitionSession()->GetReadSessionId(); TCommitOffsetSettings commitSettings {.ReadSessionId_ = message.GetPartitionSession()->GetReadSessionId()}; @@ -1620,7 +1601,6 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); UNIT_ASSERT(stats); - UNIT_ASSERT(stats->GetCommittedOffset() == 8); } @@ -1638,7 +1618,6 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); UNIT_ASSERT(stats); - UNIT_ASSERT(stats->GetCommittedOffset() == 8); } @@ -1655,7 +1634,6 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { auto stats = description.GetPartitions().at(0).GetPartitionConsumerStats(); UNIT_ASSERT(stats); - UNIT_ASSERT(stats->GetCommittedOffset() == 8); } @@ -1666,7 +1644,7 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { message.Commit(); } } - UNIT_ASSERT(writeSession_1->Write(Msg(msg, seqNo++))); + UNIT_ASSERT(writeSession_2->Write(Msg(msg, seqNo++))); } else if (auto* x = std::get_if(&event)) { x->Confirm(); Cerr << "SESSION EVENT " << x->DebugString() << Endl << Flush; @@ -1692,8 +1670,6 @@ Y_UNIT_TEST_SUITE(TopicAutoscaling) { } } - - Y_UNIT_TEST(PartitionSplit_AutosplitByLoad) { TTopicSdkTestSetup setup = CreateSetup(); TTopicClient client = setup.MakeClient(); From f18d5988a895c5a28e8dab54aa2b919d2c782002 Mon Sep 17 00:00:00 2001 From: Nikita Saveliev Date: Wed, 2 Apr 2025 19:08:27 +0000 Subject: [PATCH 29/29] remove kill read session field from TCmdSetClientOffset --- ydb/core/persqueue/events/internal.h | 4 +--- ydb/core/protos/msgbus_pq.proto | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index ba3278254f65..5f8c15036a28 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -362,7 +362,7 @@ struct TEvPQ { TEvSetClientInfo(const ui64 cookie, const TString& clientId, const ui64 offset, const TString& sessionId, const ui64 partitionSessionId, const ui32 generation, const ui32 step, const TActorId& pipeClient, - ESetClientInfoType type = ESCI_OFFSET, ui64 readRuleGeneration = 0, bool strict = false, bool killReadSession = false) + ESetClientInfoType type = ESCI_OFFSET, ui64 readRuleGeneration = 0, bool strict = false) : Cookie(cookie) , ClientId(clientId) , Offset(offset) @@ -373,7 +373,6 @@ struct TEvPQ { , Type(type) , ReadRuleGeneration(readRuleGeneration) , Strict(strict) - , KillReadSession(killReadSession) , PipeClient(pipeClient) { } @@ -388,7 +387,6 @@ struct TEvPQ { ESetClientInfoType Type; ui64 ReadRuleGeneration; bool Strict; - bool KillReadSession; TActorId PipeClient; }; diff --git a/ydb/core/protos/msgbus_pq.proto b/ydb/core/protos/msgbus_pq.proto index 922c55e8f653..ae4fe4ad0e91 100644 --- a/ydb/core/protos/msgbus_pq.proto +++ b/ydb/core/protos/msgbus_pq.proto @@ -69,7 +69,6 @@ message TPersQueuePartitionRequest { optional bool MirrorerRequest = 10 [default = false]; optional bool Strict = 11 [default = false]; - optional bool KillReadSession = 12 [default = false]; } message TCmdGetClientOffset {